[Cluster-devel] [PATCH dlm 0/3] dlm_controld: add support for mark per node configuration

[Cluster-devel] [PATCH dlm 0/3] dlm_controld: add support for mark per node configuration

[Alexander Aring]

Hi,

this patch series adds support for set the in-kernel socket skb mark
value over dlm_controld. There exists two kinds of socket, one listen
socket and multiple peer sockets. Both can be set via the dlm config
file via "listen_mark" or multiple entries of:

node id=$NODEID mark=$MARK

whereas $NODEID is the corosync assigned nodeid. The given mark number
can be hexadecimal or decimal.

- Alex

Alexander Aring (3):
  dlm_controld: add support for unsigned int values
  dlm_controld: set listen skb mark setting
  dlm_controld: add support for per nodeid configuration

 dlm_controld/Makefile      |  3 +-
 dlm_controld/action.c      | 33 +++++++++++++--
 dlm_controld/dlm.conf.5    | 21 +++++++++
 dlm_controld/dlm_daemon.h  | 10 ++++-
 dlm_controld/main.c        | 12 ++++++
 dlm_controld/member.c      |  6 ++-
 dlm_controld/node_config.c | 87 ++++++++++++++++++++++++++++++++++++++
 dlm_controld/node_config.h | 33 +++++++++++++++
 8 files changed, 199 insertions(+), 6 deletions(-)
 create mode 100644 dlm_controld/node_config.c
 create mode 100644 dlm_controld/node_config.h

-- 
2.26.2

[Cluster-devel] [PATCH dlm 1/3] dlm_controld: add support for unsigned int values

[Alexander Aring]

This patch adds support for setting a unsigned integer value.
---
 dlm_controld/dlm_daemon.h | 4 ++++
 dlm_controld/main.c       | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 5b9a52da..9fe56df2 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -86,6 +86,7 @@ enum {
         req_arg_bool = 1,
         req_arg_int = 2,
         req_arg_str = 3,
+        req_arg_uint = 4,
 };
 
 enum {
@@ -125,6 +126,7 @@ struct dlm_option {
 
 	int use_int;
 	char *use_str;
+	unsigned int use_uint;
 
 	int default_int;
 	const char *default_str;
@@ -132,6 +134,7 @@ struct dlm_option {
 	int cli_set;
 	int cli_int;
 	char *cli_str;
+	unsigned int cli_uint;
 
 	int file_set;
 	int file_int;
@@ -141,6 +144,7 @@ struct dlm_option {
 EXTERN struct dlm_option dlm_options[dlm_options_max];
 #define opt(x) dlm_options[x].use_int
 #define opts(x) dlm_options[x].use_str
+#define optu(x) dlm_options[x].use_uint
 
 
 /* DLM_LOCKSPACE_LEN: maximum lockspace name length, from linux/dlmconstants.h.
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
index 8be6a4bc..b4f4ffb8 100644
--- a/dlm_controld/main.c
+++ b/dlm_controld/main.c
@@ -1972,6 +1972,9 @@ static void set_opt_cli(int argc, char **argv)
 		} else if (o->req_arg == req_arg_bool) {
 			o->cli_int = atoi(arg_str) ? 1 : 0;
 			o->use_int = o->cli_int;
+		} else if (o->req_arg == req_arg_uint) {
+			o->cli_uint = strtoul(arg_str, NULL, 0);
+			o->use_uint = o->cli_uint;
 		}
 	}
 
-- 
2.26.2

[Cluster-devel] [PATCH dlm 2/3] dlm_controld: set listen skb mark setting

[Alexander Aring]

This patch adds support to set the skb mark value for the in-kernel DLM
listen socket.
---
 dlm_controld/action.c     | 2 ++
 dlm_controld/dlm.conf.5   | 2 ++
 dlm_controld/dlm_daemon.h | 1 +
 dlm_controld/main.c       | 5 +++++
 4 files changed, 10 insertions(+)

diff --git a/dlm_controld/action.c b/dlm_controld/action.c
index ecd0d022..46747def 100644
--- a/dlm_controld/action.c
+++ b/dlm_controld/action.c
@@ -869,6 +869,8 @@ int setup_configfs_options(void)
 	if (proto_num == PROTO_SCTP)
 		set_proc_rmem();
 
+	set_configfs_cluster("mark", NULL, optu(mark_ind));
+
 	/* 
 	 * set clustername, recover_callbacks
 	 *
diff --git a/dlm_controld/dlm.conf.5 b/dlm_controld/dlm.conf.5
index 09492176..771951d4 100644
--- a/dlm_controld/dlm.conf.5
+++ b/dlm_controld/dlm.conf.5
@@ -40,6 +40,8 @@ protocol
 .br
 bind_all
 .br
+mark
+.br
 debug_logfile
 .br
 enable_plock
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 9fe56df2..86b33096 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -97,6 +97,7 @@ enum {
         protocol_ind,
         debug_logfile_ind,
 	bind_all_ind,
+        mark_ind,
         enable_fscontrol_ind,
         enable_plock_ind,
         plock_debug_ind,
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
index b4f4ffb8..022a6c7c 100644
--- a/dlm_controld/main.c
+++ b/dlm_controld/main.c
@@ -1732,6 +1732,11 @@ static void set_opt_defaults(void)
 			0, NULL,
 			""); /* do not advertise */
 
+	set_opt_default(mark_ind,
+			"mark", '\0', req_arg_uint,
+			0, NULL,
+			"set mark value for the DLM in-kernel listen socket");
+
 	set_opt_default(debug_logfile_ind,
 			"debug_logfile", 'L', no_arg,
 			0, NULL,
-- 
2.26.2

[Cluster-devel] [PATCH dlm 3/3] dlm_controld: add support for per nodeid configuration

[Alexander Aring]

This patch adds support to make a configuration per nodeid and key-value
pairs. As example this patch will introduce the key mark to set via configfs
comms per nodeid the SO_MARK socket option.
---
 dlm_controld/Makefile      |  3 +-
 dlm_controld/action.c      | 31 ++++++++++++--
 dlm_controld/dlm.conf.5    | 19 +++++++++
 dlm_controld/dlm_daemon.h  |  5 ++-
 dlm_controld/main.c        |  4 ++
 dlm_controld/member.c      |  6 ++-
 dlm_controld/node_config.c | 87 ++++++++++++++++++++++++++++++++++++++
 dlm_controld/node_config.h | 33 +++++++++++++++
 8 files changed, 182 insertions(+), 6 deletions(-)
 create mode 100644 dlm_controld/node_config.c
 create mode 100644 dlm_controld/node_config.h

diff --git a/dlm_controld/Makefile b/dlm_controld/Makefile
index 6081cf8b..fbc8926c 100644
--- a/dlm_controld/Makefile
+++ b/dlm_controld/Makefile
@@ -32,7 +32,8 @@ BIN_SOURCE = action.c \
              config.c \
              member.c \
              logging.c \
-             rbtree.c
+             rbtree.c \
+             node_config.c
 LIB_SOURCE = lib.c
 
 CFLAGS += -D_GNU_SOURCE -O2 -ggdb \
diff --git a/dlm_controld/action.c b/dlm_controld/action.c
index 46747def..90f282f7 100644
--- a/dlm_controld/action.c
+++ b/dlm_controld/action.c
@@ -570,15 +570,16 @@ static int add_configfs_base(void)
 	return rv;
 }
 
-int add_configfs_node(int nodeid, char *addr, int addrlen, int local)
+int add_configfs_node(int nodeid, char *addr, int addrlen, int local,
+		      uint32_t mark)
 {
 	char path[PATH_MAX];
 	char padded_addr[sizeof(struct sockaddr_storage)];
 	char buf[32];
 	int rv, fd;
 
-	log_debug("set_configfs_node %d %s local %d",
-		  nodeid, str_ip(addr), local);
+	log_debug("set_configfs_node %d %s local %d mark %" PRIu32,
+		  nodeid, str_ip(addr), local, mark);
 
 	/*
 	 * create comm dir for this node
@@ -639,6 +640,30 @@ int add_configfs_node(int nodeid, char *addr, int addrlen, int local)
 	}
 	close(fd);
 
+	/*
+	 * set skb mark for nodeid
+	 */
+
+	memset(path, 0, PATH_MAX);
+	snprintf(path, PATH_MAX, "%s/%d/mark", COMMS_DIR, nodeid);
+
+	fd = open(path, O_WRONLY);
+	if (fd < 0) {
+		log_error("%s: open failed: %d", path, errno);
+		return -1;
+	}
+
+	memset(buf, 0, sizeof(buf));
+	snprintf(buf, 32, "%" PRIu32, mark);
+
+	rv = do_write(fd, buf, strlen(buf));
+	if (rv < 0) {
+		log_error("%s: write failed: %d, %s", path, errno, buf);
+		close(fd);
+		return -1;
+	}
+	close(fd);
+
 	/*
 	 * set local
 	 */
diff --git a/dlm_controld/dlm.conf.5 b/dlm_controld/dlm.conf.5
index 771951d4..1ce0c644 100644
--- a/dlm_controld/dlm.conf.5
+++ b/dlm_controld/dlm.conf.5
@@ -392,6 +392,25 @@ master    foo node=2 weight=1
 In which case node 1 will master 2/3 of the total resources and node 2
 will master the other 1/3.
 
+.SS Node configuration
+
+Node configurations can be set by the node keyword followed of key-value
+pairs.
+
+.B Keys:
+
+.B mark
+The mark key can be used to set a specific mark value which is then used
+by the in-kernel DLM socket creation. This can be used to match for DLM
+specfic packets for e.g. routing.
+
+Example of setting a per socket value for nodeid 1 and a mark value
+of 42:
+
+node id=1 mark=42
+
+For local nodes this value doesn't have any effect.
+
 .SH SEE ALSO
 .BR dlm_controld (8),
 .BR dlm_tool (8)
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 86b33096..1fe88eec 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -40,6 +40,7 @@
 #include <sched.h>
 #include <signal.h>
 #include <dirent.h>
+#include <inttypes.h>
 #include <sys/sysmacros.h>
 
 #include <corosync/cpg.h>
@@ -48,6 +49,7 @@
 #include "libdlmcontrol.h"
 #include "dlm_controld.h"
 #include "fence_config.h"
+#include "node_config.h"
 #include "list.h"
 #include "rbtree.h"
 #include "linux_endian.h"
@@ -363,7 +365,8 @@ int set_sysfs_nodir(char *name, int val);
 int set_configfs_members(struct lockspace *ls, char *name,
 			 int new_count, int *new_members,
 			 int renew_count, int *renew_members);
-int add_configfs_node(int nodeid, char *addr, int addrlen, int local);
+int add_configfs_node(int nodeid, char *addr, int addrlen, int local,
+		      uint32_t mark);
 void del_configfs_node(int nodeid);
 void clear_configfs(void);
 int setup_configfs_options(void);
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
index 022a6c7c..b330f88d 100644
--- a/dlm_controld/main.c
+++ b/dlm_controld/main.c
@@ -2047,6 +2047,10 @@ int main(int argc, char **argv)
 	set_opt_cli(argc, argv);
 	set_opt_file(0);
 
+	rv = node_config_init(CONF_FILE_PATH);
+	if (rv)
+		return 1;
+
 	strcpy(fence_all_device.name, "fence_all");
 	strcpy(fence_all_device.agent, "dlm_stonith");
 	fence_all_device.unfence = 0;
diff --git a/dlm_controld/member.c b/dlm_controld/member.c
index da3a1f5b..1d5bfa3d 100644
--- a/dlm_controld/member.c
+++ b/dlm_controld/member.c
@@ -109,6 +109,7 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate,
 {
 	corosync_cfg_node_address_t addrs[MAX_NODE_ADDRESSES];
 	corosync_cfg_node_address_t *addrptr = addrs;
+	const struct node_config *nc;
 	cs_error_t err;
 	int i, j, num_addrs;
 	uint64_t now = monotime();
@@ -163,12 +164,15 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate,
 				continue;
 			}
 
+			nc = node_config_get(quorum_nodes[i]);
+
 			for (j = 0; j < num_addrs; j++) {
 				add_configfs_node(quorum_nodes[i],
 						  addrptr[j].address,
 						  addrptr[j].address_length,
 						  (quorum_nodes[i] ==
-						   our_nodeid));
+						   our_nodeid),
+						  nc->mark);
 			}
 		}
 	}
diff --git a/dlm_controld/node_config.c b/dlm_controld/node_config.c
new file mode 100644
index 00000000..7eb7f1f3
--- /dev/null
+++ b/dlm_controld/node_config.c
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2020 Red Hat, Inc.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v2 or (at your option) any later version.
+ */
+
+#include "dlm_daemon.h"
+
+#define MAX_LINE 4096
+
+static struct node_config nc[MAX_NODES];
+
+static const struct node_config nc_default = {
+	.mark = 0,
+};
+
+int node_config_init(const char *path)
+{
+	char line[MAX_LINE], tmp[MAX_LINE];
+	unsigned long mark;
+	FILE *file;
+	int nodeid;
+	int rv;
+
+	/* if no config file is given we assume default node configuration */
+	file = fopen(path, "r");
+	if (!file) {
+		log_debug("No config file %s, we assume default node configuration: mark %" PRIu32,
+			  path, nc_default.mark);
+		return 0;
+	}
+
+	while (fgets(line, MAX_LINE, file)) {
+		if (line[0] == '#')
+			continue;
+		if (line[0] == '\n')
+			continue;
+
+		if (!strncmp(line, "node", strlen("node"))) {
+			rv = sscanf(line, "node id=%d mark=%s" PRIu32, &nodeid, tmp);
+			if (rv < 2) {
+				log_error("Invalid configuration line: %s", line);
+				rv = -EINVAL;
+				goto out;
+			}
+
+			/* skip invalid nodeid's */
+			if (nodeid <= 0 || nodeid > MAX_NODES - 1)
+				continue;
+
+			mark = strtoul(tmp, NULL, 0);
+			if (mark == ULONG_MAX) {
+				log_error("Failed to pars mark value %s will use %" PRIu32,
+					  tmp, nc_default.mark);
+				mark = nc_default.mark;
+			}
+			nc[nodeid].mark = mark;
+
+			log_debug("parsed node config id=%d mark=%" PRIu32,
+				  nodeid, mark);
+		}
+	}
+
+	fclose(file);
+	return 0;
+
+out:
+	fclose(file);
+	return rv;
+}
+
+const struct node_config *node_config_get(int nodeid)
+{
+	if (nodeid <= 0 || nodeid > MAX_NODES - 1) {
+		log_debug("node config requested for id=%d returning defaults", nodeid);
+		return &nc_default;
+	}
+
+	return &nc[nodeid];
+}
+
+void node_config_free()
+{
+	memset(nc, 0, sizeof(nc));
+}
diff --git a/dlm_controld/node_config.h b/dlm_controld/node_config.h
new file mode 100644
index 00000000..3daf3298
--- /dev/null
+++ b/dlm_controld/node_config.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020 Red Hat, Inc.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v2 or (at your option) any later version.
+ */
+
+#ifndef _NODE_CONFIG_H_
+#define _NODE_CONFIG_H_
+
+#include <stdint.h>
+
+struct node_config {
+	uint32_t mark;
+};
+
+/*
+ * Returns -ENOENT if path does not exist or there is no
+ * config for nodeid in the file.
+ *
+ * Returns -EXYZ if there's a problem with the config.
+ *
+ * Returns 0 if a config was found with no problems.
+ */
+
+int node_config_init(const char *path);
+
+const struct node_config *node_config_get(int nodeid);
+
+void node_config_free(void);
+
+#endif
-- 
2.26.2

[Cluster-devel] [PATCH dlm 3/3] dlm_controld: add support for per nodeid configuration

[Alexander Ahring Oder Aring]

Hi,

On Tue, Jun 16, 2020 at 1:07 PM Alexander Aring <aahringo@redhat.com> wrote:
>
> This patch adds support to make a configuration per nodeid and key-value
> pairs. As example this patch will introduce the key mark to set via configfs
> comms per nodeid the SO_MARK socket option.
> ---
>  dlm_controld/Makefile      |  3 +-
>  dlm_controld/action.c      | 31 ++++++++++++--
>  dlm_controld/dlm.conf.5    | 19 +++++++++
>  dlm_controld/dlm_daemon.h  |  5 ++-
>  dlm_controld/main.c        |  4 ++
>  dlm_controld/member.c      |  6 ++-
>  dlm_controld/node_config.c | 87 ++++++++++++++++++++++++++++++++++++++
>  dlm_controld/node_config.h | 33 +++++++++++++++
>  8 files changed, 182 insertions(+), 6 deletions(-)
>  create mode 100644 dlm_controld/node_config.c
>  create mode 100644 dlm_controld/node_config.h
>
...
> +
> +int node_config_init(const char *path)
> +{
> +       char line[MAX_LINE], tmp[MAX_LINE];
> +       unsigned long mark;
> +       FILE *file;
> +       int nodeid;
> +       int rv;
> +
> +       /* if no config file is given we assume default node configuration */
> +       file = fopen(path, "r");
> +       if (!file) {
> +               log_debug("No config file %s, we assume default node configuration: mark %" PRIu32,
> +                         path, nc_default.mark);
> +               return 0;
> +       }
> +
> +       while (fgets(line, MAX_LINE, file)) {
> +               if (line[0] == '#')
> +                       continue;
> +               if (line[0] == '\n')
> +                       continue;
> +
> +               if (!strncmp(line, "node", strlen("node"))) {
> +                       rv = sscanf(line, "node id=%d mark=%s" PRIu32, &nodeid, tmp);

I will send a v2 which removes the PRIu32 here, this was a leftover
from previous code. I changed the code to deal with hexadecimal values
from user input as well.

- Alex