[Patch] tabled: parse Chunk parameters in CLD

[Pete Zaitcev <zaitcev@redhat.com>]

This patch probably has all that's necessary to flip the configuration
to use CLD to configure available Chunk nodes and dispose with
<StorageNode> forever. However, it did not complete all tests that's
necessary to declare victory. So for now we continue to use <StorageNode>.
This should be relaiable in Koji, but we have -D in build scripts for now.

The parsing itself uses the same code that our parser of the static
configuration uses, although it appears resistant to refactoring without
going full tilt to a table-driven system, and so we copy-pasted a bunch
from config.c to storparse.c.

Signed-Off-By: Pete Zaitcev <zaitcev@redhat.com>

diff --git a/server/Makefile.am b/server/Makefile.am
index e01f728..f994b36 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -4,7 +4,8 @@ INCLUDES	= -I$(top_srcdir)/include @GLIB_CFLAGS@ @CHUNKDC_CFLAGS@ @CLDC_CFLAGS@
 sbin_PROGRAMS	= tabled tdbadm
 
 tabled_SOURCES	= tabled.h		\
-		  bucket.c object.c server.c storage.c cldu.c config.c util.c
+		  bucket.c object.c server.c storage.c storparse.c cldu.c \
+		  config.c util.c
 tabled_LDADD	= ../lib/libhttputil.a ../lib/libtdb.a		\
 		  @CHUNKDC_LIBS@ @CLDC_LIBS@ @PCRE_LIBS@ @GLIB_LIBS@ \
 		  @CRYPTO_LIBS@ @DB4_LIBS@ @EVENT_LIBS@ @ARGP_LIBS@ @SSL_LIBS@
diff --git a/server/cldu.c b/server/cldu.c
index bbb651d..b275234 100644
--- a/server/cldu.c
+++ b/server/cldu.c
@@ -5,6 +5,7 @@
 #include "tabled-config.h"
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <sys/time.h>
 #include <glib.h>
 #include <syslog.h>
 #include <string.h>
@@ -33,11 +34,14 @@ struct cld_session {
 	bool forced_hosts;		/* Administrator overrode default CLD */
 	bool sess_open;
 	struct cldc_udp *lib;		/* library state */
+	struct event tm;
+	int retry_cnt;
 
 	int actx;		/* Active host cldv[actx] */
 	struct cld_host cldv[N_CLD];
 
 	char *thiscell;
+	char *thishost;
 	struct event ev;	/* Associated with fd */
 	char *cfname;		/* /tabled-cell directory */
 	struct cldc_fh *cfh;	/* /tabled-cell directory, keep open for scan */
@@ -49,8 +53,6 @@ struct cld_session {
 	struct cldc_fh *yfh;	/* /chunk-cell/NID file */
 
 	struct list_head chunks;	/* found in xfname, struct chunk_node */
-
-	void (*state_cb)(enum st_cld);
 };
 
 static int cldu_set_cldc(struct cld_session *sp, int newactive);
@@ -60,6 +62,7 @@ static int cldu_open_f_cb(struct cldc_call_opts *carg, enum cle_err_codes errc);
 static int cldu_lock_cb(struct cldc_call_opts *carg, enum cle_err_codes errc);
 static int cldu_put_cb(struct cldc_call_opts *carg, enum cle_err_codes errc);
 static int cldu_get_1_cb(struct cldc_call_opts *carg, enum cle_err_codes errc);
+static void try_open_x(struct cld_session *sp);
 static int cldu_open_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc);
 static int cldu_get_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc);
 static int cldu_close_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc);
@@ -70,6 +73,8 @@ static int cldu_close_y_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
 static void add_remote(const char *name);
 static void add_chunk_node(struct cld_session *sp, const char *name);
 
+static struct timeval cldu_retry_delay = { 5, 0 };
+
 /*
  * Identify the next host to be tried.
  *
@@ -104,6 +109,9 @@ static int cldu_setcell(struct cld_session *sp,
 	sp->thiscell = strdup(thiscell);
 	if (!sp->thiscell)
 		goto err_oom;
+	sp->thishost = strdup(thishost);
+	if (!sp->thishost)
+		goto err_oom;
 
 	if (asprintf(&mem, "/tabled-%s", thiscell) == -1)
 		goto err_oom;
@@ -124,6 +132,15 @@ err_oom:
 	return 0;
 }
 
+static void cldu_timer(int fd, short events, void *userdata)
+{
+	struct cld_session *sp = userdata;
+
+	if (debugging)
+		applog(LOG_DEBUG, "Trying to open %s\n", sp->xfname);
+	try_open_x(sp);
+}
+
 static void cldu_event(int fd, short events, void *userdata)
 {
 	struct cld_session *sp = userdata;
@@ -451,8 +468,6 @@ static int cldu_put_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
 static int cldu_get_1_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
 {
 	struct cld_session *sp = carg->private;
-	struct cldc_call_opts copts;
-	int rc;
 	const char *ptr;
 	int dir_len;
 	int total_len, rec_len, name_len;
@@ -479,7 +494,7 @@ static int cldu_get_1_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
 		else
 			buf[64] = 0;
 
-		if (!strcmp(buf, tabled_srv.ourhost)) {	/* use thishost XXX */
+		if (!strcmp(buf, sp->thishost)) {
 			if (debugging)
 				applog(LOG_DEBUG, " %s (ourselves)", buf);
 		} else {
@@ -492,12 +507,32 @@ static int cldu_get_1_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
 		dir_len -= total_len;
 	}
 
-	if (sp->state_cb)
-		(*sp->state_cb)(ST_CLD_ACTIVE);
-
 	/*
-	 * Now we can collect the Chunk nodes in our cell.
+	 * This will go away with the demise of <StorageNode>.
 	 */
+	if (tabled_srv.num_stor) {
+		stor_update_cb();
+		return 0;
+	}
+
+	sp->retry_cnt = 0;
+	try_open_x(sp);
+	return 0;
+}
+
+/*
+ * Open the xfname, so we can collect registered Chunk servers.
+ */
+static void try_open_x(struct cld_session *sp)
+{
+	struct cldc_call_opts copts;
+	int rc;
+
+	if (++sp->retry_cnt >= 5) {
+		applog(LOG_INFO, "Out of retries for %s, bailing", sp->xfname);
+		exit(1);
+	}
+
 	memset(&copts, 0, sizeof(copts));
 	copts.cb = cldu_open_x_cb;
 	copts.private = sp;
@@ -507,7 +542,6 @@ static int cldu_get_1_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
 	if (rc) {
 		applog(LOG_ERR, "cldc_open(%s) call error: %d", sp->xfname, rc);
 	}
-	return 0;
 }
 
 static int cldu_open_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
@@ -517,8 +551,14 @@ static int cldu_open_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
 	int rc;
 
 	if (errc != CLE_OK) {
-		applog(LOG_ERR, "CLD open(%s) failed: %d", sp->xfname, errc);
-		/* XXX recycle, maybe Chunks aren't up yet. */
+		if (errc == CLE_INODE_INVAL) {
+			applog(LOG_ERR, "CLD open(%s) failed: %d, retrying",
+			       sp->xfname, errc);
+			evtimer_add(&sp->tm, &cldu_retry_delay);
+		} else {
+			applog(LOG_ERR, "CLD open(%s) failed: %d",
+			       sp->xfname, errc);
+		}
 		return 0;
 	}
 	if (sp->xfh == NULL) {
@@ -530,7 +570,7 @@ static int cldu_open_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
 		return 0;
 	}
 
-	// if (debugging)
+	if (debugging)
 		applog(LOG_DEBUG, "CLD directory \"%s\" opened", sp->xfname);
 
 	/*
@@ -606,10 +646,12 @@ static int cldu_close_x_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
 		return 0;
 	}
 
-	if (list_empty(&sp->chunks))
-		applog(LOG_INFO, "No Chunk nodes found");
-	else
+	if (list_empty(&sp->chunks)) {
+		applog(LOG_INFO, "%s: No Chunk nodes found", sp->xfname);
+		try_open_x(sp);
+	} else {
 		next_chunk(sp);
+	}
 	return 0;
 }
 
@@ -698,7 +740,7 @@ static int cldu_get_y_cb(struct cldc_call_opts *carg, enum cle_err_codes errc)
 	if (debugging)
 		applog(LOG_DEBUG,
 		       "got %d bytes from %s\n", len, sp->yfname);
-	stor_add_node(ptr, len);
+	stor_parse(sp->yfname, ptr, len);
 
 close_and_next:
 	memset(&copts, 0, sizeof(copts));
@@ -777,29 +819,31 @@ static void add_chunk_node(struct cld_session *sp, const char *name)
 static struct cld_session ses;
 
 /*
- * This initiates our sole session with a CLD instance.
+ * Global and 1-instance initialization.
  */
-int cld_begin(const char *thishost, const char *thiscell,
-	      void (*cb)(enum st_cld))
+void cld_init()
 {
-
 	cldc_init();
-	INIT_LIST_HEAD(&ses.chunks);
 
-	/*
-	 * As long as we permit pre-seeding lists of CLD hosts,
-	 * we cannot wipe our session anymore. Note though, as long
-	 * as cld_end terminates it right, we can call cld_begin again.
-	 */
 	// memset(&ses, 0, sizeof(struct cld_session));
-	ses.state_cb = cb;
+	INIT_LIST_HEAD(&ses.chunks);
+}
 
-	if (cldu_setcell(&ses, thiscell, thishost)) {
+/*
+ * This initiates our sole session with a CLD instance.
+ */
+int cld_begin(const char *thishost, const char *thiscell)
+{
+	static struct cld_session *sp = &ses;
+
+	evtimer_set(&ses.tm, cldu_timer, &ses);
+
+	if (cldu_setcell(sp, thiscell, thishost)) {
 		/* Already logged error */
 		goto err_cell;
 	}
 
-	if (!ses.forced_hosts) {
+	if (!sp->forced_hosts) {
 		GList *tmp, *host_list = NULL;
 		int i;
 
@@ -815,9 +859,9 @@ int cld_begin(const char *thishost, const char *thiscell,
 		for (tmp = host_list; tmp; tmp = tmp->next) {
 			struct cldc_host *hp = tmp->data;
 			if (i < N_CLD) {
-				memcpy(&ses.cldv[i].h, hp,
+				memcpy(&sp->cldv[i].h, hp,
 				       sizeof(struct cldc_host));
-				ses.cldv[i].known = 1;
+				sp->cldv[i].known = 1;
 				i++;
 			} else {
 				free(hp->host);
@@ -834,7 +878,7 @@ int cld_begin(const char *thishost, const char *thiscell,
 	 * -- Actually, it only works when recovering from CLD failure.
 	 *    Thereafter, any slave CLD redirects us to the master.
 	 */
-	if (cldu_set_cldc(&ses, 0)) {
+	if (cldu_set_cldc(sp, 0)) {
 		/* Already logged error */
 		goto err_net;
 	}
@@ -847,32 +891,6 @@ err_cell:
 	return -1;
 }
 
-void cld_end(void)
-{
-	int i;
-
-	if (ses.lib) {
-		event_del(&ses.ev);
-		// if (ses.sess_open)	/* kill it always, include half-open */
-		cldc_kill_sess(ses.lib->sess);
-		cldc_udp_free(ses.lib);
-		ses.lib = NULL;
-	}
-
-	if (!ses.forced_hosts) {
-		for (i = 0; i < N_CLD; i++) {
-			if (ses.cldv[i].known)
-				free(ses.cldv[i].h.host);
-		}
-	}
-
-	free(ses.cfname);
-	free(ses.ffname);
-	free(ses.xfname);
-	free(ses.yfname);
-	free(ses.thiscell);
-}
-
 void cldu_add_host(const char *hostname, unsigned int port)
 {
 	static struct cld_session *sp = &ses;
@@ -894,3 +912,41 @@ void cldu_add_host(const char *hostname, unsigned int port)
 
 	sp->forced_hosts = true;
 }
+
+void cld_end(void)
+{
+	static struct cld_session *sp = &ses;
+	int i;
+
+	if (sp->lib) {
+		event_del(&sp->ev);
+		// if (sp->sess_open)	/* kill it always, include half-open */
+		cldc_kill_sess(sp->lib->sess);
+		cldc_udp_free(sp->lib);
+		sp->lib = NULL;
+	}
+
+	if (!sp->forced_hosts) {
+		for (i = 0; i < N_CLD; i++) {
+			if (sp->cldv[i].known) {
+				free(sp->cldv[i].h.host);
+				sp->cldv[i].known = false;
+			}
+		}
+	}
+
+	evtimer_del(&sp->tm);
+
+	free(sp->cfname);
+	sp->cfname = NULL;
+	free(sp->ffname);
+	sp->ffname = NULL;
+	free(sp->xfname);
+	sp->xfname = NULL;
+	free(sp->yfname);
+	sp->yfname = NULL;
+	free(sp->thiscell);
+	sp->thiscell = NULL;
+	free(sp->thishost);
+	sp->thishost = NULL;
+}
diff --git a/server/config.c b/server/config.c
index 7ef9c1a..45cee26 100644
--- a/server/config.c
+++ b/server/config.c
@@ -6,14 +6,11 @@
 #include "tabled-config.h"
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <sys/socket.h>
 #include <glib.h>
 #include <syslog.h>
 #include <string.h>
 #include <stdio.h>
-#include <unistd.h>
 #include <errno.h>
-#include <netdb.h>
 #include <ctype.h>
 #include "tabled.h"
 
@@ -24,6 +21,7 @@ struct config_context {
 	struct listen_cfg tmp_listen;
 
 	bool		in_storage;
+	unsigned int	stor_nid;
 	char		*stor_port;
 	char		*stor_host;
 
@@ -51,6 +49,7 @@ static void cfg_elm_start (GMarkupParseContext *context,
 	else if (!strcmp(element_name, "StorageNode")) {
 		if (!cc->in_storage) {
 			cc->in_storage = true;
+			cc->stor_nid++;
 		} else {
 			applog(LOG_ERR, "Nested StorageNode in configuration");
 		}
@@ -64,71 +63,10 @@ static void cfg_elm_start (GMarkupParseContext *context,
 	}
 }
 
-static void cfg_add_storage(const char *hostname, const char *portstr)
-{
-	struct addrinfo hints;
-	struct addrinfo *res, *res0;
-	struct storage_node *sn;
-	int rc;
-
-	memset(&hints, 0, sizeof(struct addrinfo));
-	hints.ai_family = PF_UNSPEC;
-	hints.ai_socktype = SOCK_DGRAM;
-
-	rc = getaddrinfo(hostname, portstr, &hints, &res0);
-	if (rc) {
-		applog(LOG_WARNING, "getaddrinfo(%s:%s) failed: %s",
-		       hostname, portstr, gai_strerror(rc));
-		return;
-	}
-
-	for (res = res0; res; res = res->ai_next) {
-		if (res->ai_family != AF_INET && res->ai_family != AF_INET6)
-			continue;
-
-		if (res->ai_addrlen > ADDRSIZE)		/* should not happen */
-			continue;
-
-		if ((sn = malloc(sizeof(struct storage_node))) == NULL) {
-			applog(LOG_WARNING, "No core (%ld)",
-			       (long) sizeof(struct storage_node));
-			break;
-		}
-		memset(sn, 0, sizeof(struct storage_node));
-
-		memcpy(&sn->addr, res->ai_addr, res->ai_addrlen);
-		sn->addr_af = res->ai_family;
-		sn->alen = res->ai_addrlen;
-
-		if ((sn->hostname = strdup(hostname)) == NULL) {
-			applog(LOG_WARNING, "No core");
-			free(sn);
-			break;
-		}
-
-		if (debugging) {
-			char nhost[41];
-			char nport[6];
-			if (getnameinfo((struct sockaddr *) &sn->addr, sn->alen,
-					nhost, sizeof(nhost),
-				        nport, sizeof(nport),
-					NI_NUMERICHOST|NI_NUMERICSERV) == 0) {
-				applog(LOG_INFO, "Found Chunk host %s port %s",
-				       nhost, nport);
-			} else {
-				applog(LOG_INFO, "Found Chunk host");
-			}
-		}
-
-		list_add(&sn->all_link, &tabled_srv.all_stor);
-	}
-
-	freeaddrinfo(res0);
-	return;
-}
-
 static void cfg_elm_end_storage(struct config_context *cc)
 {
+	struct geo dummy_loc;
+
 	if (cc->text) {
 		applog(LOG_WARNING, "Extra text in StorageNode element: \"%s\"",
 		       cc->text);
@@ -146,7 +84,15 @@ static void cfg_elm_end_storage(struct config_context *cc)
 		goto end;
 	}
 
-	cfg_add_storage(cc->stor_host, cc->stor_port);
+	memset(&dummy_loc, 0, sizeof(struct geo));
+	stor_add_node(cc->stor_nid, cc->stor_host, cc->stor_port, &dummy_loc);
+	/*
+	 * We don't call stor_update_cb here because doing it so early
+	 * hangs TDB replication for some reason (and produces a process
+	 * that needs -9 to kill).
+	 *
+	 * Instead, there's a little plug in cldu.c that does it.
+	 */
 
 end:
 	free(cc->stor_host);
diff --git a/server/server.c b/server/server.c
index 5930712..0398a77 100644
--- a/server/server.c
+++ b/server/server.c
@@ -113,9 +113,6 @@ struct compiled_pat patterns[] = {
 	{ "\\d+\\.\\d+\\.\\d+\\.\\d+" },
 };
 
-static char *state_name_cld[] = {
-	"Init", "Active"
-};
 static char *state_name_tdb[ST_TDBNUM] = {
 	"Init", "Open", "Active", "Master", "Slave"
 };
@@ -383,8 +380,7 @@ static void stats_dump(void)
 	X(event);
 	X(tcp_accept);
 	X(opt_write);
-	applog(LOG_INFO, "State: CLD %s TDB %s",
-	    state_name_cld[tabled_srv.state_cld],
+	applog(LOG_INFO, "State: TDB %s",
 	    state_name_tdb[tabled_srv.state_tdb]);
 }
 
@@ -1373,31 +1369,48 @@ static void tdb_state_cb(enum db_event event)
 	}
 }
 
-static void cld_state_cb(enum st_cld newstate)
+/*
+ * Due to the way storage_node management is tightly woven into the
+ * server, the management of nodes is not in storage.c, which deals
+ * with the interface to Chunk and little more.
+ *
+ * We don't even bother with registering this callback, just call it by name. 
+ */
+void stor_update_cb(void)
 {
 	unsigned int env_flags;
 
-	if (debugging) {
-		applog(LOG_DEBUG, "CLD state %s > %s",
-		       state_name_cld[tabled_srv.state_cld],
-		       state_name_cld[newstate]);
-	}
-	tabled_srv.state_cld = newstate;
-	if (newstate == ST_CLD_ACTIVE) {
-		if (tabled_srv.state_tdb == ST_TDB_INIT) {
-			tabled_srv.state_tdb = ST_TDB_OPEN;
-
-			env_flags = DB_RECOVER | DB_CREATE | DB_THREAD;
-			if (tdb_init(&tdb, tabled_srv.tdb_dir, NULL,
-				     env_flags, "tabled", true,
-				     tabled_srv.rep_remotes,
-				     tabled_srv.ourhost, tabled_srv.rep_port,
-				     tdb_state_cb)) {
-				tabled_srv.state_tdb = ST_TDB_INIT;
-				applog(LOG_ERR, "Failed to open TDB, limping");
-			}
+	if (debugging)
+		applog(LOG_DEBUG, "We now have %d storage node(s)",
+		       tabled_srv.num_stor);
+	if (tabled_srv.num_stor < 1) {
+		/*
+		 * FIXME In the future, initiate net_close here if net was up.
+		 */
+		return;
+	}
+
+	/*
+	 * We initiate operations even if there's no redundancy in order
+	 * to permit bootstrapping and build-time self-checking.
+	 */
+	if (tabled_srv.state_tdb == ST_TDB_INIT) {
+		tabled_srv.state_tdb = ST_TDB_OPEN;
+
+		env_flags = DB_RECOVER | DB_CREATE | DB_THREAD;
+		if (tdb_init(&tdb, tabled_srv.tdb_dir, NULL,
+			     env_flags, "tabled", true,
+			     tabled_srv.rep_remotes,
+			     tabled_srv.ourhost, tabled_srv.rep_port,
+			     tdb_state_cb)) {
+			tabled_srv.state_tdb = ST_TDB_INIT;
+			applog(LOG_ERR, "Failed to open TDB, limping");
 		}
-		/* FIXME re-poke in case of TDB_MASTER, slave list may change */
+	} else if (tabled_srv.state_tdb == ST_TDB_MASTER) {
+		/*
+		 * FIXME This is where we should process redundancy decreases.
+		 */
+		;
 	}
 }
 
@@ -1571,7 +1584,6 @@ int main (int argc, char *argv[])
 	int rc = 1;
 
 	INIT_LIST_HEAD(&tabled_srv.all_stor);
-	tabled_srv.state_cld = ST_CLD_INIT;
 	tabled_srv.state_tdb = ST_TDB_INIT;
 
 	/* isspace() and strcasecmp() consistency requires this */
@@ -1579,11 +1591,13 @@ int main (int argc, char *argv[])
 
 	compile_patterns();
 
-	cldc_init();
-	stc_init();
 	SSL_library_init();
 	SSL_load_error_strings();
 
+	stc_init();
+
+	cld_init();
+
 	/*
 	 * parse command line
 	 */
@@ -1645,7 +1659,7 @@ int main (int argc, char *argv[])
 	if (rc)
 		goto err_out_net;
 
-	if (cld_begin(tabled_srv.ourhost, NULL, cld_state_cb) != 0) {
+	if (cld_begin(tabled_srv.ourhost, NULL) != 0) {
 		rc = 1;
 		goto err_cld_session;
 	}
diff --git a/server/storage.c b/server/storage.c
index 23765b0..380f8b0 100644
--- a/server/storage.c
+++ b/server/storage.c
@@ -97,7 +97,7 @@ int stor_put_start(struct open_chunk *cep, uint64_t key, uint64_t size)
 	cep->wtogo = size;
 	cep->wkey = key;
 	if (debugging)
-		applog(LOG_INFO, "stor put %s new for %lld\n",
+		applog(LOG_INFO, "stor put %s new for %lld",
 		       stckey, (long long) size);
 
 	return 0;
@@ -310,12 +310,73 @@ bool stor_obj_test(struct open_chunk *cep, uint64_t key)
 	return true;
 }
 
-/*
- * Add a node using its parameters file (not nul-terminated).
- */
-void stor_add_node(const char *data, size_t len)
+void stor_add_node(uint32_t nid, const char *hostname, const char *portstr,
+		   struct geo *locp)
 {
-	/* P3 */ applog(LOG_INFO, "Adding some node or sumthin.");
+	struct addrinfo hints;
+	struct addrinfo *res, *res0;
+	struct storage_node *sn;
+	int rc;
+
+	/* XXX FIXME search all_stor for dup NIDs */
+
+	memset(&hints, 0, sizeof(struct addrinfo));
+	hints.ai_family = PF_UNSPEC;
+	hints.ai_socktype = SOCK_DGRAM;
+
+	rc = getaddrinfo(hostname, portstr, &hints, &res0);
+	if (rc) {
+		applog(LOG_WARNING, "getaddrinfo(%s:%s) failed: %s",
+		       hostname, portstr, gai_strerror(rc));
+		return;
+	}
+
+	for (res = res0; res; res = res->ai_next) {
+		if (res->ai_family != AF_INET && res->ai_family != AF_INET6)
+			continue;
+
+		if (res->ai_addrlen > ADDRSIZE)		/* should not happen */
+			continue;
+
+		if ((sn = malloc(sizeof(struct storage_node))) == NULL) {
+			applog(LOG_WARNING, "No core (%ld)",
+			       (long) sizeof(struct storage_node));
+			break;
+		}
+		memset(sn, 0, sizeof(struct storage_node));
+
+		sn->id = nid;
+
+		memcpy(&sn->addr, res->ai_addr, res->ai_addrlen);
+		sn->addr_af = res->ai_family;
+		sn->alen = res->ai_addrlen;
+
+		if ((sn->hostname = strdup(hostname)) == NULL) {
+			applog(LOG_WARNING, "No core");
+			free(sn);
+			break;
+		}
+
+		if (debugging) {
+			char nhost[41];
+			char nport[6];
+			if (getnameinfo((struct sockaddr *) &sn->addr, sn->alen,
+					nhost, sizeof(nhost),
+				        nport, sizeof(nport),
+					NI_NUMERICHOST|NI_NUMERICSERV) == 0) {
+				applog(LOG_INFO, "Found Chunk host %s port %s",
+				       nhost, nport);
+			} else {
+				applog(LOG_INFO, "Found Chunk host");
+			}
+		}
+
+		list_add(&sn->all_link, &tabled_srv.all_stor);
+		tabled_srv.num_stor++;
+	}
+
+	freeaddrinfo(res0);
+	return;
 }
 
 /*
diff --git a/server/tabled.h b/server/tabled.h
index 357be73..918425a 100644
--- a/server/tabled.h
+++ b/server/tabled.h
@@ -80,12 +80,21 @@ struct compiled_pat {
 	pcre		*re;
 };
 
+struct geo {
+	char			*area;
+	char			*zone;		/* Building */
+	char			*rack;
+};
+
 struct storage_node {
 	struct list_head	all_link;
+	uint32_t		id;
+
 	unsigned		alen;
 	int			addr_af;
 	struct sockaddr_in6	addr;
 	char *hostname;		/* Only used because stc_new is overly smart. */
+
 	int nchu;		/* number of open_chunk */
 };
 
@@ -170,10 +179,6 @@ struct client {
 	char			req_buf[CLI_REQ_BUF_SZ]; /* input buffer */
 };
 
-enum st_cld {
-	ST_CLD_INIT, ST_CLD_ACTIVE
-};
-
 enum st_tdb {
 	ST_TDB_INIT, ST_TDB_OPEN, ST_TDB_ACTIVE, ST_TDB_MASTER, ST_TDB_SLAVE,
 	ST_TDBNUM
@@ -215,9 +220,9 @@ struct server {
 
 	GList			*sockets;
 	struct list_head	all_stor;	/* struct storage_node */
+	int			num_stor;	/* number of storage_node's  */
 	uint64_t		object_count;
 
-	enum st_cld		state_cld;
 	enum st_tdb		state_tdb, state_tdb_new;
 	enum st_net		state_net;
 
@@ -255,9 +260,9 @@ extern void cli_out_end(struct client *cli);
 extern void cli_in_end(struct client *cli);
 
 /* cldc.c */
+extern void cld_init(void);
+extern int cld_begin(const char *fqdn, const char *cell);
 extern void cldu_add_host(const char *host, unsigned int port);
-extern int cld_begin(const char *fqdn, const char *cell,
-		     void (*state_cb)(enum st_cld));
 extern void cld_end(void);
 
 /* util.c */
@@ -295,6 +300,7 @@ extern bool cli_cb_free(struct client *cli, struct client_write *wr,
 extern bool cli_write_start(struct client *cli);
 extern int cli_req_avail(struct client *cli);
 extern void applog(int prio, const char *fmt, ...);
+extern void stor_update_cb(void);
 
 /* config.c */
 extern void read_config(void);
@@ -312,7 +318,11 @@ extern bool stor_put_end(struct open_chunk *cep);
 extern ssize_t stor_get_buf(struct open_chunk *cep, void *data, size_t len);
 extern int stor_obj_del(struct storage_node *stn, uint64_t key);
 extern bool stor_obj_test(struct open_chunk *cep, uint64_t key);
-extern void stor_add_node(const char *data, size_t len);
+extern void stor_add_node(uint32_t nid, const char *hostname,
+			  const char *portstr, struct geo *locp);
 extern void stor_init(void);
 
+/* storparse.c */
+extern void stor_parse(char *fname, const char *text, size_t len);
+
 #endif /* __TABLED_H__ */
diff --git a/test/chunkd-test.conf b/test/chunkd-test.conf
index 9256b9d..119aa00 100644
--- a/test/chunkd-test.conf
+++ b/test/chunkd-test.conf
@@ -9,7 +9,11 @@
 </Listen>
 <PID>chunkd.pid</PID>
 <Path>data/chunk</Path>
-<NID>1</NID>
+<!--
+ Excluding NID prevents Chunk from registering with CLD and thus conflicting
+ with itself as described by <StorageNode> in tabled.conf.
+ -->
+<!-- <NID>1</NID> -->
 <CLD>
   <Port>18081</Port>
   <Host>localhost</Host>
diff --git a/test/start-daemon b/test/start-daemon
index 590da92..203e392 100755
--- a/test/start-daemon
+++ b/test/start-daemon
@@ -20,10 +20,10 @@ fi
 cld -d data/cld -P cld.pid -p 18081 -E
 chunkd -C $top_srcdir/test/chunkd-test.conf -E
 
-# 3 is enough, but we like to let chunkd to come up eary and register with CLD
+# 3 is enough, but we like to let chunkd to come up early and register with CLD
 sleep 7
 
-../server/tabled -C $top_srcdir/test/tabled-test.conf -E
+../server/tabled -C $top_srcdir/test/tabled-test.conf -E -D
 
 sleep 3