[Patch 3/4] chunkd: add objcache

[Pete Zaitcev <zaitcev@redhat.com>]

This a mechanism by which self-check may know what objects were updated
and thus should not be disturbed if their checksums fail.

Signed-off-by: Pete Zaitcev <zaitcev@redhat.com>

---
 include/Makefile.am  |    2 
 include/objcache.h   |   72 +++++++++++++++++++++++
 server/Makefile.am   |    3 
 server/chunkd.h      |    3 
 server/objcache.c    |  128 +++++++++++++++++++++++++++++++++++++++++
 server/object.c      |    9 ++
 server/server.c      |    7 ++
 test/.gitignore      |    1 
 test/Makefile.am     |    5 +
 test/objcache-unit.c |   58 ++++++++++++++++++
 10 files changed, 285 insertions(+), 3 deletions(-)

commit ab575b65211f00c73feffd4c8c58044142b1163f
Author: Master <zaitcev@lembas.zaitcev.lan>
Date:   Sun Dec 27 16:04:43 2009 -0700

    Add the objcache.

diff --git a/include/Makefile.am b/include/Makefile.am
index ddc2b8a..7abe0b5 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -1,5 +1,5 @@
 
 include_HEADERS = chunkc.h chunk_msg.h
 
-EXTRA_DIST	= elist.h chunk_msg.h chunksrv.h chunk-private.h
+EXTRA_DIST	= elist.h chunk_msg.h chunksrv.h chunk-private.h objcache.h
 
diff --git a/include/objcache.h b/include/objcache.h
new file mode 100644
index 0000000..fce1485
--- /dev/null
+++ b/include/objcache.h
@@ -0,0 +1,72 @@
+
+/*
+ * Copyright 2009 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+#ifndef _CHUNKD_OBJCACHE_H_
+#define _CHUNKD_OBJCACHE_H_
+
+#include <glib.h>
+#include <elist.h>
+#include <stdbool.h>
+
+struct objcache {
+	struct list_head head;
+	GMutex *lock;
+};
+
+struct objcache_entry {
+	struct list_head link;
+	unsigned int hash;
+	unsigned int flags;
+	int ref;
+};
+
+#define OC_F_DIRTY   0x1
+
+/*
+ * Get an entry and set flags.
+ * A method for every flag is needed because our locks are internal to
+ * the cache, and we want this to be atomic.
+ */
+#define objcache_get(c, k, l)		__objcache_get(c, k, l, 0)
+#define objcache_get_dirty(c, k, l)	__objcache_get(c, k, l, OC_F_DIRTY)
+extern struct objcache_entry *__objcache_get(struct objcache *cache,
+					     const char *key, int klen,
+					     unsigned int flag);
+
+/*
+ * Test for dirty.
+ */
+extern bool objcache_test_dirty(struct objcache *cache,
+				struct objcache_entry *entry);
+
+/*
+ * Put an entry (decrement and free, or an equivalent).
+ */
+extern void objcache_put(struct objcache *cache, struct objcache_entry *entry);
+
+/*
+ * Init a cache. Call once. May fail since it allocates a mutex.
+ */
+extern int objcache_init(struct objcache *cache);
+
+/*
+ * Terminate a cache.
+ */
+extern void objcache_fini(struct objcache *cache);
+
+#endif
diff --git a/server/Makefile.am b/server/Makefile.am
index 516b081..824c0b3 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -6,7 +6,8 @@ sbin_PROGRAMS	= chunkd
 
 chunkd_SOURCES	= chunkd.h		\
 		  ../lib/chunksrv.c	\
-		  be-fs.c object.c server.c config.c cldu.c util.c
+		  be-fs.c object.c server.c config.c cldu.c util.c \
+		  objcache.c
 chunkd_LDADD	= \
 		  @CLDC_LIBS@ @GLIB_LIBS@ @CRYPTO_LIBS@ \
 		  @SSL_LIBS@ @EVENT_LIBS@ @TOKYOCABINET_LIBS@
diff --git a/server/chunkd.h b/server/chunkd.h
index a97088d..18dd31a 100644
--- a/server/chunkd.h
+++ b/server/chunkd.h
@@ -28,6 +28,7 @@
 #include <chunk_msg.h>
 #include <hail_log.h>
 #include <tchdb.h>
+#include <objcache.h>
 
 #ifndef ARRAY_SIZE
 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
@@ -118,6 +119,7 @@ struct client {
 	long			out_len;
 
 	struct backend_obj	*out_bo;
+	struct objcache_entry	*out_ce;
 
 	long			in_len;
 	struct backend_obj	*in_obj;
@@ -210,6 +212,7 @@ struct server {
 	struct geo		loc;
 
 	TCHDB			*tbl_master;
+	struct objcache		actives;
 
 	struct server_stats	stats;		/* global statistics */
 };
diff --git a/server/objcache.c b/server/objcache.c
new file mode 100644
index 0000000..475ac23
--- /dev/null
+++ b/server/objcache.c
@@ -0,0 +1,128 @@
+
+/*
+ * Copyright 2009 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+#include <objcache.h>
+#include <stdlib.h>
+
+/*
+ * We really should not screw around with hand-rolled garbage and use
+ * something like Paul Hsieh's SuperFastHash, but licenses are too confusing.
+ */
+static unsigned int objcache_hash(const char *key, int klen)
+{
+	unsigned int hash;
+	int i;
+	unsigned char c;
+
+	hash = 0x55555555;
+	for (i = 0; i < klen; i++) {
+		c = (unsigned char) *key++;
+		hash ^= hash << 16;
+		hash ^= c;
+		hash = (hash << 8) | (hash >> 24);
+	}
+	return hash;
+}
+
+static struct objcache_entry *objcache_lookup(struct objcache *cache,
+					      unsigned int hash)
+{
+	struct objcache_entry *cep;
+
+	list_for_each_entry(cep, &cache->head, link) {
+		if (cep->hash == hash)
+			return cep;
+	}
+	return NULL;
+}
+
+static struct objcache_entry *objcache_insert(struct objcache *cache,
+					      unsigned int hash)
+{
+	struct objcache_entry *cep;
+
+	cep = malloc(sizeof(struct objcache_entry));
+	if (!cep)
+		return NULL;
+	cep->hash = hash;
+	cep->flags = 0;
+	cep->ref = 1;
+	list_add(&cep->link, &cache->head);
+	return cep;
+}
+
+struct objcache_entry *__objcache_get(struct objcache *cache,
+				      const char *key, int klen,
+				      unsigned int flag)
+{
+	struct objcache_entry *cep;
+	unsigned int hash;
+
+	hash = objcache_hash(key, klen);
+	g_mutex_lock(cache->lock);
+	cep = objcache_lookup(cache, hash);
+	if (cep) {
+		cep->ref++;
+	} else {
+		cep = objcache_insert(cache, hash);
+	}
+	cep->flags |= flag;
+	g_mutex_unlock(cache->lock);
+	return cep;
+}
+
+bool objcache_test_dirty(struct objcache *cache, struct objcache_entry *cep)
+{
+	bool ret;
+
+	g_mutex_lock(cache->lock);
+	ret = cep->flags & OC_F_DIRTY;
+	g_mutex_unlock(cache->lock);
+	return ret;
+}
+
+void objcache_put(struct objcache *cache, struct objcache_entry *cep)
+{
+	g_mutex_lock(cache->lock);
+	if (!cep->ref) {
+		g_mutex_unlock(cache->lock);
+		/* Must not happen, or a leak for Valgrind to catch. */
+		return;
+	}
+	--cep->ref;
+	if (!cep->ref) {
+		list_del(&cep->link);
+		free(cep);
+	}
+	g_mutex_unlock(cache->lock);
+}
+
+int objcache_init(struct objcache *cache)
+{
+	cache->lock = g_mutex_new();
+	if (!cache->lock)
+		return -1;
+	INIT_LIST_HEAD(&cache->head);
+	return 0;
+}
+
+void objcache_fini(struct objcache *cache)
+{
+	g_mutex_free(cache->lock);
+}
diff --git a/server/object.c b/server/object.c
index a1205f5..5cc596a 100644
--- a/server/object.c
+++ b/server/object.c
@@ -73,6 +73,10 @@ void cli_out_end(struct client *cli)
 		fs_obj_free(cli->out_bo);
 		cli->out_bo = NULL;
 	}
+	if (cli->out_ce) {
+		objcache_put(&chunkd_srv.actives, cli->out_ce);
+		cli->out_ce = NULL;
+	}
 
 	free(cli->out_user);
 	cli->out_user = NULL;
@@ -217,6 +221,11 @@ bool object_put(struct client *cli)
 	if (!user)
 		return cli_err(cli, che_AccessDenied, true);
 
+	cli->out_ce = objcache_get_dirty(&chunkd_srv.actives,
+					 cli->key, cli->key_len);
+	if (!cli->out_ce)
+		return cli_err(cli, che_InternalError, true);
+
 	cli->out_bo = fs_obj_new(cli->table_id, cli->key, cli->key_len, &err);
 	if (!cli->out_bo)
 		return cli_err(cli, err, true);
diff --git a/server/server.c b/server/server.c
index 3f38cca..e955dc8 100644
--- a/server/server.c
+++ b/server/server.c
@@ -1659,6 +1659,11 @@ int main (int argc, char *argv[])
 		goto err_out_session;
 	}
 
+	if (objcache_init(&chunkd_srv.actives) != 0) {
+		rc = 1;
+		goto err_out_objcache;
+	}
+
 	INIT_LIST_HEAD(&chunkd_srv.wr_trash);
 	chunkd_srv.trash_sz = 0;
 
@@ -1692,6 +1697,8 @@ err_out_listen:
 err_out_cld:
 	fs_close();
 err_out_fs:
+	objcache_fini(&chunkd_srv.actives);
+err_out_objcache:
 	if (strict_free)
 		g_hash_table_destroy(chunkd_srv.fd_info);
 err_out_session:
diff --git a/test/.gitignore b/test/.gitignore
index ccb6bdb..a929882 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -4,6 +4,7 @@ it-works
 large-object
 lotsa-objects
 nop
+objcache-unit
 
 .libs
 libtest.a
diff --git a/test/Makefile.am b/test/Makefile.am
index d9c10b5..84b4837 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -15,6 +15,7 @@ EXTRA_DIST =			\
 	ssl-key.pem ssl-cert.pem
 
 TESTS =				\
+	objcache-unit		\
 	prep-db			\
 	start-daemon		\
 	pid-exists		\
@@ -29,7 +30,7 @@ TESTS =				\
 	clean-db
 
 check_PROGRAMS		= auth basic-object it-works large-object \
-			  lotsa-objects nop
+			  lotsa-objects nop objcache-unit
 
 TESTLDADD		= ../lib/libchunkdc.la	\
 			  libtest.a		\
@@ -42,6 +43,8 @@ large_object_LDADD	= $(TESTLDADD)
 lotsa_objects_LDADD	= $(TESTLDADD)
 nop_LDADD		= $(TESTLDADD)
 
+objcache_unit_LDADD	= @GLIB_LIBS@
+
 noinst_LIBRARIES	= libtest.a
 
 libtest_a_SOURCES	= libtest.c
diff --git a/test/objcache-unit.c b/test/objcache-unit.c
new file mode 100644
index 0000000..f101445
--- /dev/null
+++ b/test/objcache-unit.c
@@ -0,0 +1,58 @@
+
+/*
+ * Copyright 2009 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+#include "../server/objcache.c"
+#include "test.h"
+
+int main(int argc, char *argv[])
+{
+	static char k1[] = { 'a' };
+	static char k2[] = { 'a', 'a' };
+	static char k3[] = { 'a', '\0', 'a' };
+	struct objcache cache;
+	struct objcache_entry *ep1, *ep2, *ep3;
+	int rc;
+
+	g_thread_init(NULL);
+	rc = objcache_init(&cache);
+	OK(rc==0);
+
+	ep1 = objcache_get(&cache, k1, sizeof(k1));
+	OK(ep1 != NULL);
+
+	ep2 = objcache_get(&cache, k2, sizeof(k2));
+	OK(ep2 != NULL);
+
+	ep3 = objcache_get(&cache, k3, sizeof(k3));
+	OK(ep3 != NULL);
+
+	OK(ep1->ref == 1);	/* no collisions */
+
+	objcache_put(&cache, ep1);
+	objcache_put(&cache, ep2);
+	objcache_put(&cache, ep3);
+
+	ep2 = objcache_get(&cache, k2, sizeof(k2));
+	OK(ep2 != NULL);
+	OK(ep2->ref == 1);	/* new */
+	objcache_put(&cache, ep2);
+
+	objcache_fini(&cache);
+	return 0;
+}