[PATCH 0/3] update chunkd checksum verification scheme

[PATCH 0/3] update chunkd checksum verification scheme

[Jeff Garzik]

This patchset is part of the work necessary to get ranged-GET (aka
partial GET) working.  As explained in
http://marc.info/?l=hail-devel&m=127871407125539&w=2 the current chunkd
checksum scheme does not work at all for partial retrievals, and must be
revamped.

These patches present step 1 of 4, adding a table of checksums to
chunkd's local on-disk format.

There are no protocol or API changes in this patchset, existing clients
should work fine without any changes.

Nevertheless, this will not be committed to the main branch until
partial retrieval is actually implemented.  I don't commit changes
unless they are actually neeeded.  This checksum table and sendfile
removal work is not required until partial-GET actually exists.

	Jeff

[PATCH 1/3] chunkd: remove sendfile(2) support

[Jeff Garzik]

commit d663521ba7e6a808be02633e57dbeb7a95973c0f
Author: Jeff Garzik <jeff@garzik.org>
Date:   Thu Jul 15 13:50:10 2010 -0400

    chunkd: remove sendfile(2) zero-copy support
    
    chunkd will be soon checksumming data in main memory.  That removes
    the utility of a zero-copy interface which bypasses the on-heap
    data requirement.
    
    Signed-off-by: Jeff Garzik <jgarzik@redhat.com>

 chunkd/be-fs.c  |   60 --------------------------------------------------------
 chunkd/chunkd.h |   14 -------------
 chunkd/object.c |   31 ++++++++++++----------------
 chunkd/server.c |   28 --------------------------
 configure.ac    |    3 --
 5 files changed, 15 insertions(+), 121 deletions(-)

diff --git a/chunkd/be-fs.c b/chunkd/be-fs.c
index f72ed48..5c97388 100644
--- a/chunkd/be-fs.c
+++ b/chunkd/be-fs.c
@@ -25,9 +25,6 @@
 #include <sys/stat.h>
 #include <sys/socket.h>
 #include <sys/uio.h>
-#if defined(HAVE_SYS_SENDFILE_H)
-#include <sys/sendfile.h>
-#endif
 #include <stdlib.h>
 #include <unistd.h>
 #include <stdio.h>
@@ -52,7 +49,6 @@ struct fs_obj {
 
 	int			in_fd;
 	char			*in_fn;
-	off_t			sendfile_ofs;
 };
 
 struct be_fs_obj_hdr {
@@ -542,62 +538,6 @@ ssize_t fs_obj_write(struct backend_obj *bo, const void *ptr, size_t len)
 	return rc;
 }
 
-#if defined(HAVE_SENDFILE) && defined(__linux__)
-
-ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len)
-{
-	struct fs_obj *obj = bo->private;
-	ssize_t rc;
-
-	if (obj->sendfile_ofs == 0) {
-		obj->sendfile_ofs += sizeof(struct be_fs_obj_hdr);
-		obj->sendfile_ofs += bo->key_len;
-	}
-
-	rc = sendfile(out_fd, obj->in_fd, &obj->sendfile_ofs, len);
-	if (rc < 0)
-		applog(LOG_ERR, "obj sendfile(%s) failed: %s",
-		       obj->in_fn, strerror(errno));
-
-	return rc;
-}
-
-#elif defined(HAVE_SENDFILE) && defined(__FreeBSD__)
-
-ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len)
-{
-	struct fs_obj *obj = bo->private;
-	ssize_t rc;
-	off_t sbytes = 0;
-
-	if (obj->sendfile_ofs == 0) {
-		obj->sendfile_ofs += sizeof(struct be_fs_obj_hdr);
-		obj->sendfile_ofs += bo->key_len;
-	}
-
-	rc = sendfile(obj->in_fd, out_fd, obj->sendfile_ofs, len,
-		      NULL, &sbytes, 0);
-	if (rc < 0) {
-		applog(LOG_ERR, "obj sendfile(%s) failed: %s",
-		       obj->in_fn, strerror(errno));
-		return rc;
-	}
-
-	obj->sendfile_ofs += sbytes;
-
-	return sbytes;
-}
-
-#else
-
-ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len)
-{
-	applog(LOG_ERR, "BUG: sendfile used but not supported");
-	return -EOPNOTSUPP;
-}
-
-#endif /* HAVE_SENDFILE && HAVE_SYS_SENDFILE_H */
-
 bool fs_obj_write_commit(struct backend_obj *bo, const char *user,
 			 unsigned char *md, bool sync_data)
 {
diff --git a/chunkd/chunkd.h b/chunkd/chunkd.h
index 1e1b1d3..1e3741a 100644
--- a/chunkd/chunkd.h
+++ b/chunkd/chunkd.h
@@ -48,8 +48,6 @@ enum {
 	STD_COOKIE_MIN		= 7,
 
 	STD_TRASH_MAX		= 1000,
-
-	CLI_MAX_SENDFILE_SZ	= 512 * 1024,
 };
 
 struct client;
@@ -63,7 +61,6 @@ struct client_write {
 	uint64_t		len;		/* write buffer length */
 	cli_write_func		cb;		/* callback */
 	void			*cb_data;	/* data passed to cb */
-	bool			sendfile;	/* using sendfile? */
 
 	struct list_head	node;
 };
@@ -275,7 +272,6 @@ extern bool fs_obj_delete(uint32_t table_id, const char *user,
 		          const void *kbuf, size_t klen,
 			  enum chunk_errcode *err_code);
 extern int fs_obj_disable(const char *fn);
-extern ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len);
 extern int fs_list_objs_open(struct fs_obj_lister *t,
 			     const char *root_path, uint32_t table_id);
 extern int fs_list_objs_next(struct fs_obj_lister *t, char **fnp);
@@ -330,7 +326,6 @@ extern void applog(int prio, const char *fmt, ...);
 extern bool cli_err(struct client *cli, enum chunk_errcode code, bool recycle_ok);
 extern int cli_writeq(struct client *cli, const void *buf, unsigned int buflen,
 		     cli_write_func cb, void *cb_data);
-extern bool cli_wr_sendfile(struct client *, cli_write_func);
 extern bool cli_rd_set_poll(struct client *cli, bool readable);
 extern void cli_wr_set_poll(struct client *cli, bool writable);
 extern bool cli_cb_free(struct client *cli, struct client_write *wr,
@@ -349,15 +344,6 @@ extern void read_config(void);
 /* selfcheck.c */
 extern int chk_spawn(TCHDB *hdb);
 
-static inline bool use_sendfile(struct client *cli)
-{
-#if defined(HAVE_SENDFILE) && defined(HAVE_SYS_SENDFILE_H)
-	return cli->ssl ? false : true;
-#else
-	return false;
-#endif
-}
-
 #ifndef HAVE_STRNLEN
 extern size_t strnlen(const char *s, size_t maxlen);
 #endif
diff --git a/chunkd/object.c b/chunkd/object.c
index bb9e696..badd199 100644
--- a/chunkd/object.c
+++ b/chunkd/object.c
@@ -252,28 +252,23 @@ void cli_in_end(struct client *cli)
 
 static bool object_read_bytes(struct client *cli)
 {
-	if (use_sendfile(cli)) {
-		if (!cli_wr_sendfile(cli, object_get_more))
-			return false;
-	} else {
-		ssize_t bytes;
+	ssize_t bytes;
 
-		bytes = fs_obj_read(cli->in_obj, cli->netbuf_out,
-				    MIN(cli->in_len, CLI_DATA_BUF_SZ));
-		if (bytes < 0)
-			return false;
-		if (bytes == 0 && cli->in_len != 0)
-			return false;
+	bytes = fs_obj_read(cli->in_obj, cli->netbuf_out,
+			    MIN(cli->in_len, CLI_DATA_BUF_SZ));
+	if (bytes < 0)
+		return false;
+	if (bytes == 0 && cli->in_len != 0)
+		return false;
 
-		cli->in_len -= bytes;
+	cli->in_len -= bytes;
 
-		if (!cli->in_len)
-			cli_in_end(cli);
+	if (!cli->in_len)
+		cli_in_end(cli);
 
-		if (cli_writeq(cli, cli->netbuf_out, bytes,
-			       cli->in_len ? object_get_more : NULL, NULL))
-			return false;
-	}
+	if (cli_writeq(cli, cli->netbuf_out, bytes,
+		       cli->in_len ? object_get_more : NULL, NULL))
+		return false;
 
 	return true;
 }
diff --git a/chunkd/server.c b/chunkd/server.c
index c3984e9..783415a 100644
--- a/chunkd/server.c
+++ b/chunkd/server.c
@@ -504,14 +504,7 @@ restart:
 
 	/* execute non-blocking write */
 do_write:
-	if (tmp->sendfile) {
-		rc = fs_obj_sendfile(cli->in_obj, cli->fd,
-				     MIN(cli->in_len, CLI_MAX_SENDFILE_SZ));
-		if (rc < 0)
-			goto err_out;
-
-		cli->in_len -= rc;
-	} else if (cli->ssl) {
+	if (cli->ssl) {
 		rc = SSL_write(cli->ssl, tmp->buf, tmp->len);
 		if (rc <= 0) {
 			rc = SSL_get_error(cli->ssl, rc);
@@ -612,31 +605,12 @@ int cli_writeq(struct client *cli, const void *buf, unsigned int buflen,
 	wr->len = buflen;
 	wr->cb = cb;
 	wr->cb_data = cb_data;
-	wr->sendfile = false;
 
 	list_add_tail(&wr->node, &cli->write_q);
 
 	return 0;
 }
 
-bool cli_wr_sendfile(struct client *cli, cli_write_func cb)
-{
-	struct client_write *wr;
-
-	wr = calloc(1, sizeof(struct client_write));
-	if (!wr)
-		return false;
-
-	wr->len = cli->in_len;
-	wr->cb = cb;
-	wr->sendfile = true;
-	INIT_LIST_HEAD(&wr->node);
-
-	list_add_tail(&wr->node, &cli->write_q);
-
-	return true;
-}
-
 static int cli_read_data(struct client *cli, void *buf, size_t buflen)
 {
 	ssize_t rc;
diff --git a/configure.ac b/configure.ac
index 6b48f1e..db10242 100644
--- a/configure.ac
+++ b/configure.ac
@@ -65,7 +65,6 @@ AM_PROG_LIBTOOL
 dnl Checks for header files.
 AC_HEADER_STDC
 dnl AC_CHECK_HEADERS(sys/ioctl.h unistd.h)
-AC_CHECK_HEADERS(sys/sendfile.h sys/filio.h)
 AC_CHECK_HEADER(db.h,[],exit 1)
 
 dnl Checks for typedefs, structures, and compiler characteristics.
@@ -97,7 +96,7 @@ PKG_CHECK_MODULES(TOKYOCABINET, tokyocabinet)
 dnl -------------------------------------
 dnl Checks for optional library functions
 dnl -------------------------------------
-AC_CHECK_FUNCS(strnlen daemon memmem memrchr sendfile)
+AC_CHECK_FUNCS(strnlen daemon memmem memrchr)
 AC_CHECK_FUNC(xdr_sizeof,
 	[AC_DEFINE([HAVE_XDR_SIZEOF], [1],
 		[Define to 1 if you have xdr_sizeof.])],

[PATCH 2/3] chunkd: pass data-length to PUT backend init

[Jeff Garzik]

commit 7a032e72bee324d8503d0c296aa940b05c8a12b2
Author: Jeff Garzik <jeff@garzik.org>
Date:   Thu Jul 15 13:50:30 2010 -0400

    chunkd: Pass PUT data length to new-object backend storage method
    
    It is useful to know the size of the object at creation time.
    
    Signed-off-by: Jeff Garzik <jgarzik@redhat.com>

 chunkd/be-fs.c  |    1 +
 chunkd/chunkd.h |    4 +++-
 chunkd/object.c |    6 ++++--
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/chunkd/be-fs.c b/chunkd/be-fs.c
index 5c97388..671c8fd 100644
--- a/chunkd/be-fs.c
+++ b/chunkd/be-fs.c
@@ -316,6 +316,7 @@ static bool key_valid(const void *key, size_t key_len)
 
 struct backend_obj *fs_obj_new(uint32_t table_id,
 			       const void *key, size_t key_len,
+			       uint64_t data_len,
 			       enum chunk_errcode *err_code)
 {
 	struct fs_obj *obj;
diff --git a/chunkd/chunkd.h b/chunkd/chunkd.h
index 1e3741a..f0cd409 100644
--- a/chunkd/chunkd.h
+++ b/chunkd/chunkd.h
@@ -258,7 +258,9 @@ struct fs_obj_lister {
 extern int fs_open(void);
 extern void fs_close(void);
 extern void fs_free(void);
-extern struct backend_obj *fs_obj_new(uint32_t table_id, const void *kbuf, size_t klen,
+extern struct backend_obj *fs_obj_new(uint32_t table_id,
+				      const void *kbuf, size_t klen,
+				      uint64_t data_len,
 				      enum chunk_errcode *err_code);
 extern struct backend_obj *fs_obj_open(uint32_t table_id, const char *user,
 				       const void *kbuf, size_t klen,
diff --git a/chunkd/object.c b/chunkd/object.c
index badd199..fb48848 100644
--- a/chunkd/object.c
+++ b/chunkd/object.c
@@ -223,7 +223,8 @@ bool object_put(struct client *cli)
 	if (!cli->out_ce)
 		return cli_err(cli, che_InternalError, true);
 
-	cli->out_bo = fs_obj_new(cli->table_id, cli->key, cli->key_len, &err);
+	cli->out_bo = fs_obj_new(cli->table_id, cli->key, cli->key_len,
+				 content_len, &err);
 	if (!cli->out_bo)
 		return cli_err(cli, err, true);
 
@@ -374,7 +375,8 @@ static void worker_cp_thr(struct worker_info *wi)
 		goto out;
 
 	cli->out_bo = out_obj = fs_obj_new(cli->table_id,
-					   cli->key, cli->key_len, &err);
+					   cli->key, cli->key_len,
+					   obj->size, &err);
 	if (!cli->out_bo)
 		goto out;
 

[PATCH 3/3] chunkd: on-disk format stores per-64k checksums

[Jeff Garzik]

commit e6fcc02bea062af291148771a59ee2028ae98834
Author: Jeff Garzik <jeff@garzik.org>
Date:   Thu Jul 15 13:57:17 2010 -0400

    chunkd: Add checksum table to on-disk format, one sum per 64k of data
    
    Signed-off-by: Jeff Garzik <jgarzik@redhat.com>

 chunkd/be-fs.c |  145 +++++++++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 127 insertions(+), 18 deletions(-)

diff --git a/chunkd/be-fs.c b/chunkd/be-fs.c
index 671c8fd..1bd85ea 100644
--- a/chunkd/be-fs.c
+++ b/chunkd/be-fs.c
@@ -40,6 +40,11 @@
 
 #define BE_FS_OBJ_MAGIC		"CHU1"
 
+enum {
+	CHUNK_BLK_ORDER		= 16,			/* 64k blocks */
+	CHUNK_BLK_SZ		= 1 << CHUNK_BLK_ORDER,
+};
+
 struct fs_obj {
 	struct backend_obj	bo;
 
@@ -49,14 +54,23 @@ struct fs_obj {
 
 	int			in_fd;
 	char			*in_fn;
+
+	size_t			checked_bytes;
+	SHA_CTX			checksum;
+	unsigned int		csum_idx;
+	void			*csum_tbl;
+	size_t			csum_tbl_sz;
+
+	unsigned int		n_blk;
 };
 
 struct be_fs_obj_hdr {
 	char			magic[4];
 	uint32_t		key_len;
 	uint64_t		value_len;
+	uint32_t		n_blk;
 
-	char			reserved[16];
+	char			reserved[12];
 
 	unsigned char		hash[CHD_CSUM_SZ];
 	char			owner[128];
@@ -204,6 +218,8 @@ static struct fs_obj *fs_obj_alloc(void)
 	obj->out_fd = -1;
 	obj->in_fd = -1;
 
+	SHA1_Init(&obj->checksum);
+
 	return obj;
 }
 
@@ -314,6 +330,17 @@ static bool key_valid(const void *key, size_t key_len)
 	return true;
 }
 
+static unsigned int fs_blk_count(uint64_t data_len)
+{
+	uint64_t n_blk;
+
+	n_blk = data_len >> CHUNK_BLK_ORDER;
+	if (data_len & (CHUNK_BLK_SZ - 1))
+		n_blk++;
+
+	return (unsigned int) n_blk;
+}
+
 struct backend_obj *fs_obj_new(uint32_t table_id,
 			       const void *key, size_t key_len,
 			       uint64_t data_len,
@@ -321,6 +348,7 @@ struct backend_obj *fs_obj_new(uint32_t table_id,
 {
 	struct fs_obj *obj;
 	char *fn = NULL;
+	size_t csum_bytes;
 	enum chunk_errcode erc = che_InternalError;
 	off_t skip_len;
 
@@ -335,6 +363,13 @@ struct backend_obj *fs_obj_new(uint32_t table_id,
 		return NULL;
 	}
 
+	obj->n_blk = fs_blk_count(data_len);
+	csum_bytes = obj->n_blk * CHD_CSUM_SZ;
+	obj->csum_tbl = malloc(csum_bytes);
+	if (!obj->csum_tbl)
+		goto err_out;
+	obj->csum_tbl_sz = csum_bytes;
+
 	/* build local fs pathname */
 	fn = fs_obj_pathname(table_id, key, key_len);
 	if (!fn)
@@ -355,7 +390,7 @@ struct backend_obj *fs_obj_new(uint32_t table_id,
 	obj->out_fn = fn;
 
 	/* calculate size of front-of-file metadata area */
-	skip_len = sizeof(struct be_fs_obj_hdr) + key_len;
+	skip_len = sizeof(struct be_fs_obj_hdr) + key_len + csum_bytes;
 
 	/* position file pointer where object data (as in, not metadata)
 	 * will begin
@@ -391,8 +426,11 @@ struct backend_obj *fs_obj_open(uint32_t table_id, const char *user,
 	struct stat st;
 	struct be_fs_obj_hdr hdr;
 	ssize_t rrc;
-	uint64_t value_len;
+	uint64_t value_len, tmp64;
+	size_t csum_bytes;
 	enum chunk_errcode erc = che_InternalError;
+	struct iovec iov[2];
+	size_t total_rd_len;
 
 	if (!key_valid(key, key_len)) {
 		*err_code = che_InvalidKey;
@@ -447,25 +485,49 @@ struct backend_obj *fs_obj_open(uint32_t table_id, const char *user,
 	}
 
 	/* verify object key length matches input key length */
-	if (GUINT32_FROM_LE(hdr.key_len) != key_len)
+	if (G_UNLIKELY(GUINT32_FROM_LE(hdr.key_len) != key_len))
 		goto err_out;
 
-	/* verify file size large enough to contain value */
 	value_len = GUINT64_FROM_LE(hdr.value_len);
-	if ((st.st_size - sizeof(hdr) - key_len) < value_len) {
+	obj->n_blk = GUINT32_FROM_LE(hdr.n_blk);
+	csum_bytes = obj->n_blk * CHD_CSUM_SZ;
+
+	/* verify file size large enough to contain value */
+	tmp64 = value_len + sizeof(hdr) + key_len + csum_bytes;
+	if (G_UNLIKELY(st.st_size < tmp64)) {
 		applog(LOG_ERR, "obj(%s) unexpected size change", obj->in_fn);
 		goto err_out;
 	}
 
+	/* verify expected size of checksum table */
+	if (G_UNLIKELY(fs_blk_count(value_len) != obj->n_blk)) {
+		applog(LOG_ERR, "obj(%s) unexpected blk count "
+		       "(%u from val sz, %u from hdr)",
+		       obj->in_fn, fs_blk_count(value_len), obj->n_blk);
+		goto err_out;
+	}
+
+	obj->csum_tbl = malloc(csum_bytes);
+	if (!obj->csum_tbl)
+		goto err_out;
+	obj->csum_tbl_sz = csum_bytes;
+
 	obj->bo.key = malloc(key_len);
 	obj->bo.key_len = key_len;
 	if (!obj->bo.key)
 		goto err_out;
 
-	/* read object variable-length header */
-	rrc = read(obj->in_fd, obj->bo.key, key_len);
-	if ((rrc != key_len) || (memcmp(key, obj->bo.key, key_len))) {
-		applog(LOG_ERR, "read hdr key obj(%s) failed: %s",
+	/* init additional header segment list */
+	iov[0].iov_base = obj->bo.key;
+	iov[0].iov_len = key_len;
+	iov[1].iov_base = obj->csum_tbl;
+	iov[1].iov_len = csum_bytes;
+	total_rd_len = iov[0].iov_len + iov[1].iov_len;
+
+	/* read additional header segments (key, checksum table) */
+	rrc = readv(obj->in_fd, iov, ARRAY_SIZE(iov));
+	if ((rrc != total_rd_len) || (memcmp(key, obj->bo.key, key_len))) {
+		applog(LOG_ERR, "read addnl hdrs(%s) failed: %s",
 			obj->in_fn,
 			(rrc < 0) ? strerror(errno) : "<unknown reasons>");
 		goto err_out;
@@ -508,6 +570,7 @@ void fs_obj_free(struct backend_obj *bo)
 	if (obj->in_fd >= 0)
 		close(obj->in_fd);
 
+	free(obj->csum_tbl);
 	free(obj);
 }
 
@@ -524,17 +587,48 @@ ssize_t fs_obj_read(struct backend_obj *bo, void *ptr, size_t len)
 	return rc;
 }
 
+static void obj_flush_csum(struct backend_obj *bo)
+{
+	struct fs_obj *obj = bo->private;
+	unsigned char md[CHD_CSUM_SZ];
+
+	SHA1_Final(md, &obj->checksum);
+
+	memcpy(obj->csum_tbl + ((obj->csum_idx++) * CHD_CSUM_SZ),
+	       md, CHD_CSUM_SZ);
+
+	obj->checked_bytes = 0;
+	SHA1_Init(&obj->checksum);
+}
+
 ssize_t fs_obj_write(struct backend_obj *bo, const void *ptr, size_t len)
 {
 	struct fs_obj *obj = bo->private;
-	ssize_t rc;
+	ssize_t rc = 0;
+
+	while (len > 0) {
+		size_t unchecked;
+
+		unchecked = CHUNK_BLK_SZ - obj->checked_bytes;
+
+		rc = write(obj->out_fd, ptr, MIN(unchecked, len));
+		if (rc < 0) {
+			applog(LOG_ERR, "obj write(%s) failed: %s",
+			       obj->out_fn, strerror(errno));
+			break;
+		}
+
+		SHA1_Update(&obj->checksum, ptr, rc);
 
-	rc = write(obj->out_fd, ptr, len);
-	if (rc < 0)
-		applog(LOG_ERR, "obj write(%s) failed: %s",
-		       obj->out_fn, strerror(errno));
-	else
 		obj->written_bytes += rc;
+		obj->checked_bytes += rc;
+		ptr += rc;
+		len -= rc;
+
+		/* if at end of 64k block, update csum table with new csum */
+		if (obj->checked_bytes == CHUNK_BLK_SZ)
+			obj_flush_csum(bo);
+	}
 
 	return rc;
 }
@@ -546,7 +640,7 @@ bool fs_obj_write_commit(struct backend_obj *bo, const char *user,
 	struct be_fs_obj_hdr hdr;
 	ssize_t wrc;
 	size_t total_wr_len;
-	struct iovec iov[2];
+	struct iovec iov[3];
 
 	memset(&hdr, 0, sizeof(hdr));
 	memcpy(hdr.magic, BE_FS_OBJ_MAGIC, strlen(BE_FS_OBJ_MAGIC));
@@ -554,6 +648,19 @@ bool fs_obj_write_commit(struct backend_obj *bo, const char *user,
 	strncpy(hdr.owner, user, sizeof(hdr.owner));
 	hdr.key_len = GUINT32_TO_LE(bo->key_len);
 	hdr.value_len = GUINT64_TO_LE(obj->written_bytes);
+	hdr.n_blk = GUINT32_TO_LE(obj->n_blk);
+
+	/* update checksum table with final csum, if necessary */
+	if (obj->checked_bytes > 0)
+		obj_flush_csum(bo);
+
+	if (G_UNLIKELY(obj->csum_idx != obj->n_blk)) {
+		applog(LOG_ERR, "BUG(%s): csum_idx/n_blk mismatch: %u/%u",
+		       obj->out_fn, obj->csum_idx, obj->n_blk);
+		return false;
+	}
+
+	obj->csum_idx = 0;
 
 	/* go back to beginning of file */
 	if (lseek(obj->out_fd, 0, SEEK_SET) < 0) {
@@ -567,7 +674,9 @@ bool fs_obj_write_commit(struct backend_obj *bo, const char *user,
 	iov[0].iov_len = sizeof(hdr);
 	iov[1].iov_base = bo->key;
 	iov[1].iov_len = bo->key_len;
-	total_wr_len = iov[0].iov_len + iov[1].iov_len;
+	iov[2].iov_base = obj->csum_tbl;
+	iov[2].iov_len = obj->csum_tbl_sz;
+	total_wr_len = iov[0].iov_len + iov[1].iov_len + iov[2].iov_len;
 
 	/* write object header segments */
 	wrc = writev(obj->out_fd, iov, ARRAY_SIZE(iov));