[Patch 2/2] tabled: add a test for larger objects

[Patch 2/2] tabled: add a test for larger objects

[Pete Zaitcev]

Existing tests only excercised operations with relatively small objects.
It did not test pipelining of object data in sufficient degree. So, let's
have a better test case for this (large-object.c).

We also change the existing basic-object.c to match.

Signed-Off-By: Pete Zaitcev <zaitcev@redhat.com>

---
 test/.gitignore     |    1 
 test/Makefile.am    |    5 -
 test/basic-object.c |   13 +-
 test/large-object.c |  197 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 209 insertions(+), 7 deletions(-)

commit 0fc948e8576b68746183013275784f5dc27b5868
Author: Master <zaitcev@lembas.zaitcev.lan>
Date:   Tue Jan 5 00:08:13 2010 -0700

    Add a new test for larger objects.

diff --git a/test/.gitignore b/test/.gitignore
index ed7f1fa..c99d4e2 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -3,5 +3,6 @@
 basic-bucket
 basic-object
 it-works
+large-object
 wait-for-listen
 
diff --git a/test/Makefile.am b/test/Makefile.am
index 071c69c..2e18349 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -23,16 +23,19 @@ TESTS =				\
 	it-works		\
 	basic-bucket		\
 	basic-object		\
+	large-object		\
 	stop-daemon		\
 	clean-db
 
-check_PROGRAMS		= wait-for-listen basic-bucket basic-object it-works
+check_PROGRAMS		= basic-bucket basic-object it-works large-object \
+			  wait-for-listen
 
 TESTLDADD		= ../lib/libhttpstor.la	\
 			  ../lib/libhttputil.a	\
 			  @LIBCURL@ @GLIB_LIBS@ @CRYPTO_LIBS@ @XML_LIBS@
 basic_bucket_LDADD	= $(TESTLDADD)
 basic_object_LDADD	= $(TESTLDADD)
+large_object_LDADD	= $(TESTLDADD)
 it_works_LDADD		= $(TESTLDADD)
 
 wait_for_listen_LDADD	= ../lib/libhttputil.a
diff --git a/test/basic-object.c b/test/basic-object.c
index 17fdd24..0761e31 100644
--- a/test/basic-object.c
+++ b/test/basic-object.c
@@ -28,12 +28,13 @@
 
 int main(int argc, char *argv[])
 {
+	static char bucket[] = "test1";
+	static char key[] = "my first key";
 	struct httpstor_client *httpstor;
 	char accbuf[80];
 	int rc;
 	bool rcb;
 	char val[] = "my first value";
-	char key[] = "my first key";
 	size_t len = 0;
 	void *mem;
 
@@ -46,25 +47,25 @@ int main(int argc, char *argv[])
 	OK(httpstor);
 
 	/* add bucket */
-	rcb = httpstor_add_bucket(httpstor, "test1");
+	rcb = httpstor_add_bucket(httpstor, bucket);
 	OK(rcb);
 
 	/* store object */
-	rcb = httpstor_put_inline(httpstor, "test1", key, val, strlen(val), NULL);
+	rcb = httpstor_put_inline(httpstor, bucket, key, val, strlen(val), NULL);
 	OK(rcb);
 
 	/* get object */
-	mem = httpstor_get_inline(httpstor, "test1", key, false, &len);
+	mem = httpstor_get_inline(httpstor, bucket, key, false, &len);
 	OK(mem);
 	OK(len == strlen(val));
 	OK(!memcmp(val, mem, strlen(val)));
 
 	/* delete object */
-	rcb = httpstor_del(httpstor, "test1", key);
+	rcb = httpstor_del(httpstor, bucket, key);
 	OK(rcb);
 
 	/* delete bucket */
-	rcb = httpstor_del_bucket(httpstor, "test1");
+	rcb = httpstor_del_bucket(httpstor, bucket);
 	OK(rcb);
 
 	return 0;
diff --git a/test/large-object.c b/test/large-object.c
new file mode 100644
index 0000000..b656333
--- /dev/null
+++ b/test/large-object.c
@@ -0,0 +1,197 @@
+
+/*
+ * Copyright 2008-2009 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+/*
+ * A large object test verifies the workings of bizarrely complicated and
+ * subtle mechanics of the sliding windows and flow control when tabled
+ * pipes the data between its client and the back-end chunkservers.
+ * As such, we have to defend against hungs as well as corruption.
+ */
+
+#define _GNU_SOURCE
+#include "tabled-config.h"
+
+#include <stdbool.h>
+#include <string.h>
+#include <stdlib.h>
+#include <locale.h>
+#include <httpstor.h>
+#include <httputil.h>
+#include "test.h"
+
+#define BLKSZ0	1024
+#define NBLKS0	1
+
+#define BLKSZ1	15001
+#define NBLKS1	211
+
+#define BLKSZ2	256
+#define NBLKS2	30000
+
+struct put_ctx {
+	unsigned int blksize;
+	unsigned long off;
+	unsigned long total;
+};
+
+struct get_ctx {
+	unsigned int blksize;
+	unsigned long off;
+};
+
+static char bucket[] = "test-large";
+static char key[] = "Key of Large Object";
+
+static size_t put_cb(void *ptr, size_t membsize, size_t nmemb, void *user_data)
+{
+	struct put_ctx *ctx = user_data;
+	unsigned char *data = ptr;
+	unsigned num;
+	size_t rem;
+	size_t off;
+
+	OK(membsize == 1);
+
+	if (ctx->off >= ctx->total)
+		return 0;
+
+	num = ctx->off / ctx->blksize;		/* current block number */
+	off = ctx->off % ctx->blksize;		/* done in the block */
+	rem = ctx->blksize - off;		/* to do in the block */
+	if (rem > nmemb)
+		rem = nmemb;
+	if (rem > ctx->total - ctx->off)
+		rem = ctx->total - ctx->off;
+
+	memset(data, 0, rem);
+	if (off + rem == ctx->blksize)
+		data[rem - 1] = ~num;
+	if (off == 0)
+		data[0] = num;
+
+	ctx->off += rem;
+
+	return rem;
+}
+
+static size_t get_one(struct get_ctx *ctx, unsigned char *data, size_t len)
+{
+	unsigned num;
+	size_t rem;
+	size_t off;
+
+	num = ctx->off / ctx->blksize;		/* current block number */
+	off = ctx->off % ctx->blksize;		/* done in the block */
+	rem = ctx->blksize - off;		/* to do in the block */
+	if (rem > len)
+		rem = len;
+
+	if (off + rem == ctx->blksize) {
+		if (data[rem - 1] != (unsigned char) ~num) {
+			fprintf(stderr, "get chk fail tail:"
+				" blk %u data 0x%02x blksize"
+				" %u off %lu rem %lu\n",
+				num, data[rem-1], ctx->blksize,
+				(long)off, (long)rem);
+			exit(1);
+		}
+	}
+	if (off == 0) {
+		if (data[0] != (unsigned char) num) {
+			fprintf(stderr, "get chk fail head:"
+				" blk %u data 0x%02x blksize %u\n",
+				num, data[0], ctx->blksize);
+			exit(1);
+		}
+	}
+
+	ctx->off += rem;
+	return rem;
+}
+
+static size_t get_cb(void *ptr, size_t membsize, size_t nmemb, void *user_data)
+{
+	struct get_ctx *ctx = user_data;
+	size_t togo, len;
+
+	OK(membsize == 1);
+
+	togo = nmemb;
+	while (togo) {
+		len = get_one(ctx, ptr, togo);
+		togo -= len;
+		ptr += len;
+	}
+	return nmemb;
+}
+
+static void runtest(struct httpstor_client *httpstor,
+		    size_t blklen, int nblks)
+{
+	off_t total = blklen * nblks;
+	struct put_ctx putctx;
+	struct get_ctx getctx;
+	bool rcb;
+
+	memset(&putctx, 0, sizeof(putctx));
+	putctx.blksize = blklen;
+	putctx.total = total;
+
+	rcb = httpstor_put(httpstor, bucket, key, put_cb, total, &putctx, NULL);
+	OK(rcb);
+	OK(putctx.off == total);
+
+	memset(&getctx, 0, sizeof(getctx));
+	getctx.blksize = blklen;
+
+	rcb = httpstor_get(httpstor, bucket, key, get_cb, &getctx, false);
+	OK(rcb);
+	OK(getctx.off == total);
+}
+
+int main(int argc, char *argv[])
+{
+	struct httpstor_client *httpstor;
+	char accbuf[80];
+	int rc;
+	bool rcb;
+
+	setlocale(LC_ALL, "C");
+
+	rc = tb_readport(TEST_FILE_TB, accbuf, sizeof(accbuf));
+	OK(rc > 0);
+
+	httpstor = httpstor_new(accbuf, TEST_HOST, TEST_USER, TEST_USER_KEY);
+	OK(httpstor);
+
+	/* add bucket - since tests are independent, we do not rely on others */
+	rcb = httpstor_add_bucket(httpstor, bucket);
+	OK(rcb);
+
+	runtest(httpstor, BLKSZ0, NBLKS0);
+	runtest(httpstor, BLKSZ1, NBLKS1);
+	runtest(httpstor, BLKSZ2, NBLKS2);
+
+	rcb = httpstor_del(httpstor, bucket, key);
+	OK(rcb);
+
+	rcb = httpstor_del_bucket(httpstor, bucket);
+	OK(rcb);
+
+	return 0;
+}

Re: [Patch 2/2] tabled: add a test for larger objects

[Jeff Garzik]

On 01/05/2010 02:27 AM, Pete Zaitcev wrote:
> Existing tests only excercised operations with relatively small objects.
> It did not test pipelining of object data in sufficient degree. So, let's
> have a better test case for this (large-object.c).
>
> We also change the existing basic-object.c to match.
>
> Signed-Off-By: Pete Zaitcev<zaitcev@redhat.com>
>
> ---
>   test/.gitignore     |    1
>   test/Makefile.am    |    5 -
>   test/basic-object.c |   13 +-
>   test/large-object.c |  197 ++++++++++++++++++++++++++++++++++++++++++
>   4 files changed, 209 insertions(+), 7 deletions(-)

applied.

a follow-up patch that checks the checksums of the data sent/received 
would be nice...

Re: [Patch 2/2] tabled: add a test for larger objects

[Pete Zaitcev]

On Tue, 05 Jan 2010 04:14:08 -0500
Jeff Garzik <jeff@garzik.org> wrote:

> a follow-up patch that checks the checksums of the data sent/received 
> would be nice...

That would require more coding than using single check bytes because
a checksum can span a block boundary. But ok, you're right, I'll do it.

-- Pete