[RFC PATCH v2 16/16] Smart HTTP fetch: gzip requests

["Shawn O. Pearce" <spearce@spearce.org>]

The upload-pack requests are mostly plain text and they compress
rather well.  Deflating them with Content-Encoding: gzip can easily
drop the size of the request by 50%, reducing the amount of data
to transfer as we negotiate the common commits.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
CC: Daniel Barkalow <barkalow@iabervon.org>
---
 remote-curl.c |   52 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/remote-curl.c b/remote-curl.c
index 31d1d34..d53215d 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -261,11 +261,12 @@ static size_t rpc_in(const void *ptr, size_t eltsize,
 	return size;
 }
 
-static int post_rpc(struct rpc_state *state)
+static int post_rpc(struct rpc_state *state, int use_gzip)
 {
 	struct active_request_slot *slot;
 	struct slot_results results;
 	struct curl_slist *headers = NULL;
+	unsigned char *gzip_body = NULL;
 	int err = 0, large_request = 0;
 
 	/* Try to load the entire request, if we can fit it into the
@@ -279,6 +280,7 @@ static int post_rpc(struct rpc_state *state)
 
 		if (left < LARGE_PACKET_MAX) {
 			large_request = 1;
+			use_gzip = 0;
 			break;
 		}
 
@@ -311,6 +313,48 @@ static int post_rpc(struct rpc_state *state)
 			fflush(stderr);
 		}
 
+	} else if (use_gzip && 1024 < state->len) {
+		/* The client backend isn't giving us compressed data so
+		 * we can try to deflate it ourselves, this may save on.
+		 * the transfer time.
+		 */
+		size_t size;
+		z_stream stream;
+		int ret;
+
+		memset(&stream, 0, sizeof(stream));
+		ret = deflateInit2(&stream, Z_DEFAULT_COMPRESSION,
+				Z_DEFLATED, (15 + 16),
+				8, Z_DEFAULT_STRATEGY);
+		if (ret != Z_OK)
+			die("cannot deflate request; zlib init error %d", ret);
+		size = deflateBound(&stream, state->len);
+		gzip_body = xmalloc(size);
+
+		stream.next_in = (unsigned char *)state->buf;
+		stream.avail_in = state->len;
+		stream.next_out = gzip_body;
+		stream.avail_out = size;
+
+		ret = deflate(&stream, Z_FINISH);
+		if (ret != Z_STREAM_END)
+			die("cannot deflate request; zlib deflate error %d", ret);
+
+		ret = deflateEnd(&stream);
+		if (ret != Z_OK)
+			die("cannot deflate request; zlib end error %d", ret);
+
+		size = stream.total_out;
+
+		headers = curl_slist_append(headers, "Content-Encoding: gzip");
+		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
+		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, size);
+
+		if (state->verbose) {
+			fprintf(stderr, "POST %s (gzip %lu to %lu bytes)\n",
+				state->service_name, state->len, size);
+			fflush(stderr);
+		}
 	} else {
 		/* We know the complete request size in advance, use the
 		 * more normal Content-Length approach.
@@ -336,11 +380,13 @@ static int post_rpc(struct rpc_state *state)
 		}
 	}
 	curl_slist_free_all(headers);
+	free(gzip_body);
 	return err;
 }
 
 static int one_shot_rpc_service(const char *service,
 	int verbose,
+	int use_gzip,
 	const char **client_argv,
 	struct discovery *heads,
 	struct strbuf *result)
@@ -384,7 +430,7 @@ static int one_shot_rpc_service(const char *service,
 			break;
 		state.pos = 0;
 		state.len = n;
-		err |= post_rpc(&state);
+		err |= post_rpc(&state, use_gzip);
 	}
 	if (result)
 		strbuf_read(result, client.out, 0);
@@ -474,6 +520,7 @@ static int fetch_git(struct fetch_args *args,
 
 	err = one_shot_rpc_service("git-upload-pack",
 		args->verbose,
+		1 /* gzip request */,
 		argv, heads, &res);
 	if (res.len)
 		safe_write(1, res.buf, res.len);
@@ -611,6 +658,7 @@ static int push_git(struct discovery *heads,
 
 	err = one_shot_rpc_service("git-receive-pack",
 		args->verbose,
+		0 /* no gzip */,
 		argv, heads, &res);
 	if (res.len)
 		safe_write(1, res.buf, res.len);
-- 
1.6.5.52.g0ff2e