[PATCH] unpack_sha1_file(): zlib can only process 4GB at a time

[PATCH] unpack_sha1_file(): zlib can only process 4GB at a time

[Junio C Hamano]

The same theme as "unpack-objects" patch.

Signed-off-by: Junio C Hamano <gitster@pobox.com>
---

 * On a beefy enough machine, you could register a large binary blob
   and run fsck, fetch, or push to play with that object with these
   patches, but there are many more places that aren't safe.

 sha1_file.c |   29 ++++++++++++++++++++---------
 1 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/sha1_file.c b/sha1_file.c
index 12a166f..ab9e962 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -1257,7 +1257,7 @@ static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned lon
 	/* Get the data stream */
 	memset(stream, 0, sizeof(*stream));
 	stream->next_in = map;
-	stream->avail_in = mapsize;
+	stream->avail_in = zlib_buf_cap(mapsize);
 	stream->next_out = buffer;
 	stream->avail_out = bufsiz;
 
@@ -1291,11 +1291,12 @@ static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned lon
 	return 0;
 }
 
-static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
+static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long mapsize,
+			      unsigned long size, const unsigned char *sha1)
 {
-	int bytes = strlen(buffer) + 1;
+	unsigned long bytes = strlen(buffer) + 1;
 	unsigned char *buf = xmallocz(size);
-	unsigned long n;
+	unsigned long n, bytes_to_produce, bytes_to_consume;
 	int status = Z_OK;
 
 	n = stream->total_out - bytes;
@@ -1303,6 +1304,8 @@ static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size
 		n = size;
 	memcpy(buf, (char *) buffer + bytes, n);
 	bytes = n;
+
+	bytes_to_consume = mapsize - stream->total_in;
 	if (bytes <= size) {
 		/*
 		 * The above condition must be (bytes <= size), not
@@ -1318,17 +1321,25 @@ static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size
 		 * went well with status == Z_STREAM_END at the end.
 		 */
 		stream->next_out = buf + bytes;
-		stream->avail_out = size - bytes;
-		while (status == Z_OK)
+		bytes_to_produce = size - bytes;
+
+		while (status == Z_OK || status == Z_BUF_ERROR) {
+			unsigned char *in0 = stream->next_in;
+			unsigned char *out0 = stream->next_out;
+			stream->avail_in = zlib_buf_cap(bytes_to_consume);
+			stream->avail_out = zlib_buf_cap(bytes_to_produce);
 			status = git_inflate(stream, Z_FINISH);
+			bytes_to_produce -= stream->next_out - out0;
+			bytes_to_consume -= stream->next_in - in0;
+		}
 	}
-	if (status == Z_STREAM_END && !stream->avail_in) {
+	if (status == Z_STREAM_END && !bytes_to_consume) {
 		git_inflate_end(stream);
 		return buf;
 	}
 
 	if (status < 0)
-		error("corrupt loose object '%s'", sha1_to_hex(sha1));
+		error("corrupt loose object '%s' %d", sha1_to_hex(sha1), status);
 	else if (stream->avail_in)
 		error("garbage at end of loose object '%s'",
 		      sha1_to_hex(sha1));
@@ -1397,7 +1408,7 @@ static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type
 	if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
 		return NULL;
 
-	return unpack_sha1_rest(&stream, hdr, *size, sha1);
+	return unpack_sha1_rest(&stream, hdr, mapsize, *size, sha1);
 }
 
 unsigned long get_size_from_delta(struct packed_git *p,
-- 
1.7.6.rc1.118.ge175b4a

[PATCH] unpack_sha1_file(): zlib can only process 4GB at a time

[Junio C Hamano]

The same theme as "unpack-objects" patch.

Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 * On a beefy enough machine, you could register a large binary blob
   and run fsck, fetch, or push to play with that object with these
   patches, but there are many more places that aren't safe.

   I seem to have caught a nasty cold, so this will be the end of the
   series from me for today.  At least that is what I plan for now.

 sha1_file.c |   29 ++++++++++++++++++++---------
 1 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/sha1_file.c b/sha1_file.c
index 12a166f..ab9e962 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -1257,7 +1257,7 @@ static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned lon
 	/* Get the data stream */
 	memset(stream, 0, sizeof(*stream));
 	stream->next_in = map;
-	stream->avail_in = mapsize;
+	stream->avail_in = zlib_buf_cap(mapsize);
 	stream->next_out = buffer;
 	stream->avail_out = bufsiz;
 
@@ -1291,11 +1291,12 @@ static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned lon
 	return 0;
 }
 
-static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
+static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long mapsize,
+			      unsigned long size, const unsigned char *sha1)
 {
-	int bytes = strlen(buffer) + 1;
+	unsigned long bytes = strlen(buffer) + 1;
 	unsigned char *buf = xmallocz(size);
-	unsigned long n;
+	unsigned long n, bytes_to_produce, bytes_to_consume;
 	int status = Z_OK;
 
 	n = stream->total_out - bytes;
@@ -1303,6 +1304,8 @@ static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size
 		n = size;
 	memcpy(buf, (char *) buffer + bytes, n);
 	bytes = n;
+
+	bytes_to_consume = mapsize - stream->total_in;
 	if (bytes <= size) {
 		/*
 		 * The above condition must be (bytes <= size), not
@@ -1318,17 +1321,25 @@ static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size
 		 * went well with status == Z_STREAM_END at the end.
 		 */
 		stream->next_out = buf + bytes;
-		stream->avail_out = size - bytes;
-		while (status == Z_OK)
+		bytes_to_produce = size - bytes;
+
+		while (status == Z_OK || status == Z_BUF_ERROR) {
+			unsigned char *in0 = stream->next_in;
+			unsigned char *out0 = stream->next_out;
+			stream->avail_in = zlib_buf_cap(bytes_to_consume);
+			stream->avail_out = zlib_buf_cap(bytes_to_produce);
 			status = git_inflate(stream, Z_FINISH);
+			bytes_to_produce -= stream->next_out - out0;
+			bytes_to_consume -= stream->next_in - in0;
+		}
 	}
-	if (status == Z_STREAM_END && !stream->avail_in) {
+	if (status == Z_STREAM_END && !bytes_to_consume) {
 		git_inflate_end(stream);
 		return buf;
 	}
 
 	if (status < 0)
-		error("corrupt loose object '%s'", sha1_to_hex(sha1));
+		error("corrupt loose object '%s' %d", sha1_to_hex(sha1), status);
 	else if (stream->avail_in)
 		error("garbage at end of loose object '%s'",
 		      sha1_to_hex(sha1));
@@ -1397,7 +1408,7 @@ static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type
 	if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
 		return NULL;
 
-	return unpack_sha1_rest(&stream, hdr, *size, sha1);
+	return unpack_sha1_rest(&stream, hdr, mapsize, *size, sha1);
 }
 
 unsigned long get_size_from_delta(struct packed_git *p,
-- 
1.7.6.rc1.118.ge175b4a

Re: [PATCH] unpack_sha1_file(): zlib can only process 4GB at a time

[Sverre Rabbelier]

Heya,

On Thu, Jun 9, 2011 at 22:15, Junio C Hamano <gitster@pobox.com> wrote:
>  * On a beefy enough machine, you could register a large binary blob
>   and run fsck, fetch, or push to play with that object with these
>   patches, but there are many more places that aren't safe.

Do you know what those places are, and do you plan to fix them when
you get better? If not, should we record those places somewhere (if
nothing else just on the list)?

-- 
Cheers,

Sverre Rabbelier