[PATCH] index-pack: support --pack-stream to concatenate packs

[PATCH] index-pack: support --pack-stream to concatenate packs

[Nguyễn Thái Ngọc Duy]

--pack-stream --stdin assumes stdin is a stream of packs and creates
a single pack containing all objects of the stream.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 Don't know if it's useful for anybody else. Might be a cheaper way
 than git-repack to reduce the number of packs.

 builtin/index-pack.c |  119 ++++++++++++++++++++++++++++++++++++++++++--------
 1 files changed, 101 insertions(+), 18 deletions(-)

diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index 8dc5c0b..4864913 100644
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -59,6 +59,7 @@ static int nr_deltas;
 static int nr_resolved_deltas;
 
 static int from_stdin;
+static int pack_stream;
 static int strict;
 static int verbose;
 
@@ -68,7 +69,7 @@ static struct progress *progress;
 static unsigned char input_buffer[4096];
 static unsigned int input_offset, input_len;
 static off_t consumed_bytes;
-static git_SHA_CTX input_ctx;
+static git_SHA_CTX input_ctx, pack_stream_ctx;
 static uint32_t input_crc32;
 static int input_fd, output_fd, pack_fd;
 
@@ -115,23 +116,35 @@ static void check_objects(void)
 
 
 /* Discard current buffer used content. */
-static void flush(void)
+static void flush_1(int in_pack_stream)
 {
 	if (input_offset) {
-		if (output_fd >= 0)
-			write_or_die(output_fd, input_buffer, input_offset);
+		if (output_fd >= 0 &&
+		    (!pack_stream || (pack_stream && in_pack_stream)))
+				write_or_die(output_fd, input_buffer, input_offset);
+
 		git_SHA1_Update(&input_ctx, input_buffer, input_offset);
+		if (pack_stream && in_pack_stream)
+			git_SHA1_Update(&pack_stream_ctx, input_buffer, input_offset);
 		memmove(input_buffer, input_buffer + input_offset, input_len);
 		input_offset = 0;
 	}
 }
 
+static void flush(void)
+{
+	flush_1(1);
+}
+
 /*
  * Make sure at least "min" bytes are available in the buffer, and
  * return the pointer to the buffer.
  */
-static void *fill(int min)
+static void *fill_gently(int min, int *got_eof)
 {
+	if (got_eof)
+		*got_eof = 0;
+
 	if (min <= input_len)
 		return input_buffer + input_offset;
 	if (min > sizeof(input_buffer))
@@ -141,9 +154,16 @@ static void *fill(int min)
 		ssize_t ret = xread(input_fd, input_buffer + input_len,
 				sizeof(input_buffer) - input_len);
 		if (ret <= 0) {
-			if (!ret)
-				die("early EOF");
-			die_errno("read error on input");
+			if (!ret) {
+				if (!got_eof)
+					die("early EOF");
+				else {
+					*got_eof = 1;
+					return NULL;
+				}
+			}
+			else
+				die_errno("read error on input");
 		}
 		input_len += ret;
 		if (from_stdin)
@@ -152,6 +172,11 @@ static void *fill(int min)
 	return input_buffer;
 }
 
+static void *fill(int min)
+{
+	return fill_gently(min, NULL);
+}
+
 static void use(int bytes)
 {
 	if (bytes > input_len)
@@ -188,6 +213,8 @@ static const char *open_pack_file(const char *pack_name)
 		pack_fd = input_fd;
 	}
 	git_SHA1_Init(&input_ctx);
+	if (pack_stream)
+		git_SHA1_Init(&pack_stream_ctx);
 	return pack_name;
 }
 
@@ -202,7 +229,7 @@ static void parse_pack_header(void)
 		die("pack version %"PRIu32" unsupported",
 			ntohl(hdr->hdr_version));
 
-	nr_objects = ntohl(hdr->hdr_entries);
+	nr_objects += ntohl(hdr->hdr_entries);
 	use(sizeof(struct pack_header));
 }
 
@@ -590,10 +617,10 @@ static int compare_delta_entry(const void *a, const void *b)
 }
 
 /* Parse all objects and return the pack content SHA1 hash */
-static void parse_pack_objects(unsigned char *sha1)
+static void parse_pack_objects(unsigned char *sha1, int start_object)
 {
 	int i;
-	struct delta_entry *delta = deltas;
+	struct delta_entry *delta = deltas + nr_deltas;
 	struct stat st;
 
 	/*
@@ -606,7 +633,7 @@ static void parse_pack_objects(unsigned char *sha1)
 		progress = start_progress(
 				from_stdin ? "Receiving objects" : "Indexing objects",
 				nr_objects);
-	for (i = 0; i < nr_objects; i++) {
+	for (i = start_object; i < nr_objects; i++) {
 		struct object_entry *obj = &objects[i];
 		void *data = unpack_raw_entry(obj, &delta->base);
 		obj->real_type = obj->type;
@@ -630,11 +657,13 @@ static void parse_pack_objects(unsigned char *sha1)
 	use(20);
 
 	/* If input_fd is a file, we should have reached its end now. */
-	if (fstat(input_fd, &st))
-		die_errno("cannot fstat packfile");
-	if (S_ISREG(st.st_mode) &&
-			lseek(input_fd, 0, SEEK_CUR) - input_len != st.st_size)
-		die("pack has junk at the end");
+	if (!pack_stream) {
+		if (fstat(input_fd, &st))
+			die_errno("cannot fstat packfile");
+		if (S_ISREG(st.st_mode) &&
+		    lseek(input_fd, 0, SEEK_CUR) - input_len != st.st_size)
+			die("pack has junk at the end");
+	}
 
 	if (!nr_deltas)
 		return;
@@ -871,6 +900,54 @@ static int git_index_pack_config(const char *k, const char *v, void *cb)
 	return git_default_config(k, v, cb);
 }
 
+static void parse_following_packs(unsigned char *pack_sha1,
+				  const char *curr_pack,
+				  int fix_thin_pack)
+{
+	unsigned char sha1[20];
+
+	for (;;) {
+		int got_eof;
+		int last_nr_objects = nr_objects;
+
+		/* SHA1 trailer of the previous pack */
+		consumed_bytes -= 20;
+		flush_1(0);
+
+		fill_gently(sizeof(struct pack_header), &got_eof);
+		if (got_eof) {
+			/*
+			 * pretend we also consume the trailer like in
+			 * single pack case, a new trailer will be
+			 * rewritten by fixup_pack_header_footer().
+			 */
+			consumed_bytes += 20;
+			break;
+		}
+
+		git_SHA1_Init(&input_ctx);
+		parse_pack_header();
+		/* un-use() pack headers */
+		consumed_bytes -= sizeof(struct pack_header);
+		flush_1(0);
+
+		objects = xrealloc(objects, (nr_objects + 1) * sizeof(struct object_entry));
+		deltas = xrealloc(deltas, nr_objects * sizeof(struct delta_entry));
+		parse_pack_objects(sha1, last_nr_objects);
+	}
+	git_SHA1_Final(pack_sha1, &pack_stream_ctx);
+
+	if (!fix_thin_pack ||
+	    (nr_deltas - nr_resolved_deltas) == 0) {
+		unsigned char read_sha1[20];
+
+		hashcpy(read_sha1, pack_sha1);
+		fixup_pack_header_footer(output_fd, pack_sha1,
+					 curr_pack, nr_objects,
+					 read_sha1, consumed_bytes-20);
+	}
+}
+
 int cmd_index_pack(int argc, const char **argv, const char *prefix)
 {
 	int i, fix_thin_pack = 0;
@@ -898,6 +975,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
 				from_stdin = 1;
 			} else if (!strcmp(arg, "--fix-thin")) {
 				fix_thin_pack = 1;
+			} else if (!strcmp(arg, "--pack-stream")) {
+				pack_stream = 1;
 			} else if (!strcmp(arg, "--strict")) {
 				strict = 1;
 			} else if (!strcmp(arg, "--keep")) {
@@ -946,6 +1025,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
 		usage(index_pack_usage);
 	if (fix_thin_pack && !from_stdin)
 		die("--fix-thin cannot be used without --stdin");
+	if (pack_stream && !from_stdin)
+		die("--pack-stream cannot be used without --stdin");
 	if (!index_name && pack_name) {
 		int len = strlen(pack_name);
 		if (!has_extension(pack_name, ".pack"))
@@ -971,7 +1052,9 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
 	parse_pack_header();
 	objects = xmalloc((nr_objects + 1) * sizeof(struct object_entry));
 	deltas = xmalloc(nr_objects * sizeof(struct delta_entry));
-	parse_pack_objects(pack_sha1);
+	parse_pack_objects(pack_sha1, 0);
+	if (pack_stream)
+		parse_following_packs(pack_sha1, curr_pack, fix_thin_pack);
 	if (nr_deltas == nr_resolved_deltas) {
 		stop_progress(&progress);
 		/* Flush remaining pack final 20-byte SHA1. */
-- 
1.7.2.2