[PATCH 1/2] index-pack: rationalize delta resolution code

[PATCH 1/2] index-pack: rationalize delta resolution code

[Nicolas Pitre]

Instead of having strange loops for walking unresolved deltas with the
same base duplicated in many places, let's rework the code so this is
done in a single place instead.  This simplifies callers quite a bit too.

Signed-off-by: Nicolas Pitre <nico@cam.org>
---
 index-pack.c |  123 +++++++++++++++++++++++++++------------------------------
 1 files changed, 58 insertions(+), 65 deletions(-)

diff --git a/index-pack.c b/index-pack.c
index d3a4d31..82843e7 100644
--- a/index-pack.c
+++ b/index-pack.c
@@ -244,7 +244,8 @@ static void link_base_data(struct base_data *base, struct base_data *c)
 
 	c->base = base;
 	c->child = NULL;
-	base_cache_used += c->size;
+	if (c->data)
+		base_cache_used += c->size;
 	prune_base_data(c);
 }
 
@@ -494,8 +495,10 @@ static void *get_base_data(struct base_data *c)
 			free(raw);
 			if (!c->data)
 				bad_object(obj->idx.offset, "failed to apply delta");
-		} else
+		} else {
 			c->data = get_data_from_pack(obj);
+			c->size = obj->size;
+		}
 
 		base_cache_used += c->size;
 		prune_base_data(c);
@@ -504,49 +507,73 @@ static void *get_base_data(struct base_data *c)
 }
 
 static void resolve_delta(struct object_entry *delta_obj,
-			  struct base_data *base_obj, enum object_type type)
+			  struct base_data *base, struct base_data *result)
 {
 	void *delta_data;
 	unsigned long delta_size;
-	union delta_base delta_base;
-	int j, first, last;
-	struct base_data result;
 
-	delta_obj->real_type = type;
+	delta_obj->real_type = base->obj->type;
 	delta_data = get_data_from_pack(delta_obj);
 	delta_size = delta_obj->size;
-	result.data = patch_delta(get_base_data(base_obj), base_obj->size,
-			     delta_data, delta_size,
-			     &result.size);
+	result->obj = delta_obj;
+	result->data = patch_delta(get_base_data(base), base->obj->size,
+				   delta_data, delta_size, &result->size);
 	free(delta_data);
-	if (!result.data)
+	if (!result->data)
 		bad_object(delta_obj->idx.offset, "failed to apply delta");
-	sha1_object(result.data, result.size, type, delta_obj->idx.sha1);
+	sha1_object(result->data, result->size, delta_obj->real_type,
+		    delta_obj->idx.sha1);
 	nr_resolved_deltas++;
+}
+
+static void find_unresolved_deltas(struct base_data *base,
+				   struct base_data *prev_base)
+{
+	int i, ref, ref_first, ref_last, ofs, ofs_first, ofs_last;
+
+	/*
+	 * This is a recursive function. Those brackets should help reducing
+	 * stack usage by limiting the scope of the delta_base union.
+	 */
+	{
+		union delta_base base_spec;
+
+		hashcpy(base_spec.sha1, base->obj->idx.sha1);
+		ref = !find_delta_children(&base_spec, &ref_first, &ref_last);
+
+		memset(&base_spec, 0, sizeof(base_spec));
+		base_spec.offset = base->obj->idx.offset;
+		ofs = !find_delta_children(&base_spec, &ofs_first, &ofs_last);
+	}
+
+	if (!ref && !ofs)
+		return;
 
-	result.obj = delta_obj;
-	link_base_data(base_obj, &result);
+	link_base_data(prev_base, base);
 
-	hashcpy(delta_base.sha1, delta_obj->idx.sha1);
-	if (!find_delta_children(&delta_base, &first, &last)) {
-		for (j = first; j <= last; j++) {
-			struct object_entry *child = objects + deltas[j].obj_no;
-			if (child->real_type == OBJ_REF_DELTA)
-				resolve_delta(child, &result, type);
+	if (ref) {
+		for (i = ref_first; i <= ref_last; i++) {
+			struct object_entry *child = objects + deltas[i].obj_no;
+			if (child->real_type == OBJ_REF_DELTA) {
+				struct base_data result;
+				resolve_delta(child, base, &result);
+				find_unresolved_deltas(&result, base);
+			}
 		}
 	}
 
-	memset(&delta_base, 0, sizeof(delta_base));
-	delta_base.offset = delta_obj->idx.offset;
-	if (!find_delta_children(&delta_base, &first, &last)) {
-		for (j = first; j <= last; j++) {
-			struct object_entry *child = objects + deltas[j].obj_no;
-			if (child->real_type == OBJ_OFS_DELTA)
-				resolve_delta(child, &result, type);
+	if (ofs) {
+		for (i = ofs_first; i <= ofs_last; i++) {
+			struct object_entry *child = objects + deltas[i].obj_no;
+			if (child->real_type == OBJ_OFS_DELTA) {
+				struct base_data result;
+				resolve_delta(child, base, &result);
+				find_unresolved_deltas(&result, base);
+			}
 		}
 	}
 
-	unlink_base_data(&result);
+	unlink_base_data(base);
 }
 
 static int compare_delta_entry(const void *a, const void *b)
@@ -622,37 +649,13 @@ static void parse_pack_objects(unsigned char *sha1)
 		progress = start_progress("Resolving deltas", nr_deltas);
 	for (i = 0; i < nr_objects; i++) {
 		struct object_entry *obj = &objects[i];
-		union delta_base base;
-		int j, ref, ref_first, ref_last, ofs, ofs_first, ofs_last;
 		struct base_data base_obj;
 
 		if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA)
 			continue;
-		hashcpy(base.sha1, obj->idx.sha1);
-		ref = !find_delta_children(&base, &ref_first, &ref_last);
-		memset(&base, 0, sizeof(base));
-		base.offset = obj->idx.offset;
-		ofs = !find_delta_children(&base, &ofs_first, &ofs_last);
-		if (!ref && !ofs)
-			continue;
-		base_obj.data = get_data_from_pack(obj);
-		base_obj.size = obj->size;
 		base_obj.obj = obj;
-		link_base_data(NULL, &base_obj);
-
-		if (ref)
-			for (j = ref_first; j <= ref_last; j++) {
-				struct object_entry *child = objects + deltas[j].obj_no;
-				if (child->real_type == OBJ_REF_DELTA)
-					resolve_delta(child, &base_obj, obj->type);
-			}
-		if (ofs)
-			for (j = ofs_first; j <= ofs_last; j++) {
-				struct object_entry *child = objects + deltas[j].obj_no;
-				if (child->real_type == OBJ_OFS_DELTA)
-					resolve_delta(child, &base_obj, obj->type);
-			}
-		unlink_base_data(&base_obj);
+		base_obj.data = NULL;
+		find_unresolved_deltas(&base_obj, NULL);
 		display_progress(progress, nr_resolved_deltas);
 	}
 }
@@ -745,7 +748,6 @@ static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved)
 	for (i = 0; i < n; i++) {
 		struct delta_entry *d = sorted_by_pos[i];
 		enum object_type type;
-		int j, first, last;
 		struct base_data base_obj;
 
 		if (objects[d->obj_no].real_type != OBJ_REF_DELTA)
@@ -759,16 +761,7 @@ static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved)
 			die("local object %s is corrupt", sha1_to_hex(d->base.sha1));
 		base_obj.obj = append_obj_to_pack(f, d->base.sha1,
 					base_obj.data, base_obj.size, type);
-		link_base_data(NULL, &base_obj);
-
-		find_delta_children(&d->base, &first, &last);
-		for (j = first; j <= last; j++) {
-			struct object_entry *child = objects + deltas[j].obj_no;
-			if (child->real_type == OBJ_REF_DELTA)
-				resolve_delta(child, &base_obj, type);
-		}
-
-		unlink_base_data(&base_obj);
+		find_unresolved_deltas(&base_obj, NULL);
 		display_progress(progress, nr_resolved_deltas);
 	}
 	free(sorted_by_pos);
-- 
1.6.0.2.711.gf1ba4

[PATCH 2/2] index-pack: smarter memory usage during delta resolution

[Nicolas Pitre]

There is no need to keep the base object data around after its last delta
has been resolved.  This also means that long delta chains with only one
delta per base won't grow the cache size unnecessarily as the base will
be freed before recursing down.

To make it easy, find_delta_children() is modified so the first and last
indices are initialized in all cases.

Signed-off-by: Nicolas Pitre <nico@cam.org>
---
 index-pack.c |   73 +++++++++++++++++++++++++++++++---------------------------
 1 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/index-pack.c b/index-pack.c
index 82843e7..9def641 100644
--- a/index-pack.c
+++ b/index-pack.c
@@ -221,17 +221,23 @@ static void bad_object(unsigned long offset, const char *format, ...)
 	die("pack has bad object at offset %lu: %s", offset, buf);
 }
 
+static void free_base_data(struct base_data *c)
+{
+	if (c->data) {
+		free(c->data);
+		c->data = NULL;
+		base_cache_used -= c->size;
+	}
+}
+
 static void prune_base_data(struct base_data *retain)
 {
 	struct base_data *b = base_cache;
 	for (b = base_cache;
 	     base_cache_used > delta_base_cache_limit && b;
 	     b = b->child) {
-		if (b->data && b != retain) {
-			free(b->data);
-			b->data = NULL;
-			base_cache_used -= b->size;
-		}
+		if (b->data && b != retain)
+			free_base_data(b);
 	}
 }
 
@@ -256,10 +262,7 @@ static void unlink_base_data(struct base_data *c)
 		base->child = NULL;
 	else
 		base_cache = NULL;
-	if (c->data) {
-		free(c->data);
-		base_cache_used -= c->size;
-	}
+	free_base_data(c);
 }
 
 static void *unpack_entry_data(unsigned long offset, unsigned long size)
@@ -409,22 +412,24 @@ static int find_delta(const union delta_base *base)
         return -first-1;
 }
 
-static int find_delta_children(const union delta_base *base,
-			       int *first_index, int *last_index)
+static void find_delta_children(const union delta_base *base,
+				int *first_index, int *last_index)
 {
 	int first = find_delta(base);
 	int last = first;
 	int end = nr_deltas - 1;
 
-	if (first < 0)
-		return -1;
+	if (first < 0) {
+		*first_index = 0;
+		*last_index = -1;
+		return;
+	}
 	while (first > 0 && !memcmp(&deltas[first - 1].base, base, UNION_BASE_SZ))
 		--first;
 	while (last < end && !memcmp(&deltas[last + 1].base, base, UNION_BASE_SZ))
 		++last;
 	*first_index = first;
 	*last_index = last;
-	return 0;
 }
 
 static void sha1_object(const void *data, unsigned long size,
@@ -529,7 +534,7 @@ static void resolve_delta(struct object_entry *delta_obj,
 static void find_unresolved_deltas(struct base_data *base,
 				   struct base_data *prev_base)
 {
-	int i, ref, ref_first, ref_last, ofs, ofs_first, ofs_last;
+	int i, ref_first, ref_last, ofs_first, ofs_last;
 
 	/*
 	 * This is a recursive function. Those brackets should help reducing
@@ -539,37 +544,37 @@ static void find_unresolved_deltas(struct base_data *base,
 		union delta_base base_spec;
 
 		hashcpy(base_spec.sha1, base->obj->idx.sha1);
-		ref = !find_delta_children(&base_spec, &ref_first, &ref_last);
+		find_delta_children(&base_spec, &ref_first, &ref_last);
 
 		memset(&base_spec, 0, sizeof(base_spec));
 		base_spec.offset = base->obj->idx.offset;
-		ofs = !find_delta_children(&base_spec, &ofs_first, &ofs_last);
+		find_delta_children(&base_spec, &ofs_first, &ofs_last);
 	}
 
-	if (!ref && !ofs)
+	if (ref_last == -1 && ofs_last == -1)
 		return;
 
 	link_base_data(prev_base, base);
 
-	if (ref) {
-		for (i = ref_first; i <= ref_last; i++) {
-			struct object_entry *child = objects + deltas[i].obj_no;
-			if (child->real_type == OBJ_REF_DELTA) {
-				struct base_data result;
-				resolve_delta(child, base, &result);
-				find_unresolved_deltas(&result, base);
-			}
+	for (i = ref_first; i <= ref_last; i++) {
+		struct object_entry *child = objects + deltas[i].obj_no;
+		if (child->real_type == OBJ_REF_DELTA) {
+			struct base_data result;
+			resolve_delta(child, base, &result);
+			if (i == ref_last && ofs_last == -1)
+				free_base_data(base);
+			find_unresolved_deltas(&result, base);
 		}
 	}
 
-	if (ofs) {
-		for (i = ofs_first; i <= ofs_last; i++) {
-			struct object_entry *child = objects + deltas[i].obj_no;
-			if (child->real_type == OBJ_OFS_DELTA) {
-				struct base_data result;
-				resolve_delta(child, base, &result);
-				find_unresolved_deltas(&result, base);
-			}
+	for (i = ofs_first; i <= ofs_last; i++) {
+		struct object_entry *child = objects + deltas[i].obj_no;
+		if (child->real_type == OBJ_OFS_DELTA) {
+			struct base_data result;
+			resolve_delta(child, base, &result);
+			if (i == ofs_last)
+				free_base_data(base);
+			find_unresolved_deltas(&result, base);
 		}
 	}
 
-- 
1.6.0.2.711.gf1ba4