[PATCH v2 11/13] loose: map object IDs using pack index v3

[Christian Couder <christian.couder@gmail.com>]

From: "brian m. carlson" <sandals@crustytoothpaste.net>

Right now, we can only map an object if it exists in the loose object
map.  However, we have a perfectly good way to look up packed objects,
which is the pack index.  Let's use that to look up any packed objects
if they're not in the loose object map.

In order to do so, let's add a variant of nth_packed_object_id that
handles an arbitrary hash algorithm.

Only pack index v3 stores object IDs in more than one algorithm, so
skip packs with an older index when mapping.

Also make the new variant fail when the index cannot provide the
requested object ID, for example because it has no data for one of the
algorithms.

Add a test in t1016 to check that packed objects are mapped using a v3
index, but not using a v2 index, when the loose object map has no entry
for them.

Signed-off-by: brian m. carlson <sandals@crustytoothpaste.net>
Signed-off-by: Christian Couder <christian.couder@gmail.com>
---
 odb/source-packed.c           | 30 ++++++++++++++++++++++++++----
 packfile.c                    | 28 +++++++++++++++++++++++-----
 packfile.h                    |  8 ++++++++
 t/t1016-compatObjectFormat.sh | 27 +++++++++++++++++++++++++++
 4 files changed, 84 insertions(+), 9 deletions(-)

diff --git a/odb/source-packed.c b/odb/source-packed.c
index ee1edfdb2b..4ecc2de818 100644
--- a/odb/source-packed.c
+++ b/odb/source-packed.c
@@ -863,12 +863,34 @@ static void odb_source_packed_close(struct odb_source *source)
 	packed->midx = NULL;
 }
 
-static int odb_source_packed_map_oid(struct odb_source *source UNUSED,
-				     const struct object_id *src UNUSED,
-				     const struct git_hash_algo *to UNUSED,
-				     struct object_id *dest UNUSED,
+static int odb_source_packed_map_oid(struct odb_source *source,
+				     const struct object_id *src,
+				     const struct git_hash_algo *to,
+				     struct object_id *dest,
 				     enum odb_map_oid_flags flags UNUSED)
 {
+	struct odb_source_packed *packed = odb_source_packed_downcast(source);
+	struct repository *repo = packed->base.odb->repo;
+	struct packfile_list_entry *e;
+
+	for (e = packfile_store_get_packs(packed); e; e = e->next) {
+		struct packed_git *p = e->pack;
+		uint32_t off;
+		const struct git_hash_algo *src_algo = src->algo ?
+						       &hash_algos[src->algo] :
+						       repo->hash_algo;
+
+		if (open_pack_index(p) || !p->num_objects)
+			continue;
+		/* Only v3 indices store object IDs in both algorithms. */
+		if (p->index_version < 3)
+			continue;
+		if (!bsearch_pack(src, p, &off))
+			continue;
+		if (nth_packed_object_id_algop(dest, p, off, src_algo, to))
+			continue;
+		return 0;
+	}
 	return -1;
 }
 
diff --git a/packfile.c b/packfile.c
index f6e2db45d5..3bfd427c6c 100644
--- a/packfile.c
+++ b/packfile.c
@@ -1861,9 +1861,19 @@ int bsearch_pack(const struct object_id *oid, const struct packed_git *p, uint32
 int nth_packed_object_id(struct object_id *oid,
 			 struct packed_git *p,
 			 uint32_t n)
+{
+	return nth_packed_object_id_algop(oid, p, n, p->repo->hash_algo,
+					  p->repo->hash_algo);
+}
+
+int nth_packed_object_id_algop(struct object_id *oid,
+			       struct packed_git *p,
+			       uint32_t n,
+			       const struct git_hash_algo *src,
+			       const struct git_hash_algo *dest)
 {
 	const unsigned char *index = p->index_data;
-	const unsigned int hashsz = p->repo->hash_algo->rawsz;
+	const unsigned int hashsz = dest->rawsz;
 	if (!index) {
 		if (open_pack_index(p))
 			return -1;
@@ -1871,17 +1881,25 @@ int nth_packed_object_id(struct object_id *oid,
 	}
 	if (n >= p->num_objects)
 		return -1;
+	if (p->index_version < 3 && (src != dest || dest != p->repo->hash_algo))
+		return -1; /* older indices only store main object IDs */
 	if (p->index_version == 1) {
 		index += 4 * 256;
 		oidread(oid, index + st_add(st_mult(hashsz + 4, n), 4),
-			p->repo->hash_algo);
+			dest);
 	} else if (p->index_version == 2) {
 		index += 4 * 256 + 8;
-		oidread(oid, index + st_mult(hashsz, n), p->repo->hash_algo);
+		oidread(oid, index + st_mult(hashsz, n), dest);
 	} else {
-		int algo = hash_algo_by_ptr(p->repo->hash_algo);
+		int algo = hash_algo_by_ptr(dest);
 		const struct packed_git_format *pfp = &p->formats[algo-1];
-		oidread(oid, index + pfp->full_oid_offset + (hashsz * n), p->repo->hash_algo);
+		uint32_t pack_order_idx;
+
+		if (!pfp->data_offset ||
+		    !p->formats[hash_algo_by_ptr(src) - 1].data_offset)
+			return -1; /* no data for one of the algorithms */
+		pack_order_idx = nth_packed_object_pack_order_algop(p, n, src);
+		oidread(oid, index + st_add(pfp->full_oid_offset, st_mult(hashsz, pack_order_idx)), dest);
 	}
 	return 0;
 }
diff --git a/packfile.h b/packfile.h
index 22278cc6c6..73452b2a43 100644
--- a/packfile.h
+++ b/packfile.h
@@ -289,6 +289,14 @@ int bsearch_pack(const struct object_id *oid, const struct packed_git *p, uint32
  */
 int nth_packed_object_id(struct object_id *, struct packed_git *, uint32_t n);
 
+/*
+ * The same as nth_packed_object_id, but the nth object is counted in src
+ * and the object ID looked up is in dest.
+ */
+int nth_packed_object_id_algop(struct object_id *, struct packed_git *,
+			       uint32_t n, const struct git_hash_algo *src,
+			       const struct git_hash_algo *dest);
+
 /*
  * Return the offset of the nth object within the specified packfile.
  * The index must already be opened.
diff --git a/t/t1016-compatObjectFormat.sh b/t/t1016-compatObjectFormat.sh
index af9532aa1f..030d6000b5 100755
--- a/t/t1016-compatObjectFormat.sh
+++ b/t/t1016-compatObjectFormat.sh
@@ -242,6 +242,33 @@ do
 		git -C large-repo cat-file blob :large >actual &&
 		test_cmp large-repo/large actual
 	'
+
+	test_expect_success 'packed objects are only mapped using pack index v3' '
+		test_when_finished "rm -rf map-repo v3.idx loose-object-idx" &&
+		create_compat_repo $hash map-repo &&
+		echo content >map-repo/file &&
+		git -C map-repo add file &&
+		git -C map-repo commit --message A &&
+		oid=$(git -C map-repo rev-parse HEAD) &&
+		git -C map-repo rev-parse --output-object-format=$(compat_hash $hash) "$oid" >expect &&
+		git -C map-repo repack -a -d &&
+		pack=$(echo map-repo/.git/objects/pack/pack-*.pack) &&
+		git -C map-repo index-pack --index-version=3 \
+			-o "$(pwd)/v3.idx" "$(pwd)/$pack" &&
+		mv map-repo/.git/objects/loose-object-idx loose-object-idx &&
+
+		# A v2 index only has object IDs in the main algorithm, so it
+		# must not provide a mapping.
+		git -C map-repo rev-parse --output-object-format=$(compat_hash $hash) "$oid" >actual &&
+		! test_cmp expect actual &&
+		test $(wc -c <actual) -ne $(($(test_oid --hash=$(compat_hash $hash) hexsz) + 1)) &&
+
+		# A v3 index provides it.
+		rm -f "${pack%.pack}.idx" "${pack%.pack}.rev" &&
+		mv v3.idx "${pack%.pack}.idx" &&
+		git -C map-repo rev-parse --output-object-format=$(compat_hash $hash) "$oid" >actual &&
+		test_cmp expect actual
+	'
 done
 cd "$base"
 
-- 
2.56.0.215.g47fdf2208c