[RFC PATCH 14/18] midx: load midx files when loading packs

[Derrick Stolee <stolee@gmail.com>]

Replace prepare_packed_git() with prepare_packed_git_internal(use_midx) to
allow some consumers of prepare_packed_git() with a way to load MIDX files.
Consumers should only use the new method if they are prepared to use the
midxed_git struct alongside the packed_git struct.

If a packfile is found that is not referenced by the current MIDX, then add
it to the packed_git struct. This is important to keep the MIDX useful after
adding packs due to "fetch" commands and when third-party tools (such as
libgit2) add packs directly to the repo.

If prepare_packed_git_internal is called with use_midx = 0, then unload the
MIDX file and reload the packfiles in to the packed_git struct.

Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
---
 midx.c     | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 midx.h     |  6 ++++--
 packfile.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++---------
 packfile.h |  1 +
 4 files changed, 117 insertions(+), 11 deletions(-)

diff --git a/midx.c b/midx.c
index 3ce2b736ea..a66763b9e3 100644
--- a/midx.c
+++ b/midx.c
@@ -22,6 +22,9 @@
 
 #define MIDX_LARGE_OFFSET_NEEDED 0x80000000
 
+/* MIDX-git global storage */
+struct midxed_git *midxed_git = 0;
+
 char* get_midx_filename_oid(const char *pack_dir,
 			    struct object_id *oid)
 {
@@ -197,6 +200,45 @@ struct midxed_git *get_midxed_git(const char *pack_dir, struct object_id *oid)
 	return m;
 }
 
+static char* get_midx_filename_dir(const char *pack_dir)
+{
+	struct object_id oid;
+	if (!get_midx_head_oid(pack_dir, &oid))
+		return 0;
+
+	return get_midx_filename_oid(pack_dir, &oid);
+}
+
+static int prepare_midxed_git_head(char *pack_dir, int local)
+{
+	struct midxed_git *m = midxed_git;
+	char *midx_head_path = get_midx_filename_dir(pack_dir);
+
+	if (!core_midx)
+		return 1;
+
+	if (midx_head_path) {
+		midxed_git = load_midxed_git_one(midx_head_path, pack_dir);
+		midxed_git->next = m;
+		FREE_AND_NULL(midx_head_path);
+		return 1;
+	}
+
+	return 0;
+}
+
+int prepare_midxed_git_objdir(char *obj_dir, int local)
+{
+	int ret;
+	struct strbuf pack_dir = STRBUF_INIT;
+	strbuf_addstr(&pack_dir, obj_dir);
+	strbuf_add(&pack_dir, "/pack", 5);
+
+	ret = prepare_midxed_git_head(pack_dir.buf, local);
+	strbuf_release(&pack_dir);
+	return ret;
+}
+
 struct pack_midx_details_internal {
 	uint32_t pack_int_id;
 	uint32_t internal_offset;
@@ -677,3 +719,18 @@ int close_midx(struct midxed_git *m)
 
 	return 1;
 }
+
+void close_all_midx(void)
+{
+	struct midxed_git *m = midxed_git;
+	struct midxed_git *next;
+
+	while (m) {
+		next = m->next;
+		close_midx(m);
+		free(m);
+		m = next;
+	}
+
+	midxed_git = 0;
+}
diff --git a/midx.h b/midx.h
index 27d48163e9..d8ede8121c 100644
--- a/midx.h
+++ b/midx.h
@@ -27,7 +27,7 @@ struct pack_midx_header {
 	uint32_t num_packs;
 };
 
-struct midxed_git {
+extern struct midxed_git {
 	struct midxed_git *next;
 
 	int midx_fd;
@@ -81,9 +81,10 @@ struct midxed_git {
 
 	/* something like ".git/objects/pack" */
 	char pack_dir[FLEX_ARRAY]; /* more */
-};
+} *midxed_git;
 
 extern struct midxed_git *get_midxed_git(const char *pack_dir, struct object_id *oid);
+extern int prepare_midxed_git_objdir(char *obj_dir, int local);
 
 struct pack_midx_details {
 	uint32_t pack_int_id;
@@ -118,5 +119,6 @@ extern const char *write_midx_file(const char *pack_dir,
 				   uint32_t nr_objects);
 
 extern int close_midx(struct midxed_git *m);
+extern void close_all_midx(void);
 
 #endif
diff --git a/packfile.c b/packfile.c
index c36420b33f..1c0822878b 100644
--- a/packfile.c
+++ b/packfile.c
@@ -8,6 +8,7 @@
 #include "list.h"
 #include "streaming.h"
 #include "sha1-lookup.h"
+#include "midx.h"
 
 char *odb_pack_name(struct strbuf *buf,
 		    const unsigned char *sha1,
@@ -309,10 +310,22 @@ void close_pack(struct packed_git *p)
 void close_all_packs(void)
 {
 	struct packed_git *p;
+	struct midxed_git *m;
+
+	for (m = midxed_git; m; m = m->next) {
+		int i;
+		for (i = 0; i < m->num_packs; i++) {
+			p = m->packs[i];
+			if (p && p->do_not_close)
+				BUG("want to close pack marked 'do-not-close'");
+			else if (p)
+				close_pack(p);
+		}
+	}
 
 	for (p = packed_git; p; p = p->next)
 		if (p->do_not_close)
-			die("BUG: want to close pack marked 'do-not-close'");
+			BUG("want to close pack marked 'do-not-close'");
 		else
 			close_pack(p);
 }
@@ -748,6 +761,7 @@ static void prepare_packed_git_one(char *objdir, int local)
 	dirnamelen = path.len;
 	while ((de = readdir(dir)) != NULL) {
 		struct packed_git *p;
+		struct midxed_git *m;
 		size_t base_len;
 
 		if (is_dot_or_dotdot(de->d_name))
@@ -758,15 +772,23 @@ static void prepare_packed_git_one(char *objdir, int local)
 
 		base_len = path.len;
 		if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
+			strbuf_setlen(&path, base_len + 1);
+			strbuf_add(&path, "pack", 4);
+
 			/* Don't reopen a pack we already have. */
+			for (m = midxed_git; m; m = m->next)
+				if (!memcmp(m->pack_dir, path.buf, dirnamelen - 1) &&
+				    contains_pack(m, path.buf + dirnamelen))
+					break;
 			for (p = packed_git; p; p = p->next) {
-				size_t len;
-				if (strip_suffix(p->pack_name, ".pack", &len) &&
-				    len == base_len &&
-				    !memcmp(p->pack_name, path.buf, len))
+				if (!strcmp(p->pack_name, path.buf))
 					break;
 			}
-			if (p == NULL &&
+
+			strbuf_setlen(&path, base_len + 1);
+			strbuf_add(&path, "idx", 3);
+
+			if (m == NULL && p == NULL &&
 			    /*
 			     * See if it really is a valid .idx file with
 			     * corresponding .pack file that we can map.
@@ -872,21 +894,45 @@ static void prepare_packed_git_mru(void)
 }
 
 static int prepare_packed_git_run_once = 0;
-void prepare_packed_git(void)
+static int prepare_midxed_git_run_once = 0;
+void prepare_packed_git_internal(int use_midx)
 {
 	struct alternate_object_database *alt;
+	char *obj_dir;
+
+	if (prepare_midxed_git_run_once) {
+		if (!use_midx) {
+			prepare_midxed_git_run_once = 0;
+			close_all_midx();
+			reprepare_packed_git();
+		}
+		return;
+	}
 
 	if (prepare_packed_git_run_once)
 		return;
-	prepare_packed_git_one(get_object_directory(), 1);
+
+	obj_dir = get_object_directory();
+
+	if (use_midx)
+		prepare_midxed_git_objdir(obj_dir, 1);
+	prepare_packed_git_one(obj_dir, 1);
 	prepare_alt_odb();
-	for (alt = alt_odb_list; alt; alt = alt->next)
+	for (alt = alt_odb_list; alt; alt = alt->next) {
+		if (use_midx)
+			prepare_midxed_git_objdir(alt->path, 0);
 		prepare_packed_git_one(alt->path, 0);
+	}
 	rearrange_packed_git();
 	prepare_packed_git_mru();
 	prepare_packed_git_run_once = 1;
+	prepare_midxed_git_run_once = use_midx;
 }
 
+void prepare_packed_git(void)
+{
+	prepare_packed_git_internal(0);
+}
 void reprepare_packed_git(void)
 {
 	approximate_object_count_valid = 0;
diff --git a/packfile.h b/packfile.h
index 7cf4771029..25bac91efb 100644
--- a/packfile.h
+++ b/packfile.h
@@ -32,6 +32,7 @@ extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_
 #define PACKDIR_FILE_GARBAGE 4
 extern void (*report_garbage)(unsigned seen_bits, const char *path);
 
+extern void prepare_packed_git_internal(int use_midx);
 extern void prepare_packed_git(void);
 extern void reprepare_packed_git(void);
 extern void install_packed_git(struct packed_git *pack);
-- 
2.15.0