[RFC PATCH v2 3/4] unpack_trees(): add support for sparse checkout

["Nguyễn Thái Ngọc Duy" <pclouds@gmail.com>]

This patch teaches unpack_trees() to checkout/remove entries on
working directories appropriately when sparse checkout area is
changed. Hook "sparse" is needed to help determine which entry will be
checked out, which will not be.

When the hook is run, it is prepared with a pseudo index. The hook then
can use "git update-index --[no-]assume-unchanged" to manipulate the index.
It should not do anything else on the index. Assume unchanged information
from the index will be used to shape working directory.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 cache.h                     |    3 +
 t/t1009-read-tree-sparse.sh |   42 ++++++++++++
 unpack-trees.c              |  152 +++++++++++++++++++++++++++++++++++++++++--
 unpack-trees.h              |    2 +
 4 files changed, 193 insertions(+), 6 deletions(-)
 create mode 100755 t/t1009-read-tree-sparse.sh

diff --git a/cache.h b/cache.h
index 1a2a3c9..dfad54a 100644
--- a/cache.h
+++ b/cache.h
@@ -177,6 +177,9 @@ struct cache_entry {
 #define CE_HASHED    (0x100000)
 #define CE_UNHASHED  (0x200000)
 
+/* Only remove in work directory, not index */
+#define CE_WT_REMOVE (0x400000)
+
 /*
  * Extended on-disk flags
  */
diff --git a/t/t1009-read-tree-sparse.sh b/t/t1009-read-tree-sparse.sh
new file mode 100755
index 0000000..b613a89
--- /dev/null
+++ b/t/t1009-read-tree-sparse.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+test_description='sparse hook tests'
+
+. ./test-lib.sh
+
+make_hook() {
+	echo "#!/bin/sh" > .git/hooks/sparse &&
+	echo "$1" >> .git/hooks/sparse &&
+	chmod u+x .git/hooks/sparse
+}
+
+test_expect_success setup '
+	echo one > one &&
+	echo two > two &&
+	git add one two &&
+	git commit -m onetwo &&
+	echo three > three &&
+	git add three &&
+	git commit -m three
+'
+
+mkdir .git/hooks
+
+test_expect_success 'failed hook' '
+	make_hook "exit 1" &&
+	test_must_fail git read-tree -u -m HEAD
+'
+
+test_expect_success 'remove one' '
+	make_hook "git update-index --assume-unchanged one"
+	git read-tree -u -m HEAD &&
+	test ! -f one
+'
+
+test_expect_success 're-add one' '
+	make_hook "git update-index --no-assume-unchanged one" &&
+	git read-tree -u -m HEAD &&
+	test -f one
+'
+
+test_done
diff --git a/unpack-trees.c b/unpack-trees.c
index 720f7a1..f407bf5 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -8,6 +8,7 @@
 #include "progress.h"
 #include "refs.h"
 #include "attr.h"
+#include "run-command.h"
 
 /*
  * Error messages expected by scripts out of plumbing commands such as
@@ -32,6 +33,12 @@ static struct unpack_trees_error_msgs unpack_plumbing_errors = {
 
 	/* bind_overlap */
 	"Entry '%s' overlaps with '%s'.  Cannot bind.",
+
+	/* sparse_not_uptodate_file */
+	"Entry '%s' not uptodate. Cannot update sparse checkout.",
+
+	/* would_lose_orphaned */
+	"Orphaned working tree file '%s' would be %s by sparse checkout update.",
 };
 
 #define ERRORMSG(o,fld) \
@@ -78,7 +85,7 @@ static int check_updates(struct unpack_trees_options *o)
 	if (o->update && o->verbose_update) {
 		for (total = cnt = 0; cnt < index->cache_nr; cnt++) {
 			struct cache_entry *ce = index->cache[cnt];
-			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE))
+			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE | CE_WT_REMOVE))
 				total++;
 		}
 
@@ -92,6 +99,13 @@ static int check_updates(struct unpack_trees_options *o)
 	for (i = 0; i < index->cache_nr; i++) {
 		struct cache_entry *ce = index->cache[i];
 
+		if (ce->ce_flags & CE_WT_REMOVE) {
+			display_progress(progress, ++cnt);
+			if (o->update)
+				unlink_entry(ce);
+			continue;
+		}
+
 		if (ce->ce_flags & CE_REMOVE) {
 			display_progress(progress, ++cnt);
 			if (o->update)
@@ -118,6 +132,106 @@ static int check_updates(struct unpack_trees_options *o)
 	return errs != 0;
 }
 
+static int verify_uptodate_sparse(struct cache_entry *ce, struct unpack_trees_options *o);
+static int verify_absent_sparse(struct cache_entry *ce, const char *action, struct unpack_trees_options *o);
+static int run_sparse_hook(struct unpack_trees_options *o)
+{
+	struct index_state *index = &o->result;
+	struct index_state sparse_index = *index;
+	struct cache_entry *ce, *sparse_ce;
+	char sparse_index_file[PATH_MAX];
+	char sparse_index_env[PATH_MAX];
+	const char *argv[2], *env[2];
+	struct child_process cp;
+	int fd, i, j;
+
+	if (access(git_path("hooks/sparse"), X_OK) < 0)
+		return 0;
+
+	strcpy(sparse_index_file, git_path("sparse"));
+	fd = open(sparse_index_file, O_WRONLY | O_CREAT, 0600);
+	if (fd < 0) {
+		error("Unable to open %s for writing", sparse_index_file);
+		return -1;
+	}
+	/* FIXME: write_index may change something */
+	if (write_index(&sparse_index, fd)) {
+		error("Unable to write index to %s", sparse_index_file);
+		close(fd);
+		return -1;
+	}
+	close(fd);
+
+	memset(&cp, 0, sizeof(cp));
+	argv[0] = git_path("hooks/sparse");
+	argv[1] = NULL;
+	cp.argv = argv;
+	cp.no_stdin = 1;
+	cp.stdout_to_stderr = 1;
+	snprintf(sparse_index_env, sizeof(sparse_index_env), "GIT_INDEX_FILE=%s", sparse_index_file);
+	env[0] = sparse_index_env;
+	env[1] = NULL;
+	cp.env = env;
+	if (run_command(&cp)) {
+		error("Failed to run hook 'sparse'");
+		unlink(sparse_index_file);
+		return -1;
+	}
+
+	discard_index(&sparse_index);
+	read_index_from(&sparse_index, sparse_index_file);
+	unlink(sparse_index_file);
+
+	ce = index->cache[0];
+	sparse_ce = sparse_index.cache[0];
+	for (i = j = 0; i < index->cache_nr; i++, ce++) {
+		int was_checkout = !(ce->ce_flags & CE_VALID);
+
+		if (ce_stage(ce))
+			continue;
+
+		/*
+		 * We only care about files getting into the checkout area
+		 * If merge strategies want to remove some, go ahead
+		 */
+		if (ce->ce_flags & CE_REMOVE)
+			continue;
+
+		while (j < sparse_index.cache_nr &&
+		       cache_name_compare(sparse_ce->name, sparse_ce->ce_flags, ce->name, ce->ce_flags) < 0) {
+			sparse_ce++;
+			j++;
+		}
+		if (j < sparse_index.cache_nr &&
+		    !cache_name_compare(sparse_ce->name, sparse_ce->ce_flags, ce->name, ce->ce_flags))
+			ce->ce_flags = (ce->ce_flags & ~CE_VALID) | (sparse_ce->ce_flags & CE_VALID);
+
+		/* Update worktree, add/remove entries if needed */
+
+		if (was_checkout && ce->ce_flags & CE_VALID) {
+			/*
+			 * If CE_UPDATE is set, verify_uptodate() must be called already
+			 * also stat info may have lost after merged_entry() so calling
+			 * verify_uptodate() again may fail
+			 */
+			if (!(ce->ce_flags & CE_UPDATE) && verify_uptodate_sparse(ce, o))
+				return -1;
+			ce->ce_flags |= CE_WT_REMOVE;
+		}
+		if (!was_checkout && !(ce->ce_flags & CE_VALID)) {
+			if (verify_absent_sparse(ce, "overwritten", o))
+				return -1;
+			ce->ce_flags |= CE_UPDATE;
+		}
+
+		/* merge strategies may set CE_UPDATE outside checkout area */
+		if (ce->ce_flags & CE_VALID)
+			ce->ce_flags &= ~CE_UPDATE;
+
+	}
+	return 0;
+}
+
 static inline int call_unpack_fn(struct cache_entry **src, struct unpack_trees_options *o)
 {
 	int ret = o->fn(src, o);
@@ -416,6 +530,9 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options
 	if (o->trivial_merges_only && o->nontrivial_merge)
 		return unpack_failed(o, "Merge requires file-level merging");
 
+	if (run_sparse_hook(o))
+		return unpack_failed(o, NULL);
+
 	o->src_index = NULL;
 	ret = check_updates(o) ? (-2) : 0;
 	if (o->dst_index)
@@ -445,8 +562,9 @@ static int same(struct cache_entry *a, struct cache_entry *b)
  * When a CE gets turned into an unmerged entry, we
  * want it to be up-to-date
  */
-static int verify_uptodate(struct cache_entry *ce,
-		struct unpack_trees_options *o)
+static int verify_uptodate_generic(struct cache_entry *ce,
+				   struct unpack_trees_options *o,
+				   const char *error_msg)
 {
 	struct stat st;
 
@@ -471,7 +589,18 @@ static int verify_uptodate(struct cache_entry *ce,
 	if (errno == ENOENT)
 		return 0;
 	return o->gently ? -1 :
-		error(ERRORMSG(o, not_uptodate_file), ce->name);
+		error(error_msg, ce->name);
+}
+
+static int verify_uptodate(struct cache_entry *ce,
+			   struct unpack_trees_options *o)
+{
+	return verify_uptodate_generic(ce, o, ERRORMSG(o, not_uptodate_file));
+}
+static int verify_uptodate_sparse(struct cache_entry *ce,
+				  struct unpack_trees_options *o)
+{
+	return verify_uptodate_generic(ce, o, ERRORMSG(o, sparse_not_uptodate_file));
 }
 
 static void invalidate_ce_path(struct cache_entry *ce, struct unpack_trees_options *o)
@@ -579,8 +708,9 @@ static int icase_exists(struct unpack_trees_options *o, struct cache_entry *dst,
  * We do not want to remove or overwrite a working tree file that
  * is not tracked, unless it is ignored.
  */
-static int verify_absent(struct cache_entry *ce, const char *action,
-			 struct unpack_trees_options *o)
+static int verify_absent_generic(struct cache_entry *ce, const char *action,
+				 struct unpack_trees_options *o,
+				 const char *error_msg)
 {
 	struct stat st;
 
@@ -660,6 +790,16 @@ static int verify_absent(struct cache_entry *ce, const char *action,
 	}
 	return 0;
 }
+static int verify_absent(struct cache_entry *ce, const char *action,
+			 struct unpack_trees_options *o)
+{
+	return verify_absent_generic(ce, action, o, ERRORMSG(o, would_lose_untracked));
+}
+static int verify_absent_sparse(struct cache_entry *ce, const char *action,
+			 struct unpack_trees_options *o)
+{
+	return verify_absent_generic(ce, action, o, ERRORMSG(o, would_lose_orphaned));
+}
 
 static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
 		struct unpack_trees_options *o)
diff --git a/unpack-trees.h b/unpack-trees.h
index d19df44..ad21823 100644
--- a/unpack-trees.h
+++ b/unpack-trees.h
@@ -14,6 +14,8 @@ struct unpack_trees_error_msgs {
 	const char *not_uptodate_dir;
 	const char *would_lose_untracked;
 	const char *bind_overlap;
+	const char *sparse_not_uptodate_file;
+	const char *would_lose_orphaned;
 };
 
 struct unpack_trees_options {
-- 
1.6.3.GIT