[RFC PATCH v3 7/8] Support sparse checkout in unpack_trees() and read-tree

["Nguyễn Thái Ngọc Duy" <pclouds@gmail.com>]

This patch makes unpack_trees() look at .git/info/sparse [1] to
determine which files should stay in working directory, after
merging, by:

 - setting CE_VALID properly so that other operations correctly ignore
   missing files
 - driving check_updates() to add/remove files in accordance to
   CE_VALID

The feature is disabled by default. Use "read-tree --sparse" to enable it.

[1] .git/info/sparse has the same syntax as .git/info/exclude. Files
that match the patterns will be set as CE_VALID.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 builtin-read-tree.c         |    4 +-
 cache.h                     |    3 +
 t/t1009-read-tree-sparse.sh |   47 ++++++++++++++++++++
 unpack-trees.c              |   98 ++++++++++++++++++++++++++++++++++++++++++-
 unpack-trees.h              |    3 +
 5 files changed, 153 insertions(+), 2 deletions(-)
 create mode 100755 t/t1009-read-tree-sparse.sh

diff --git a/builtin-read-tree.c b/builtin-read-tree.c
index 9c2d634..888f136 100644
--- a/builtin-read-tree.c
+++ b/builtin-read-tree.c
@@ -31,7 +31,7 @@ static int list_tree(unsigned char *sha1)
 }
 
 static const char * const read_tree_usage[] = {
-	"git read-tree [[-m [--trivial] [--aggressive] | --reset | --prefix=<prefix>] [-u [--exclude-per-directory=<gitignore>] | -i]]  [--index-output=<file>] <tree-ish1> [<tree-ish2> [<tree-ish3>]]",
+	"git read-tree [[-m [--trivial] [--aggressive] | --reset | --prefix=<prefix>] [-u [--exclude-per-directory=<gitignore>] | -i]] [--sparse] [--index-output=<file>] <tree-ish1> [<tree-ish2> [<tree-ish3>]]",
 	NULL
 };
 
@@ -98,6 +98,8 @@ int cmd_read_tree(int argc, const char **argv, const char *unused_prefix)
 		  PARSE_OPT_NONEG, exclude_per_directory_cb },
 		OPT_SET_INT('i', NULL, &opts.index_only,
 			    "don't check the working tree after merging", 1),
+		OPT_SET_INT(0, "sparse", &opts.apply_sparse,
+			    "apply sparse checkout filter", 1),
 		OPT_END()
 	};
 
diff --git a/cache.h b/cache.h
index 1a2a3c9..dfad54a 100644
--- a/cache.h
+++ b/cache.h
@@ -177,6 +177,9 @@ struct cache_entry {
 #define CE_HASHED    (0x100000)
 #define CE_UNHASHED  (0x200000)
 
+/* Only remove in work directory, not index */
+#define CE_WT_REMOVE (0x400000)
+
 /*
  * Extended on-disk flags
  */
diff --git a/t/t1009-read-tree-sparse.sh b/t/t1009-read-tree-sparse.sh
new file mode 100755
index 0000000..f70852c
--- /dev/null
+++ b/t/t1009-read-tree-sparse.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+
+test_description='sparse checkout tests'
+
+. ./test-lib.sh
+
+test_expect_success 'setup' '
+	test_commit one &&
+	mkdir two &&
+	test_commit two two/two.t two.t
+'
+
+test_expect_success 'read-tree without .git/info/sparse' '
+	git read-tree --sparse -m -u HEAD &&
+	test -f one.t &&
+	test -f two/two.t
+'
+
+test_expect_success 'read-tree with empty .git/info/sparse' '
+	echo > .git/info/sparse &&
+	git read-tree --sparse -m -u HEAD &&
+	test -f one.t &&
+	test -f two/two.t
+'
+
+test_expect_success 'read-tree --sparse' '
+	echo "one.t" > .git/info/sparse &&
+	git read-tree --sparse -m -u HEAD &&
+	test ! -f one.t &&
+	test -f two/two.t
+'
+
+test_expect_success 'read-tree --sparse foo where foo is "directory"' '
+	echo "two" > .git/info/sparse &&
+	git read-tree --sparse -m -u HEAD &&
+	test -f one.t &&
+	test -f two/two.t
+'
+
+test_expect_success 'read-tree --sparse foo/' '
+	echo "two/" > .git/info/sparse &&
+	git read-tree --sparse -m -u HEAD &&
+	test -f one.t &&
+	test ! -f two/two.t
+'
+
+test_done
diff --git a/unpack-trees.c b/unpack-trees.c
index 02ea236..d18d333 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -32,6 +32,12 @@ static struct unpack_trees_error_msgs unpack_plumbing_errors = {
 
 	/* bind_overlap */
 	"Entry '%s' overlaps with '%s'.  Cannot bind.",
+
+	/* sparse_not_uptodate_file */
+	"Entry '%s' not uptodate. Cannot update sparse checkout.",
+
+	/* would_lose_orphaned */
+	"Working tree file '%s' would be %s by sparse checkout update.",
 };
 
 #define ERRORMSG(o,fld) \
@@ -78,7 +84,7 @@ static int check_updates(struct unpack_trees_options *o)
 	if (o->update && o->verbose_update) {
 		for (total = cnt = 0; cnt < index->cache_nr; cnt++) {
 			struct cache_entry *ce = index->cache[cnt];
-			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE))
+			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE | CE_WT_REMOVE))
 				total++;
 		}
 
@@ -92,6 +98,13 @@ static int check_updates(struct unpack_trees_options *o)
 	for (i = 0; i < index->cache_nr; i++) {
 		struct cache_entry *ce = index->cache[i];
 
+		if (ce->ce_flags & CE_WT_REMOVE) {
+			display_progress(progress, ++cnt);
+			if (o->update)
+				unlink_entry(ce);
+			continue;
+		}
+
 		if (ce->ce_flags & CE_REMOVE) {
 			display_progress(progress, ++cnt);
 			if (o->update)
@@ -118,6 +131,74 @@ static int check_updates(struct unpack_trees_options *o)
 	return errs != 0;
 }
 
+static int verify_uptodate_sparse(struct cache_entry *ce, struct unpack_trees_options *o);
+static int verify_absent_sparse(struct cache_entry *ce, const char *action, struct unpack_trees_options *o);
+static int apply_sparse_checkout(struct unpack_trees_options *o)
+{
+	struct index_state *index = &o->result;
+	struct exclude_list el;
+	int i, ret = 0;
+
+	memset(&el, 0, sizeof(el));
+	if (add_excludes_from_file_to_list(git_path("info/sparse"), "", 0, NULL, &el, 0) < 0)
+		return 0;
+
+	for (i = 0; i < index->cache_nr; i++) {
+		struct cache_entry *ce = index->cache[i];
+		const char *basename;
+		int was_valid = ce->ce_flags & CE_VALID;
+
+		if (ce_stage(ce))
+			continue;
+
+		basename = strrchr(ce->name, '/');
+		basename = basename ? basename+1 : ce->name;
+		if (excluded_from_list(ce->name, ce_namelen(ce), basename, NULL, &el) > 0)
+			ce->ce_flags |= CE_VALID;
+		else
+			ce->ce_flags &= ~CE_VALID;
+
+		/*
+		 * We only care about files getting into the checkout area
+		 * If merge strategies want to remove some, go ahead
+		 */
+		if (ce->ce_flags & CE_REMOVE)
+			continue;
+
+		if (!was_valid && (ce->ce_flags & CE_VALID)) {
+			/*
+			 * If CE_UPDATE is set, verify_uptodate() must be called already
+			 * also stat info may have lost after merged_entry() so calling
+			 * verify_uptodate() again may fail
+			 */
+			if (!(ce->ce_flags & CE_UPDATE) && verify_uptodate_sparse(ce, o)) {
+				ret = -1;
+				break;
+			}
+			ce->ce_flags |= CE_WT_REMOVE;
+		}
+		if (was_valid && !(ce->ce_flags & CE_VALID)) {
+			if (verify_absent_sparse(ce, "overwritten", o)) {
+				ret = -1;
+				break;
+			}
+			ce->ce_flags |= CE_UPDATE;
+		}
+
+		/* merge strategies may set CE_UPDATE outside checkout area */
+		if (ce->ce_flags & CE_VALID)
+			ce->ce_flags &= ~CE_UPDATE;
+
+	}
+
+	for (i = 0;i < el.nr;i++)
+		free(el.excludes[i]);
+	if (el.excludes)
+		free(el.excludes);
+
+	return ret;
+}
+
 static inline int call_unpack_fn(struct cache_entry **src, struct unpack_trees_options *o)
 {
 	int ret = o->fn(src, o);
@@ -416,6 +497,9 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options
 	if (o->trivial_merges_only && o->nontrivial_merge)
 		return unpack_failed(o, "Merge requires file-level merging");
 
+	if (o->apply_sparse && apply_sparse_checkout(o))
+		return unpack_failed(o, NULL);
+
 	o->src_index = NULL;
 	ret = check_updates(o) ? (-2) : 0;
 	if (o->dst_index)
@@ -481,6 +565,12 @@ static int verify_uptodate(struct cache_entry *ce,
 	return verify_uptodate_1(ce, o, ERRORMSG(o, not_uptodate_file));
 }
 
+static int verify_uptodate_sparse(struct cache_entry *ce,
+				  struct unpack_trees_options *o)
+{
+	return verify_uptodate_1(ce, o, ERRORMSG(o, sparse_not_uptodate_file));
+}
+
 static void invalidate_ce_path(struct cache_entry *ce, struct unpack_trees_options *o)
 {
 	if (ce)
@@ -674,6 +764,12 @@ static int verify_absent(struct cache_entry *ce, const char *action,
 	return verify_absent_1(ce, action, o, ERRORMSG(o, would_lose_untracked));
 }
 
+static int verify_absent_sparse(struct cache_entry *ce, const char *action,
+			 struct unpack_trees_options *o)
+{
+	return verify_absent_1(ce, action, o, ERRORMSG(o, would_lose_orphaned));
+}
+
 static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
 		struct unpack_trees_options *o)
 {
diff --git a/unpack-trees.h b/unpack-trees.h
index d19df44..a09077b 100644
--- a/unpack-trees.h
+++ b/unpack-trees.h
@@ -14,6 +14,8 @@ struct unpack_trees_error_msgs {
 	const char *not_uptodate_dir;
 	const char *would_lose_untracked;
 	const char *bind_overlap;
+	const char *sparse_not_uptodate_file;
+	const char *would_lose_orphaned;
 };
 
 struct unpack_trees_options {
@@ -28,6 +30,7 @@ struct unpack_trees_options {
 		     skip_unmerged,
 		     initial_checkout,
 		     diff_index_cached,
+		     apply_sparse,
 		     gently;
 	const char *prefix;
 	int pos;
-- 
1.6.3.GIT