[PATCH 02/12] git-grep: support --no-external-grep

[PATCH 02/12] git-grep: support --no-external-grep

[Nguyễn Thái Ngọc Duy]

this one is mainly for testing builtin grep

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 builtin-grep.c |    8 +++++++-
 1 files changed, 7 insertions(+), 1 deletions(-)

diff --git a/builtin-grep.c b/builtin-grep.c
index cc9ba6b..ee96d01 100644
--- a/builtin-grep.c
+++ b/builtin-grep.c
@@ -20,6 +20,8 @@
 #endif
 #endif
 
+static int no_external_grep;
+
 /*
  * git grep pathspecs are somewhat different from diff-tree pathspecs;
  * pathname wildcards are allowed.
@@ -386,7 +388,7 @@ static int grep_cache(struct grep_opt *opt, const char **paths, int cached)
 	 * we grep through the checked-out files. It tends to
 	 * be a lot more optimized
 	 */
-	if (!cached) {
+	if (!cached && !no_external_grep) {
 		hit = external_grep(opt, paths, cached);
 		if (hit >= 0)
 			return hit;
@@ -544,6 +546,10 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 			cached = 1;
 			continue;
 		}
+		if (!strcmp("--no-external-grep", arg)) {
+			no_external_grep = 1;
+			continue;
+		}
 		if (!strcmp("-a", arg) ||
 		    !strcmp("--text", arg)) {
 			opt.binary = GREP_BINARY_TEXT;
-- 
1.5.5.GIT

[PATCH 03/12] Introduce sparse prefix

[Nguyễn Thái Ngọc Duy]

Sparse prefix is actually a list of prefixes separated by colons,
that will limit worktree usage within it.

This patch adds core.sparsecheckout and
"git rev-parse --show-sparse-prefix". This also adds manipulation
functions that will get used later.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 builtin-rev-parse.c |    4 +
 cache.h             |   13 +++
 config.c            |    7 ++
 environment.c       |  221 +++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 245 insertions(+), 0 deletions(-)

diff --git a/builtin-rev-parse.c b/builtin-rev-parse.c
index aa71f4a..4200f14 100644
--- a/builtin-rev-parse.c
+++ b/builtin-rev-parse.c
@@ -499,6 +499,10 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix)
 					puts(prefix);
 				continue;
 			}
+			if (!strcmp(arg, "--show-sparse-prefix")) {
+				puts(get_sparse_prefix_str());
+				continue;
+			}
 			if (!strcmp(arg, "--show-cdup")) {
 				const char *pfx = prefix;
 				if (!is_inside_work_tree()) {
diff --git a/cache.h b/cache.h
index 38985aa..4687096 100644
--- a/cache.h
+++ b/cache.h
@@ -319,6 +319,19 @@ extern const char *get_git_work_tree(void);
 extern const char *read_gitfile_gently(const char *path);
 extern void set_git_work_tree(const char *tree);
 
+extern int sparse_checkout_enabled();
+extern char **get_sparse_prefix(void);
+extern const char *get_sparse_prefix_str(void);
+extern char **save_sparse_prefix();
+extern char **restore_sparse_prefix(char **prefix);
+extern int outside_sparse_prefix(const char *prefix);
+extern int index_outside_sparse_prefix(const char *prefix);
+extern void set_sparse_prefix(const char *prefix);
+extern char **split_prefix(const char *env);
+extern char **combine_prefix_list(char **p1, char **p2);
+extern void free_prefix_list(char **prefix_list);
+extern int outside_prefix_list(char **iprefix, const char *prefix);
+
 #define ALTERNATE_DB_ENVIRONMENT "GIT_ALTERNATE_OBJECT_DIRECTORIES"
 
 extern const char **get_pathspec(const char *prefix, const char **pathspec);
diff --git a/config.c b/config.c
index 1e066c7..e7b457b 100644
--- a/config.c
+++ b/config.c
@@ -467,6 +467,13 @@ static int git_default_core_config(const char *var, const char *value)
 		return 0;
 	}
 
+	if (!strcmp(var, "core.sparsecheckout")) {
+		if (!value)
+			return config_error_nonbool(var);
+		set_sparse_prefix(value);
+		return 0;
+	}
+
 	/* Add other config variables here and to Documentation/config.txt. */
 	return 0;
 }
diff --git a/environment.c b/environment.c
index 4a88a17..94e39b8 100644
--- a/environment.c
+++ b/environment.c
@@ -46,9 +46,132 @@ enum rebase_setup_type autorebase = AUTOREBASE_NEVER;
 char *git_work_tree_cfg;
 static char *work_tree;
 
+static char **sparse_prefix;
+
 static const char *git_dir;
 static char *git_object_dir, *git_index_file, *git_refs_dir, *git_graft_file;
 
+static char *append_slash(const char *prefix, int len)
+{
+	char *new_prefix;
+
+	if (!*prefix)
+		return NULL;
+
+	if (prefix[len-1] == '/')
+		return xstrndup(prefix, len);
+
+	new_prefix = xmalloc(len+2);
+	memcpy(new_prefix, prefix, len);
+	new_prefix[len] = '/';
+	new_prefix[len+1] = '\0';
+	return new_prefix;
+}
+
+/* this should be sorted as same order as index */
+static int prefix_compare(const void *prefix1_, const void *prefix2_)
+{
+	const char *prefix1 = *(const char**)prefix1_;
+	const char *prefix2 = *(const char**)prefix2_;
+	int len1 = strlen(prefix1);
+	int len2 = strlen(prefix2);
+	int len = len1 < len2 ? len1 : len2;
+	int cmp = memcmp(prefix1, prefix2, len);
+	if (cmp)
+		return cmp;
+	if (len1 < len2)
+		return -1;
+	if (len1 > len2)
+		return 1;
+	return 0;
+}
+
+void free_prefix_list(char **prefix_list)
+{
+	char **prefix = prefix_list;
+
+	if (!prefix)
+		return;
+
+	while (*prefix) {
+		free(*prefix);
+		prefix++;
+	}
+	free(prefix_list);
+}
+
+char **split_prefix(const char *env)
+{
+	int prefix_nr = 0;
+	int prefix_alloc = 0;
+	char **prefix_list = NULL;
+
+	if (!env)
+		return NULL;
+
+	do {
+		const char *sep = strchr(env, ':');
+		int len = sep ? sep - env : strlen(env);
+		if (prefix_alloc <= prefix_nr+1) {
+			prefix_alloc = alloc_nr(prefix_alloc);
+			prefix_list = xrealloc(prefix_list,
+						prefix_alloc * sizeof(*prefix_list));
+		}
+		prefix_list[prefix_nr++] = append_slash(env, len);
+		env += sep ? len+1 : len;
+	} while (*env);
+	prefix_list[prefix_nr] = NULL;
+	qsort(prefix_list, prefix_nr, sizeof(*prefix_list), prefix_compare);
+	return prefix_list;
+}
+
+char **combine_prefix_list(char **p1, char **p2)
+{
+	int len1 = 0, len2 = 0;
+	char **p, **p12, *last_prefix;
+	char **result, **result_p;
+
+	/*
+	 * if either of them is null, full access,
+	 * combining them would give full access as well
+	 */
+	if (!p1 || !p2)
+		return NULL;
+
+	for (p = p1; *p; p++)
+		len1++;
+	for (p = p2; *p; p++)
+		len2++;
+
+	p12 = xmalloc((len1+len2+1)*sizeof(*p12));
+	result = xmalloc((len1+len2+1)*sizeof(*result));
+	memcpy(p12, p1, len1*sizeof(*p12));
+	memcpy(p12+len1, p2, (len2+1)*sizeof(*p12));
+	qsort(p12, len1+len2, sizeof(*p12), prefix_compare);
+
+	p = p12;
+	last_prefix = *p;
+	p++;
+	result_p = result;
+	*result_p = xstrdup(last_prefix);
+	result_p++;
+	while (*p) {
+		if (!strcmp(*p, last_prefix)) {
+			p++;
+			continue;
+		}
+		if (!prefixcmp(*p, last_prefix)) {
+			p++;
+			continue;
+		}
+		*result_p = xstrdup(*p);
+		result_p++;
+		p++;
+	}
+	*result_p = NULL;
+	return result;
+}
+
 static void setup_git_env(void)
 {
 	git_dir = getenv(GIT_DIR_ENVIRONMENT);
@@ -122,6 +245,104 @@ const char *get_git_work_tree(void)
 	return work_tree;
 }
 
+void set_sparse_prefix(const char *flat_sparse_prefix)
+{
+	free_prefix_list(sparse_prefix);
+	sparse_prefix = split_prefix(flat_sparse_prefix);
+}
+
+char **get_sparse_prefix()
+{
+	return sparse_prefix;
+}
+
+const char *get_sparse_prefix_str()
+{
+	static char **sparse_prefix = NULL;
+	static char *sparse_prefix_str = NULL;
+	int len;
+	char **prefix;
+
+	if (sparse_prefix == get_sparse_prefix())
+		return sparse_prefix ? sparse_prefix_str : "";
+
+	sparse_prefix = get_sparse_prefix();
+
+	if (!sparse_prefix)
+		return "";
+
+	len = 0;
+	for (prefix = sparse_prefix; *prefix; prefix++)
+		len += strlen(*prefix)+1;
+	if (sparse_prefix_str)
+		free(sparse_prefix_str);
+	sparse_prefix_str = xmalloc(len);
+
+	prefix = sparse_prefix;
+	len = strlen(*prefix);
+	if ((*prefix)[len-1] == '/')
+		len--;
+	memcpy(sparse_prefix_str, *prefix, len);
+	prefix++;
+	while (*prefix) {
+		int len2 = strlen(*prefix);
+		sparse_prefix_str[len++] = ':';
+		if ((*prefix)[len2-1] == '/')
+			len2--;
+		memcpy(sparse_prefix_str+len, *prefix, len2);
+		len += len2;
+		prefix++;
+	}
+	sparse_prefix_str[len] = 0;
+	return sparse_prefix_str;
+}
+
+char **save_sparse_prefix()
+{
+	char **prefix = sparse_prefix;
+	sparse_prefix = NULL;
+	return prefix;
+}
+
+char **restore_sparse_prefix(char **prefix)
+{
+	char **old_prefix = sparse_prefix;
+	sparse_prefix = prefix;
+	return old_prefix;
+}
+
+int outside_prefix_list(char **iprefix, const char *prefix)
+{
+	if (!iprefix)
+		return 0;
+
+	while (*iprefix) {
+		if (!prefixcmp(prefix, *iprefix))
+			return 0;
+		iprefix++;
+	}
+	return 1;
+}
+
+int sparse_checkout_enabled()
+{
+	static int disable_sparse_checkout = -1;
+	if (disable_sparse_checkout == -1)
+		disable_sparse_checkout = getenv("GIT_SPARSE_CHECKOUT_INDEX_ONLY") != NULL;
+	return !disable_sparse_checkout && get_sparse_prefix();
+}
+
+
+int outside_sparse_prefix(const char *prefix)
+{
+	return sparse_checkout_enabled() && outside_prefix_list(sparse_prefix, prefix);
+}
+
+int index_outside_sparse_prefix(const char *prefix)
+{
+	return outside_prefix_list(sparse_prefix, prefix);
+}
+
 char *get_object_directory(void)
 {
 	if (!git_object_dir)
-- 
1.5.5.GIT

[PATCH 08/12] git-clone: support --path to do sparse clone

[Nguyễn Thái Ngọc Duy]

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 builtin-clone.c |   13 +++++++++++++
 1 files changed, 13 insertions(+), 0 deletions(-)

diff --git a/builtin-clone.c b/builtin-clone.c
index 3522245..229f2e2 100644
--- a/builtin-clone.c
+++ b/builtin-clone.c
@@ -36,6 +36,7 @@ static const char * const builtin_clone_usage[] = {
 static int option_quiet, option_no_checkout, option_bare;
 static int option_local, option_no_hardlinks, option_shared;
 static char *option_template, *option_reference, *option_depth;
+static char *option_sparse_prefix;
 static char *option_origin = NULL;
 static char *option_upload_pack = "git-upload-pack";
 
@@ -43,6 +44,8 @@ static struct option builtin_clone_options[] = {
 	OPT__QUIET(&option_quiet),
 	OPT_BOOLEAN('n', "no-checkout", &option_no_checkout,
 		    "don't create a checkout"),
+	OPT_STRING(0, "path", &option_sparse_prefix, "prefixes",
+		    "limit checkout to specified paths (sparse checkout)"),
 	OPT_BOOLEAN(0, "bare", &option_bare, "create a bare repository"),
 	OPT_BOOLEAN(0, "naked", &option_bare, "create a bare repository"),
 	OPT_BOOLEAN('l', "local", &option_local,
@@ -364,9 +367,14 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 			die("--bare and --origin %s options are incompatible.",
 			    option_origin);
 		option_no_checkout = 1;
+		if (option_sparse_prefix)
+			die("--bare and --path options are incompatible.");
 		use_separate_remote = 0;
 	}
 
+	if (option_no_checkout && option_sparse_prefix)
+		die("--no-checkout and --path options are incompatible.");
+
 	if (!option_origin)
 		option_origin = "origin";
 
@@ -549,6 +557,11 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 		/* We need to be in the new work tree for the checkout */
 		setup_work_tree();
 
+		if (option_sparse_prefix) {
+			git_config_set("core.sparsecheckout", option_sparse_prefix);
+			set_sparse_prefix(option_sparse_prefix);
+		}
+
 		fd = hold_locked_index(lock_file, 1);
 
 		memset(&opts, 0, sizeof opts);
-- 
1.5.5.GIT

[PATCH 10/12] git-checkout: support --full and --path to manipulate sparse checkout

[Nguyễn Thái Ngọc Duy]

"git checkout --full" will quit sparse checkout mode while
"git checkout --sparse=prefix" will turn on sparse checkout or
update sparse prefix.

twoway_merge has been updated to deal with sparse checkout update.
Files that are in old sparse prefix only will get removed while
files that are in new sparse prefix only will get added.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 builtin-checkout.c |   44 +++++++++++++++++++
 cache.h            |    2 +
 unpack-trees.c     |  120 ++++++++++++++++++++++++++++++++++++++++++++++++----
 unpack-trees.h     |    3 +-
 4 files changed, 160 insertions(+), 9 deletions(-)

diff --git a/builtin-checkout.c b/builtin-checkout.c
index eec1bde..410a53a 100644
--- a/builtin-checkout.c
+++ b/builtin-checkout.c
@@ -160,6 +160,9 @@ struct checkout_opts {
 	char *new_branch;
 	int new_branch_log;
 	enum branch_track track;
+
+	char *sparse_prefix;
+	int no_sparse_checkout;
 };
 
 static int reset_tree(struct tree *tree, struct checkout_opts *o, int worktree)
@@ -255,7 +258,25 @@ static int merge_working_tree(struct checkout_opts *opts,
 		tree = parse_tree_indirect(new->commit->object.sha1);
 		init_tree_desc(&trees[1], tree->buffer, tree->size);
 
+		if (opts->no_sparse_checkout)
+			topts.new_sparse_prefix = 1;
+
+		if (opts->sparse_prefix) {
+			char **new_sparse_prefix = split_prefix(opts->sparse_prefix);
+
+			if (!new_sparse_prefix) {
+				error("new sparse prefix invalid");
+				return 1;
+			}
+			topts.new_sparse_prefix = 1;
+			topts.unpack_data = new_sparse_prefix;
+		}
+
 		ret = unpack_trees(2, trees, &topts);
+
+		if (topts.new_sparse_prefix && topts.unpack_data)
+			free_prefix_list((char **)topts.unpack_data);
+
 		if (ret == -1) {
 			/*
 			 * Unpack couldn't do a trivial merge; either
@@ -299,6 +320,14 @@ static int merge_working_tree(struct checkout_opts *opts,
 	    commit_locked_index(lock_file))
 		die("unable to write new index file");
 
+	/*
+	 * update sparse checkout accordingly
+	 * show_local_changes will need it
+	 */
+	if (opts->no_sparse_checkout)
+		free_prefix_list(save_sparse_prefix());
+	if (opts->sparse_prefix)
+		set_sparse_prefix(opts->sparse_prefix);
 	if (!opts->force)
 		show_local_changes(&new->commit->object);
 
@@ -409,6 +438,13 @@ static int switch_branches(struct checkout_opts *opts, struct branch_info *new)
 
 	update_refs_for_switch(opts, &old, new);
 
+	/* now save new sparse prefix */
+	if (opts->no_sparse_checkout)
+		git_config_set("core.sparsecheckout", NULL);
+
+	if (opts->sparse_prefix)
+		git_config_set("core.sparsecheckout", opts->sparse_prefix);
+
 	ret = post_checkout_hook(old.commit, new->commit, 1);
 	return ret || opts->writeout_error;
 }
@@ -428,6 +464,8 @@ int cmd_checkout(int argc, const char **argv, const char *prefix)
 			BRANCH_TRACK_EXPLICIT),
 		OPT_BOOLEAN('f', NULL, &opts.force, "force"),
 		OPT_BOOLEAN('m', NULL, &opts.merge, "merge"),
+		OPT_STRING(0, "path", &opts.sparse_prefix, "prefixes", "update sparse checkout"),
+		OPT_BOOLEAN(0, "full", &opts.no_sparse_checkout, "quit sparse checkout"),
 		OPT_END(),
 	};
 
@@ -468,6 +506,9 @@ int cmd_checkout(int argc, const char **argv, const char *prefix)
 	if (!opts.new_branch && (opts.track != git_branch_track))
 		die("git checkout: --track and --no-track require -b");
 
+	if (opts.no_sparse_checkout && opts.sparse_prefix)
+		die("git checkout: --path and --full are incompatible");
+
 	if (opts.force && opts.merge)
 		die("git checkout: -f and -m are incompatible");
 
@@ -486,6 +527,9 @@ int cmd_checkout(int argc, const char **argv, const char *prefix)
 			}
 		}
 
+		if (opts.no_sparse_checkout || opts.sparse_prefix)
+			die("git checkout: updating paths is incompatible with setting sparse checkout");
+
 		return checkout_paths(source_tree, pathspec);
 	}
 
diff --git a/cache.h b/cache.h
index b9a1d96..9e7f146 100644
--- a/cache.h
+++ b/cache.h
@@ -138,6 +138,8 @@ struct cache_entry {
 #define CE_HASHED    (0x100000)
 #define CE_UNHASHED  (0x200000)
 
+#define CE_WD_REMOVE (0x400000)
+
 /*
  * Copy the sha1 and stat state of a cache entry from one to
  * another. But we never change the name, or the hash state!
diff --git a/unpack-trees.c b/unpack-trees.c
index 0a30d68..a88c53f 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -105,13 +105,29 @@ static int check_updates(struct unpack_trees_options *o)
 	struct index_state *index = &o->result;
 	int i;
 	int errs = 0;
+	char **old_sparse_prefix = NULL;
+	char **new_sparse_prefix = NULL;
+
+	/*
+	 * for sparse checkout update, we need to widen the prefix a bit
+	 * so that updating in new sparse prefix won't get caught by
+	 * sparse prefix checks
+	 */
+	if (o->new_sparse_prefix) {
+		old_sparse_prefix = save_sparse_prefix();
+		if (o->unpack_data) {
+			new_sparse_prefix = o->unpack_data;
+			new_sparse_prefix = combine_prefix_list(old_sparse_prefix, new_sparse_prefix);
+			restore_sparse_prefix(new_sparse_prefix);
+		}
+	}
 
 	if (o->update && o->verbose_update) {
 		for (total = cnt = 0; cnt < index->cache_nr; cnt++) {
 			struct cache_entry *ce = index->cache[cnt];
 			if (outside_sparse_prefix(ce->name))
 				continue;
-			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE))
+			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE | CE_WD_REMOVE))
 				total++;
 		}
 
@@ -125,12 +141,14 @@ static int check_updates(struct unpack_trees_options *o)
 
 		if (outside_sparse_prefix(ce->name))
 			continue;
-		if (ce->ce_flags & CE_REMOVE) {
+		if (ce->ce_flags & (CE_REMOVE | CE_WD_REMOVE)) {
 			display_progress(progress, ++cnt);
 			if (o->update)
 				unlink_entry(ce);
-			remove_index_entry_at(&o->result, i);
-			i--;
+			if (ce->ce_flags & CE_REMOVE) {
+				remove_index_entry_at(&o->result, i);
+				i--;
+			}
 			continue;
 		}
 	}
@@ -149,6 +167,10 @@ static int check_updates(struct unpack_trees_options *o)
 		}
 	}
 	stop_progress(&progress);
+	if (o->new_sparse_prefix) {
+		restore_sparse_prefix(old_sparse_prefix);
+		free_prefix_list(new_sparse_prefix);
+	}
 	return errs != 0;
 }
 
@@ -680,8 +702,8 @@ static int verify_absent(struct cache_entry *ce, const char *action,
 	return 0;
 }
 
-static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
-		struct unpack_trees_options *o)
+static int merged_entry_internal(struct cache_entry *merge, struct cache_entry *old,
+		struct unpack_trees_options *o, int set, int clear)
 {
 	int update = CE_UPDATE;
 
@@ -708,10 +730,28 @@ static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
 		invalidate_ce_path(merge, o);
 	}
 
-	add_entry(o, merge, update, CE_STAGEMASK);
+	add_entry(o, merge, (update & (~clear)) | set, CE_STAGEMASK);
 	return 1;
 }
 
+static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
+		struct unpack_trees_options *o)
+{
+	return merged_entry_internal(merge, old, o, 0, 0);
+}
+
+static int merged_entry_and_no_add(struct cache_entry *merge, struct cache_entry *old,
+		struct unpack_trees_options *o)
+{
+	return merged_entry_internal(merge, old, o, 0, CE_UPDATE);
+}
+
+static int merged_entry_and_add(struct cache_entry *merge, struct cache_entry *old,
+		struct unpack_trees_options *o)
+{
+	return merged_entry_internal(merge, old, o, CE_UPDATE, 0);
+}
+
 static int deleted_entry(struct cache_entry *ce, struct cache_entry *old,
 		struct unpack_trees_options *o)
 {
@@ -734,6 +774,20 @@ static int keep_entry(struct cache_entry *ce, struct unpack_trees_options *o)
 	return 1;
 }
 
+static int keep_entry_and_add(struct cache_entry *ce, struct unpack_trees_options *o)
+{
+	add_entry(o, ce, CE_UPDATE, 0);
+	return 1;
+}
+
+static int keep_entry_and_remove(struct cache_entry *ce, struct unpack_trees_options *o)
+{
+	if (verify_uptodate(ce, o))
+		return -1;
+	add_entry(o, ce, CE_WD_REMOVE, 0);
+	return 1;
+}
+
 #if DBRT_DEBUG
 static void show_stage_entry(FILE *o,
 			     const char *label, const struct cache_entry *ce)
@@ -918,12 +972,31 @@ int threeway_merge(struct cache_entry **stages, struct unpack_trees_options *o)
  * over a merge failure when it makes sense.  For details of the
  * "carry forward" rule, please see <Documentation/git-read-tree.txt>.
  *
+ * Two-way merge with sparse prefix update.
+ *
+ * The rules are as same as plain two-way merge, except:
+ *  1) if it's in old sparse checkout, but not the new one, mark it CE_WD_REMOVE
+ *  2) if it's in the new one, but not the old one, mark it CE_UPDATE
+ *  3) otherwise, let old two-way merge take it
+ *
+ *  case 1)
+ *     keep index implies CE_WD_REMOVE (remove workdir only, keep file in index)
+ *     use M implies no CE_UPDATE
+ *     if index is not clean, fail
+ *
+ *  case 2)
+ *     keep index implies CE_UPDATE
+ *     use M implies CE_UPDATE
+ *     index can't be unclean, so ignore this
+ *
  */
 int twoway_merge(struct cache_entry **src, struct unpack_trees_options *o)
 {
 	struct cache_entry *current = src[0];
 	struct cache_entry *oldtree = src[1];
 	struct cache_entry *newtree = src[2];
+	int remove = 0, add = 0;
+	char **new_sparse_prefix = (char **)o->unpack_data;
 
 	if (o->merge_size != 2)
 		return error("Cannot do a twoway merge of %d trees",
@@ -935,6 +1008,13 @@ int twoway_merge(struct cache_entry **src, struct unpack_trees_options *o)
 		newtree = NULL;
 
 	if (current) {
+		if (o->new_sparse_prefix) {
+			remove = !outside_sparse_prefix(current->name) &&
+				 outside_prefix_list(new_sparse_prefix, current->name);
+			add = outside_sparse_prefix(current->name) &&
+			      !outside_prefix_list(new_sparse_prefix, current->name);
+		}
+
 		if ((!oldtree && !newtree) || /* 4 and 5 */
 		    (!oldtree && newtree &&
 		     same(current, newtree)) || /* 6 and 7 */
@@ -943,6 +1023,12 @@ int twoway_merge(struct cache_entry **src, struct unpack_trees_options *o)
 		    (oldtree && newtree &&
 		     !same(oldtree, newtree) && /* 18 and 19 */
 		     same(current, newtree))) {
+			if (o->new_sparse_prefix) {
+				if (remove)
+					return keep_entry_and_remove(current, o);
+				if (add)
+					return keep_entry_and_add(current, o);
+			}
 			return keep_entry(current, o);
 		}
 		else if (oldtree && !newtree && same(current, oldtree)) {
@@ -952,6 +1038,12 @@ int twoway_merge(struct cache_entry **src, struct unpack_trees_options *o)
 		else if (oldtree && newtree &&
 			 same(current, oldtree) && !same(current, newtree)) {
 			/* 20 or 21 */
+			if (o->new_sparse_prefix) {
+				if (remove)
+					return merged_entry_and_no_add(newtree, current, o);
+				if (add)
+					return merged_entry_and_add(newtree, current, o);
+			}
 			return merged_entry(newtree, current, o);
 		}
 		else {
@@ -965,8 +1057,20 @@ int twoway_merge(struct cache_entry **src, struct unpack_trees_options *o)
 			return -1;
 		}
 	}
-	else if (newtree)
+	else if (newtree) {
+		if (o->new_sparse_prefix) {
+			remove = !outside_sparse_prefix(newtree->name) &&
+				outside_prefix_list(new_sparse_prefix, newtree->name);
+			add = outside_sparse_prefix(newtree->name) &&
+				!outside_prefix_list(new_sparse_prefix, newtree->name);
+
+			if (remove)
+				return merged_entry_and_no_add(newtree, current, o);
+			if (add)
+				return merged_entry_and_add(newtree, current, o);
+		}
 		return merged_entry(newtree, current, o);
+	}
 	return deleted_entry(oldtree, current, o);
 }
 
diff --git a/unpack-trees.h b/unpack-trees.h
index a1b46f9..5a29fc4 100644
--- a/unpack-trees.h
+++ b/unpack-trees.h
@@ -27,7 +27,8 @@ struct unpack_trees_options {
 		     aggressive:1,
 		     skip_unmerged:1,
 		     gently:1,
-		     check_index_prefix:1;
+		     check_index_prefix:1,
+		     new_sparse_prefix:1; /* unpack_data must contain new prefix */
 	const char *prefix;
 	int pos;
 	struct dir_struct *dir;
-- 
1.5.5.GIT

Re: [PATCH 02/12] git-grep: support --no-external-grep

[Petr Baudis]

  Hi,

  the patches 2, 3, 8 and 10 of your series introduce config options,
commandline switches and environment variables that are not documented
anywhere - can you please include documentation updates in your patch
series as well? It might be worth it to find some suitable place in the
documentation where to explain the concept itself; it took me quite
sometime before I actually realized (I think) what are your patches
about. (Which is also why is it useful to write cover letters for
patch series.)

-- 
				Petr "Pasky" Baudis
As in certain cults it is possible to kill a process if you know
its true name.  -- Ken Thompson and Dennis M. Ritchie

Re: [PATCH 02/12] git-grep: support --no-external-grep

[Petr Baudis]

  Hi!

On Wed, Jul 23, 2008 at 09:01:30PM +0200, Petr Baudis wrote:
> (Which is also why is it useful to write cover letters for
> patch series.)

  sorry, I did not notice the cover letter, already separated from the
rest by a sprouting thread. Which is also why it is useful to thread up
all the patches. ;-)

				Petr "Pasky" Baudis

Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Jeff King]

On Wed, Jul 23, 2008 at 09:57:18PM +0700, Nguyễn Thái Ngọc Duy wrote:

> +		if (option_sparse_prefix) {
> +			git_config_set("core.sparsecheckout", option_sparse_prefix);
> +			set_sparse_prefix(option_sparse_prefix);
> +		}
> +

As a user, I would expect "sparse clone" to also be sparse on the
fetching. That is, to not even bother fetching tree objects that we are
not going to check out. But that is a whole other can of worms from
local sparseness, so I think it is worth saving for a different series.

So instead I would suggest that this be mentioned in the documentation
for --path, but there doesn't seem to be any.

-Peff

sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Johannes Schindelin]

Hi,

On Thu, 24 Jul 2008, Jeff King wrote:

> As a user, I would expect "sparse clone" to also be sparse on the 
> fetching. That is, to not even bother fetching tree objects that we are 
> not going to check out. But that is a whole other can of worms from 
> local sparseness, so I think it is worth saving for a different series.

I think this is not even worth of a series.  Sure, it would have benefits 
for those who want sparse checkouts.  But it comes for a high price on 
everyone else:

- security issues (you'd need to open the git protocol to give you 
  something else than a ref, _including_ refs that were deleted)

- performance issues (the server would have to do a lot more, faking 
  commits, or in the alternative serving a gazillion more sessions if the 
  client does the reconstruction)

... and I am sure there are tons more issues.

Ciao,
Dscho

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Jeff King]

On Thu, Jul 24, 2008 at 06:41:03PM +0100, Johannes Schindelin wrote:

> > As a user, I would expect "sparse clone" to also be sparse on the 
> > fetching. That is, to not even bother fetching tree objects that we are 
> > not going to check out. But that is a whole other can of worms from 
> > local sparseness, so I think it is worth saving for a different series.
> 
> I think this is not even worth of a series.  Sure, it would have benefits 
> for those who want sparse checkouts.  But it comes for a high price on 
> everyone else:

I agree there are a lot of issues. I am just thinking of the person who
said they had a >100G repository. But I am also not volunteering to do
it, so I will let somebody who really cares about it try to defend the
idea.

-Peff

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Nguyen Thai Ngoc Duy]

On 7/25/08, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> Hi,
>
>  On Thu, 24 Jul 2008, Jeff King wrote:
>
>  > As a user, I would expect "sparse clone" to also be sparse on the
>  > fetching. That is, to not even bother fetching tree objects that we are
>  > not going to check out. But that is a whole other can of worms from
>  > local sparseness, so I think it is worth saving for a different series.
>
>  I think this is not even worth of a series.  Sure, it would have benefits
>  for those who want sparse checkouts.  But it comes for a high price on
>  everyone else:
>
>  - security issues (you'd need to open the git protocol to give you
>   something else than a ref, _including_ refs that were deleted)
>
>  - performance issues (the server would have to do a lot more, faking
>   commits, or in the alternative serving a gazillion more sessions if the
>   client does the reconstruction)
>
>  ... and I am sure there are tons more issues.

Widen checkout won't work and probably more.
-- 
Duy

Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Nguyen Thai Ngoc Duy]

On 7/25/08, Jeff King <peff@peff.net> wrote:
> On Wed, Jul 23, 2008 at 09:57:18PM +0700, Nguyen Thai Ngoc Duy wrote:
>
>  > +             if (option_sparse_prefix) {
>  > +                     git_config_set("core.sparsecheckout", option_sparse_prefix);
>  > +                     set_sparse_prefix(option_sparse_prefix);
>  > +             }
>  > +
>
>
> As a user, I would expect "sparse clone" to also be sparse on the
>  fetching. That is, to not even bother fetching tree objects that we are
>  not going to check out. But that is a whole other can of worms from
>  local sparseness, so I think it is worth saving for a different series.
>
>  So instead I would suggest that this be mentioned in the documentation
>  for --path, but there doesn't seem to be any.

Thanks. Will mention it when I write documentation for this.
-- 
Duy

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Petr Baudis]

  Hi,

On Thu, Jul 24, 2008 at 06:41:03PM +0100, Johannes Schindelin wrote:
> On Thu, 24 Jul 2008, Jeff King wrote:
> 
> > As a user, I would expect "sparse clone" to also be sparse on the 
> > fetching. That is, to not even bother fetching tree objects that we are 
> > not going to check out. But that is a whole other can of worms from 
> > local sparseness, so I think it is worth saving for a different series.
> 
> I think this is not even worth of a series.  Sure, it would have benefits 
> for those who want sparse checkouts.  But it comes for a high price on 
> everyone else:
> 
> - security issues (you'd need to open the git protocol to give you 
>   something else than a ref, _including_ refs that were deleted)
> 
> - performance issues (the server would have to do a lot more, faking 
>   commits, or in the alternative serving a gazillion more sessions if the 
>   client does the reconstruction)

  I don't follow how these two issues arise, if the server will do the
pruning for you. It will just skip entering some tree objects when doing
object traversal; why opening the git protocol or faking commits? This
would be a simple extra capability in the protocol.

  One question is what to do with delta chains including unwanted
objects, but I think that given the objects' associativity for delta
chains, this shouldn't be huge practical issues and it could be
affordable in principle to include even unwanted objects.

> ... and I am sure there are tons more issues.

  I do agree on this. :-)

-- 
				Petr "Pasky" Baudis
As in certain cults it is possible to kill a process if you know
its true name.  -- Ken Thompson and Dennis M. Ritchie

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Sverre Rabbelier]

On Thu, Jul 24, 2008 at 8:53 PM, Petr Baudis <pasky@suse.cz> wrote:
>  I don't follow how these two issues arise, if the server will do the
> pruning for you. It will just skip entering some tree objects when doing
> object traversal; why opening the git protocol or faking commits? This
> would be a simple extra capability in the protocol.

Wouldn't that be as simple as passing a pathspec to git-rev-list? Not
a lot of overhead there I reckon.

> One question is what to do with delta chains including unwanted
> objects, but I think that given the objects' associativity for delta
> chains, this shouldn't be huge practical issues and it could be
> affordable in principle to include even unwanted objects.

Just keep them? What we're doing here is trying to optimize in the
case that someone has a sparse checkout, nothing bad will happen if
they get too many info surely? (Save for them not getting as much
improvement as would have been possible would the pack have been
created differently.)

-- 
Cheers,

Sverre Rabbelier

Re: [PATCH 02/12] git-grep: support --no-external-grep

[Alex Riesen]

Nguyễn Thái Ngọc Duy, Wed, Jul 23, 2008 16:55:49 +0200:
>  
> +static int no_external_grep;
> +

If you have used direct logic instead of negative, your patch would
have been simplier and the code would be easier to understand. I.e.:

+static int external_grep = 1;

> @@ -386,7 +388,7 @@ static int grep_cache(struct grep_opt *opt, const char **paths, int cached)
>  	 * we grep through the checked-out files. It tends to
>  	 * be a lot more optimized
>  	 */
> -	if (!cached) {
> +	if (!cached && !no_external_grep) {

+	if (!cached && external_grep) {

> @@ -544,6 +546,10 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
>  			cached = 1;
>  			continue;
>  		}
> +		if (!strcmp("--no-external-grep", arg)) {
> +			no_external_grep = 1;
> +			continue;
> +		}

+		if (!strcmp("--no-external-grep", arg)) {
+			external_grep = 0;
+			continue;
+		}

BTW, how about a config option for this? And maybe --external-grep={yes|no}

Re: [PATCH 02/12] git-grep: support --no-external-grep

[Nguyen Thai Ngoc Duy]

On 7/25/08, Alex Riesen <raa.lkml@gmail.com> wrote:
> Nguyen Thai Ngoc Duy, Wed, Jul 23, 2008 16:55:49 +0200:
>
> >
>  > +static int no_external_grep;
>  > +
>
>
> If you have used direct logic instead of negative, your patch would
>  have been simplier and the code would be easier to understand. I.e.:

Point taken.

>  +               if (!strcmp("--no-external-grep", arg)) {
>  +                       external_grep = 0;
>  +                       continue;
>  +               }
>
>  BTW, how about a config option for this? And maybe --external-grep={yes|no}

I don't need --external-grep=yes now, but that would be good for
testing external grep. About config option, is it really needed?
-- 
Duy

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Johannes Schindelin]

Hi,

On Thu, 24 Jul 2008, Jeff King wrote:

> On Thu, Jul 24, 2008 at 06:41:03PM +0100, Johannes Schindelin wrote:
> 
> > > As a user, I would expect "sparse clone" to also be sparse on the 
> > > fetching. That is, to not even bother fetching tree objects that we 
> > > are not going to check out. But that is a whole other can of worms 
> > > from local sparseness, so I think it is worth saving for a different 
> > > series.
> > 
> > I think this is not even worth of a series.  Sure, it would have 
> > benefits for those who want sparse checkouts.  But it comes for a high 
> > price on everyone else:
> 
> I agree there are a lot of issues. I am just thinking of the person who 
> said they had a >100G repository. But I am also not volunteering to do 
> it, so I will let somebody who really cares about it try to defend the 
> idea.

I never said that there were no benefits.  I argued that there are too 
many _downsides_ to those who _don't_ benefit.

Ciao,
Dscho

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Johannes Schindelin]

Hi,

On Thu, 24 Jul 2008, Sverre Rabbelier wrote:

> On Thu, Jul 24, 2008 at 8:53 PM, Petr Baudis <pasky@suse.cz> wrote:
> >  I don't follow how these two issues arise, if the server will do the 
> > pruning for you. It will just skip entering some tree objects when 
> > doing object traversal; why opening the git protocol or faking 
> > commits? This would be a simple extra capability in the protocol.
> 
> Wouldn't that be as simple as passing a pathspec to git-rev-list? Not a 
> lot of overhead there I reckon.

So the server would _not_ have to deflate the objects to inspect them?  I 
thought you knew more about Git's object database.

> > One question is what to do with delta chains including unwanted 
> > objects, but I think that given the objects' associativity for delta 
> > chains, this shouldn't be huge practical issues and it could be 
> > affordable in principle to include even unwanted objects.
> 
> Just keep them?

You'd still have to inspect the objects, which is way more work than the 
current code has to do.  Remember: in the optimal case, upload-pack does 
not more than just serve the existing deltas/base objects.

Ciao,
Dscho

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Petr Baudis]

  Hi,

On Fri, Jul 25, 2008 at 02:12:31AM +0200, Johannes Schindelin wrote:
> On Thu, 24 Jul 2008, Sverre Rabbelier wrote:
> 
> > On Thu, Jul 24, 2008 at 8:53 PM, Petr Baudis <pasky@suse.cz> wrote:
> > >  I don't follow how these two issues arise, if the server will do the 
> > > pruning for you. It will just skip entering some tree objects when 
> > > doing object traversal; why opening the git protocol or faking 
> > > commits? This would be a simple extra capability in the protocol.
> > 
> > Wouldn't that be as simple as passing a pathspec to git-rev-list? Not a 
> > lot of overhead there I reckon.
> 
> So the server would _not_ have to deflate the objects to inspect them?  I 
> thought you knew more about Git's object database.
..snip..
> You'd still have to inspect the objects, which is way more work than the 
> current code has to do.  Remember: in the optimal case, upload-pack does 
> not more than just serve the existing deltas/base objects.

  then right now, exactly how does the server decide that the blob
7a7ff130 should be served along git.git HEAD? I still see upload-pack.c
calling traverse_commit_list() that does process_tree() on every tree,
etc. But the code is not straightforward, maybe I'm missing some
shortcut?

-- 
				Petr "Pasky" Baudis
As in certain cults it is possible to kill a process if you know
its true name.  -- Ken Thompson and Dennis M. Ritchie

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[James Pickens]

Jeff King <peff <at> peff.net> writes:

> I agree there are a lot of issues. I am just thinking of the person who
> said they had a >100G repository. But I am also not volunteering to do
> it, so I will let somebody who really cares about it try to defend the
> idea.

If you're referring to me (I mentioned a 144G CVS repo), then let me
clarify a couple of things:

1.  Probably more than 50% of the 144G is crud that should never have
been checked in, but I have some undisciplined coworkers who like to
blindly check in everything in their work trees.  If/when we moved to
git, I would get rid of all that crud.  I'm also thinking about throwing
out a lot of the history, since those same undisciplined coworkers like
to use empty and/or useless log messages, so a lot of the history isn't
very valuable anyways.

2.  Git of course will store the remaining ~70G much more efficiently
than CVS.  I think git will be especially better than CVS for this repo,
because it contains many instances of the same file(s) being checked in
in multiple directories.

I expect the git repo size to be less than 7G.  In addition, all our
work is done on site on nfs, so we can use clone -s to avoid copying the
whole 7G.

To sum it up, sparse cloning would not be important to me.

James

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path?to do sparse clone

[Jeff King]

On Fri, Jul 25, 2008 at 12:46:52AM +0000, James Pickens wrote:

> If you're referring to me (I mentioned a 144G CVS repo), then let me
> clarify a couple of things:
> [...]
> To sum it up, sparse cloning would not be important to me.

I was referring to you, so thanks for the clarification.

-Peff

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Sverre Rabbelier]

On Fri, Jul 25, 2008 at 02:12, Johannes Schindelin
<Johannes.Schindelin@gmx.de> wrote:
> On Thu, 24 Jul 2008, Sverre Rabbelier wrote:
>> Wouldn't that be as simple as passing a pathspec to git-rev-list? Not a
>> lot of overhead there I reckon.
>
> So the server would _not_ have to deflate the objects to inspect them?  I
> thought you knew more about Git's object database.

Nope, I did not know this. I thought that the server already had to do
all that to decide what to send, since not every request asks for the
same pack (someone might not have updated in 2 month, or someone might
might be up to date, or anything in between).

>> Just keep them?
>
> You'd still have to inspect the objects, which is way more work than the
> current code has to do.  Remember: in the optimal case, upload-pack does
> not more than just serve the existing deltas/base objects.

Ah, I can see how that would produce overhead then.

-- 
Cheers,

Sverre Rabbelier

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Junio C Hamano]

Jeff King <peff@peff.net> writes:

> On Thu, Jul 24, 2008 at 06:41:03PM +0100, Johannes Schindelin wrote:
>
>> > As a user, I would expect "sparse clone" to also be sparse on the 
>> > fetching. That is, to not even bother fetching tree objects that we are 
>> > not going to check out. But that is a whole other can of worms from 
>> > local sparseness, so I think it is worth saving for a different series.
>> 
>> I think this is not even worth of a series.  Sure, it would have benefits 
>> for those who want sparse checkouts.  But it comes for a high price on 
>> everyone else:
>
> I agree there are a lot of issues. I am just thinking of the person who
> said they had a >100G repository. But I am also not volunteering to do
> it, so I will let somebody who really cares about it try to defend the
> idea.

I think sparse fetch is a lot worse than grafts and shallow clones which
are already bad.  These are all ways to introduce local inconsistency at
the object level and pretend everything is Ok, but the latter two do so
only at commit boundary and it is somewhat more manageable (but we still
do not handle it very well).  With sparse fetch, you cannot even guarantee
the integrity of individual commits with subtrees here and there missing.

I do think shallow checkout that says "I'll have the whole tree in the
index but the work tree will have only these paths checked out" makes
sense.  You do not need a fully populated work tree to create commits or
merges -- the only absolute minimum you need is a fully populated index.

In that sense, I think "protect index entries outside of these paths" (I
remember that the first round of this series was done around that notion)
is a wrong mentality to handle this.  We should think of this as more like
"you still populate the index with the whole tree, and you are free to
update them in any way you want, but we do not touch work tree outside
these areas".

This has a few ramifications:

 - If the user can somehow check out a path outside the "sparse" area, it
   is perfectly fine for the user to edit and "git add" it.  Such a method
   to check out a path outside the "sparse" area is a way to widen the
   "sparse" area the user originally set up;

 - When the user runs "merge", and it needs to present the user a working
   tree file because of conflicts at the file level, the user has to agree
   to widen the "sparse" area before being able to do so.  One way to do
   this is to refuse and fail the merge (and then the user needs to do
   that "unspecified way" of widening the "sparse" area first).  Another
   way would be to automatically widen the "sparse" area to include such
   conflicting paths.

 - And you would want to narrow it down after you do such a widening.

For many projects that has src/ and doc/ (git.git being one of them), it
is perfectly valid for a code person and a doc person to work in tandem.
In such a project, after the code person makes changes in her sparsely
checked out repository and making changes only to the src/ area and pushes
the results out, the doc person would run "git pull && git log -p
ORIG_HEAD" and updates the documentation in his sparsely checked out
repository that has only doc/ area.  The two parts are tied together and
they advance more or less in sync.  I think sparse checkout would be a
useful feature to help such a configuration.

Having said that, I however think that this can easily be misused as a CVS
style "one CVSROOT houses millions of totally unrelated projects" layout.
In CVS, the layout is perfectly fine because the system does not track
changes at anything higher than the level of individual files, but when
you naïvely map the layout to a system with tree-wide atomic commits, such
as git, it will defeat the whole point of using such a system.  The pace
these millions of unrelated projects advance do not have any relationship
with each other, but by tying them together in the same top-level tree,
the layout is introducing an unnecessary ordering between their commits.

Re: sparse fetch, was Re: [PATCH 08/12] git-clone: support --path to do sparse clone

[Sverre Rabbelier]

On Fri, Jul 25, 2008 at 10:47, Junio C Hamano <gitster@pobox.com> wrote:
> For many projects that has src/ and doc/ (git.git being one of them), it

We are? That's great, that'd mean I can actually do a 'ls' in the
git.git root! Oh wait...
$ ls | wc -l
693

-- 
Cheers,

Sverre Rabbelier