[PATCH] unpack-trees.c: assume submodules are clean during check-out

[PATCH] unpack-trees.c: assume submodules are clean during check-out

[Sven Verdoolaege]

In particular, when moving back to a commit without a given submodule
and then moving back forward to a commit with the given submodule,
we shouldn't complain that updating would lose untracked file in
the submodule, because git currently does not checkout subprojects
during superproject check-out.

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---

I'm not sure if t7400-submodule-basic is the best place
for the test, but it has the right context for the test.

skimo

 t/t7400-submodule-basic.sh |    9 +++++
 unpack-trees.c             |   75 +++++++++++++++++++++++++++++--------------
 2 files changed, 59 insertions(+), 25 deletions(-)

diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh
index 5e91db6..e8ce7cd 100755
--- a/t/t7400-submodule-basic.sh
+++ b/t/t7400-submodule-basic.sh
@@ -21,6 +21,10 @@ subcommands of git-submodule.
 #  -add an entry to .gitmodules for submodule 'example'
 #
 test_expect_success 'Prepare submodule testing' '
+	: > t &&
+	git-add t &&
+	git-commit -m "initial commit" &&
+	git branch initial HEAD &&
 	mkdir lib &&
 	cd lib &&
 	git init &&
@@ -166,4 +170,9 @@ test_expect_success 'status should be "up-to-date" after update' '
 	git-submodule status | grep "^ $rev1"
 '
 
+test_expect_success 'checkout superproject with subproject already present' '
+	git-checkout initial &&
+	git-checkout master
+'
+
 test_done
diff --git a/unpack-trees.c b/unpack-trees.c
index 89dd279..7cc029e 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -5,6 +5,7 @@
 #include "cache-tree.h"
 #include "unpack-trees.h"
 #include "progress.h"
+#include "refs.h"
 
 #define DBRT_DEBUG 1
 
@@ -425,11 +426,24 @@ static void invalidate_ce_path(struct cache_entry *ce)
 		cache_tree_invalidate_path(active_cache_tree, ce->name);
 }
 
-static int verify_clean_subdirectory(const char *path, const char *action,
+/*
+ * Check that checking out ce->sha1 in subdir ce->name is not
+ * going to overwrite any working files.
+ *
+ * Currently, git does not checkout subprojects during a superproject
+ * checkout, so it is not going to overwrite anything.
+ */
+static int verify_clean_submodule(struct cache_entry *ce, const char *action,
+				      struct unpack_trees_options *o)
+{
+	return 0;
+}
+
+static int verify_clean_subdirectory(struct cache_entry *ce, const char *action,
 				      struct unpack_trees_options *o)
 {
 	/*
-	 * we are about to extract "path"; we would not want to lose
+	 * we are about to extract "ce->name"; we would not want to lose
 	 * anything in the existing directory there.
 	 */
 	int namelen;
@@ -437,13 +451,24 @@ static int verify_clean_subdirectory(const char *path, const char *action,
 	struct dir_struct d;
 	char *pathbuf;
 	int cnt = 0;
+	unsigned char sha1[20];
+
+	if (S_ISGITLINK(ntohl(ce->ce_mode)) &&
+	    resolve_gitlink_ref(ce->name, "HEAD", sha1) == 0) {
+		/* If we are not going to update the submodule, then
+		 * we don't care.
+		 */
+		if (!hashcmp(sha1, ce->sha1))
+			return 0;
+		return verify_clean_submodule(ce, action, o);
+	}
 
 	/*
 	 * First let's make sure we do not have a local modification
 	 * in that directory.
 	 */
-	namelen = strlen(path);
-	pos = cache_name_pos(path, namelen);
+	namelen = strlen(ce->name);
+	pos = cache_name_pos(ce->name, namelen);
 	if (0 <= pos)
 		return cnt; /* we have it as nondirectory */
 	pos = -pos - 1;
@@ -451,7 +476,7 @@ static int verify_clean_subdirectory(const char *path, const char *action,
 		struct cache_entry *ce = active_cache[i];
 		int len = ce_namelen(ce);
 		if (len < namelen ||
-		    strncmp(path, ce->name, namelen) ||
+		    strncmp(ce->name, ce->name, namelen) ||
 		    ce->name[namelen] != '/')
 			break;
 		/*
@@ -469,16 +494,16 @@ static int verify_clean_subdirectory(const char *path, const char *action,
 	 * present file that is not ignored.
 	 */
 	pathbuf = xmalloc(namelen + 2);
-	memcpy(pathbuf, path, namelen);
+	memcpy(pathbuf, ce->name, namelen);
 	strcpy(pathbuf+namelen, "/");
 
 	memset(&d, 0, sizeof(d));
 	if (o->dir)
 		d.exclude_per_dir = o->dir->exclude_per_dir;
-	i = read_directory(&d, path, pathbuf, namelen+1, NULL);
+	i = read_directory(&d, ce->name, pathbuf, namelen+1, NULL);
 	if (i)
 		die("Updating '%s' would lose untracked files in it",
-		    path);
+		    ce->name);
 	free(pathbuf);
 	return cnt;
 }
@@ -487,7 +512,7 @@ static int verify_clean_subdirectory(const char *path, const char *action,
  * We do not want to remove or overwrite a working tree file that
  * is not tracked, unless it is ignored.
  */
-static void verify_absent(const char *path, const char *action,
+static void verify_absent(struct cache_entry *ce, const char *action,
 		struct unpack_trees_options *o)
 {
 	struct stat st;
@@ -495,15 +520,15 @@ static void verify_absent(const char *path, const char *action,
 	if (o->index_only || o->reset || !o->update)
 		return;
 
-	if (has_symlink_leading_path(path, NULL))
+	if (has_symlink_leading_path(ce->name, NULL))
 		return;
 
-	if (!lstat(path, &st)) {
+	if (!lstat(ce->name, &st)) {
 		int cnt;
 
-		if (o->dir && excluded(o->dir, path))
+		if (o->dir && excluded(o->dir, ce->name))
 			/*
-			 * path is explicitly excluded, so it is Ok to
+			 * ce->name is explicitly excluded, so it is Ok to
 			 * overwrite it.
 			 */
 			return;
@@ -515,7 +540,7 @@ static void verify_absent(const char *path, const char *action,
 			 * files that are in "foo/" we would lose
 			 * it.
 			 */
-			cnt = verify_clean_subdirectory(path, action, o);
+			cnt = verify_clean_subdirectory(ce, action, o);
 
 			/*
 			 * If this removed entries from the index,
@@ -543,7 +568,7 @@ static void verify_absent(const char *path, const char *action,
 		 * delete this path, which is in a subdirectory that
 		 * is being replaced with a blob.
 		 */
-		cnt = cache_name_pos(path, strlen(path));
+		cnt = cache_name_pos(ce->name, strlen(ce->name));
 		if (0 <= cnt) {
 			struct cache_entry *ce = active_cache[cnt];
 			if (!ce_stage(ce) && !ce->ce_mode)
@@ -551,7 +576,7 @@ static void verify_absent(const char *path, const char *action,
 		}
 
 		die("Untracked working tree file '%s' "
-		    "would be %s by merge.", path, action);
+		    "would be %s by merge.", ce->name, action);
 	}
 }
 
@@ -575,7 +600,7 @@ static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
 		}
 	}
 	else {
-		verify_absent(merge->name, "overwritten", o);
+		verify_absent(merge, "overwritten", o);
 		invalidate_ce_path(merge);
 	}
 
@@ -590,7 +615,7 @@ static int deleted_entry(struct cache_entry *ce, struct cache_entry *old,
 	if (old)
 		verify_uptodate(old, o);
 	else
-		verify_absent(ce->name, "removed", o);
+		verify_absent(ce, "removed", o);
 	ce->ce_mode = 0;
 	add_cache_entry(ce, ADD_CACHE_OK_TO_ADD|ADD_CACHE_OK_TO_REPLACE);
 	invalidate_ce_path(ce);
@@ -707,18 +732,18 @@ int threeway_merge(struct cache_entry **stages,
 	if (o->aggressive) {
 		int head_deleted = !head && !df_conflict_head;
 		int remote_deleted = !remote && !df_conflict_remote;
-		const char *path = NULL;
+		struct cache_entry *ce = NULL;
 
 		if (index)
-			path = index->name;
+			ce = index;
 		else if (head)
-			path = head->name;
+			ce = head;
 		else if (remote)
-			path = remote->name;
+			ce = remote;
 		else {
 			for (i = 1; i < o->head_idx; i++) {
 				if (stages[i] && stages[i] != o->df_conflict_entry) {
-					path = stages[i]->name;
+					ce = stages[i];
 					break;
 				}
 			}
@@ -733,8 +758,8 @@ int threeway_merge(struct cache_entry **stages,
 		    (remote_deleted && head && head_match)) {
 			if (index)
 				return deleted_entry(index, index, o);
-			else if (path && !head_deleted)
-				verify_absent(path, "removed", o);
+			else if (ce && !head_deleted)
+				verify_absent(ce, "removed", o);
 			return 0;
 		}
 		/*
-- 
1.5.3.rc2.10.g148b

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Junio C Hamano]

Passing of ce instead of path in the unpack-trees callchain
looks like the right thing to do.  Good job.

Thanks.

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Sven Verdoolaege]

On Wed, Jul 18, 2007 at 12:29:52AM -0700, Junio C Hamano wrote:
> Passing of ce instead of path in the unpack-trees callchain
> looks like the right thing to do.  Good job.

Actually, my patch only fixes the tip of the iceberg.
If you have a submodule checked out and you go back (or forward)
to a revision of the supermodule that contains a different
revision of the submodule and then switch to another revision,
it will complain that the submodule is not uptodate, because
git simply didn't update the submodule in the first move.

Now, you may say that I simply need to run 'git submodule update'
after every such move, but this is very inconvenient, especially
if you're doing a bisect or a rebase.

How do other people deal with this problem?

How about just replacing the body of ce_compare_gitlink
with "return 0" until git actually (optionally) updates
the submodules during an update of the supermodule?

skimo

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Junio C Hamano]

Sven Verdoolaege <skimo@kotnet.org> writes:

> If you have a submodule checked out and you go back (or forward)
> to a revision of the supermodule that contains a different
> revision of the submodule and then switch to another revision,
> it will complain that the submodule is not uptodate, because
> git simply didn't update the submodule in the first move.
>
> Now, you may say that I simply need to run 'git submodule update'
> after every such move, but this is very inconvenient, especially
> if you're doing a bisect or a rebase.
>
> How do other people deal with this problem?
>
> How about just replacing the body of ce_compare_gitlink
> with "return 0" until git actually (optionally) updates
> the submodules during an update of the supermodule?

Let me understand the problem first.  If your first checkout
does not check out the submodule, switching between revisions
that has different commit of the submodule there would not fail,
but once you checkout the submodule, switching without updating
the submodule would be Ok (because by design updating the
submodule is optional) but then further switching out of that
state will fail because submodule in the supermodule tree and
checked-out submodule repository are now out of sync.  Is that
the problem?

In any case, I doubt ce_compare_gitlink() is the right layer to
work this around -- it is not about "can we switch" but is about
"is it different".  It is at too low a level.

The current policy is to consider it is perfectly normal that
checked-out submodule is out-of-sync wrt the supermodule index,
if I am reading you right.  I think it is a good policy, at
least until we introduce a superproject repository configuration
option that says "in this repository, I do care about this
submodule and at any time I move around in the superproject,
recursively check out the submodule to match".  The most extreme
case of this policy is that the superproject index knows about
the submodule but the subdirectory does not even have to be
checked out, which is what we have now.

Where does the "No you are not up-to-date, I wouldn't let you
switch" come from?  Is that verify_uptodate() called from
merged_entry() called from twoway_merge()?  I think the right
approach to deal with this is to teach verify_uptodate() about
the policy.  The function is about "make sure the filesystem
entity that corresponds to this cache entry is up to date, lest
we lose the local modifications".  As we explicitly allow
submodule checkout to drift from the supermodule index entry,
the check should say "Ok, for submodules, not matching is the
norm" for now.  Later when we have the ability to mark "I care
about this submodule to be always in sync with the superproject"
(thereby implementing automatic recursive checkout and perhaps
diff, among other things), we should check if the submodule in
question is marked as such and perform the current test.

How about doing something like this instead?

 unpack-trees.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/unpack-trees.c b/unpack-trees.c
index 3b32718..dfd985b 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -407,6 +407,15 @@ static void verify_uptodate(struct cache_entry *ce,
 		unsigned changed = ce_match_stat(ce, &st, 1);
 		if (!changed)
 			return;
+		/*
+		 * NEEDSWORK: the current default policy is to allow
+		 * submodule to be out of sync wrt the supermodule
+		 * index.  This needs to be tightened later for
+		 * submodules that are marked to be automatically
+		 * checked out.
+		 */
+		if (S_ISGITLINK(ntohl(ce->ce_mode)))
+			return;
 		errno = 0;
 	}
 	if (errno == ENOENT)

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Lars Hjemli]

On 8/4/07, Junio C Hamano <gitster@pobox.com> wrote:
> As we explicitly allow
> submodule checkout to drift from the supermodule index entry,
> the check should say "Ok, for submodules, not matching is the
> norm" for now.  Later when we have the ability to mark "I care
> about this submodule to be always in sync with the superproject"
> (thereby implementing automatic recursive checkout and perhaps
> diff, among other things), we should check if the submodule in
> question is marked as such and perform the current test.

Yes, this sounds like a sane plan (and a good explanation of the
current semantics: maybe something to include in the release notes for
1.5.3?)

Btw: I've applied your patch to rc-4 and tested the result in my cgit
repo: very nice, and very ack'd ;-)

--
larsh

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Eran Tromer]

On 2007-08-04 01:13, Junio C Hamano wrote:
> Let me understand the problem first.  If your first checkout
> does not check out the submodule, switching between revisions
> that has different commit of the submodule there would not fail,
> but once you checkout the submodule, switching without updating
> the submodule would be Ok (because by design updating the
> submodule is optional) but then further switching out of that
> state will fail because submodule in the supermodule tree and
> checked-out submodule repository are now out of sync.  Is that
> the problem?
> 
[snip]

> Where does the "No you are not up-to-date, I wouldn't let you
> switch" come from?  Is that verify_uptodate() called from
> merged_entry() called from twoway_merge()?  I think the right
> approach to deal with this is to teach verify_uptodate() about
> the policy.  The function is about "make sure the filesystem
> entity that corresponds to this cache entry is up to date, lest
> we lose the local modifications".  As we explicitly allow
> submodule checkout to drift from the supermodule index entry,
> the check should say "Ok, for submodules, not matching is the
> norm" for now.  Later when we have the ability to mark "I care
> about this submodule to be always in sync with the superproject"
> (thereby implementing automatic recursive checkout and perhaps
> diff, among other things), we should check if the submodule in
> question is marked as such and perform the current test.
> 
> How about doing something like this instead?
> 
>  unpack-trees.c |    9 +++++++++

Works here: it silences the check and allows switching branches. Still,
leaving the working tree dirty can inadvertently affect subsequent
commits. Consider the most ordinary of sequences:

$ git checkout experimental-death-ray
$ git submodules update
(return a week later, woozy from the vacation.)
$ git checkout master
(hack hack hack)
$ git commit -a -m "fixed typos"
$ git push
(Oops. You've just accidentally committed the wrong submodule heads.)

So to safely make new commits you must remember to always run "git
submodule update", or forgo use of "git commit -a", whenever submodules
might be involved.

I guess you can hack around this by excluding submodules from "commit
-a" and (for scripts) "ls-files -m" too...

Another approach is for pull, checkout etc. to automatically update the
submodule' head ref, but no more. In this case the supermodule always
sees a consistent state with traditional semantics, but the *submodule*
ends up with a dirty working tree and a head referring to a
possibly-missing commit; "git submodule update" would need to clean that up.

  Eran

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Junio C Hamano]

"Lars Hjemli" <hjemli@gmail.com> writes:

> On 8/4/07, Junio C Hamano <gitster@pobox.com> wrote:
>> As we explicitly allow
>> submodule checkout to drift from the supermodule index entry,
>> the check should say "Ok, for submodules, not matching is the
>> norm" for now.  Later when we have the ability to mark "I care
>> about this submodule to be always in sync with the superproject"
>> (thereby implementing automatic recursive checkout and perhaps
>> diff, among other things), we should check if the submodule in
>> question is marked as such and perform the current test.
>
> Yes, this sounds like a sane plan (and a good explanation of the
> current semantics: maybe something to include in the release notes for
> 1.5.3?)

The submodule Porcelain is a new thing in 1.5.3, so we would
need a good description of what the current rules are and what
our vision for future enhancement will be.

I am not certain however the above is the accurate description
of the current design and the direction we would want to go,
though.  Somebody needs to sanity check me.

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Junio C Hamano]

Eran Tromer <git2eran@tromer.org> writes:

> Another approach is for pull, checkout etc. to automatically update the
> submodule' head ref, but no more. In this case the supermodule always
> sees a consistent state with traditional semantics, but the *submodule*
> ends up with a dirty working tree and a head referring to a
> possibly-missing commit; "git submodule update" would need to clean that up.

That however would introduce the same problem as "pushing into
live repository to update its HEAD while the index and working
tree are looking the other way".  We would need to somehow make
it possible for the subdirectory to remember which commit the
index and the working tree are derived from, so that the local
changes can be merged back to the revision the submodule HEAD
points at.

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Sven Verdoolaege]

On Fri, Aug 03, 2007 at 10:13:09PM -0700, Junio C Hamano wrote:
> Let me understand the problem first.  If your first checkout
> does not check out the submodule, switching between revisions
> that has different commit of the submodule there would not fail,
> but once you checkout the submodule, switching without updating
> the submodule would be Ok (because by design updating the
> submodule is optional) but then further switching out of that
> state will fail because submodule in the supermodule tree and
> checked-out submodule repository are now out of sync.  Is that
> the problem?

Yes.

> In any case, I doubt ce_compare_gitlink() is the right layer to
> work this around -- it is not about "can we switch" but is about
> "is it different".  It is at too low a level.

You are right.  I followed the logic down to ce_compare_gitlink,
but I should have backed up again.

> The current policy is to consider it is perfectly normal that
> checked-out submodule is out-of-sync wrt the supermodule index,
> if I am reading you right.  I think it is a good policy, at
> least until we introduce a superproject repository configuration
> option that says "in this repository, I do care about this
> submodule and at any time I move around in the superproject,
> recursively check out the submodule to match".  The most extreme
> case of this policy is that the superproject index knows about
> the submodule but the subdirectory does not even have to be
> checked out, which is what we have now.

Yes.  Alex Riesen convinced me that having the submodule checked
out is a good indication that you care about the submodule being
in sync (although you then apparently convinced him that this is
not a good idea).  You didn't take any patch that implements this
yet, so I'll probably try again after you release 1.5.3.

Since some people seem to like the current situation, I'd only do
the updating of checked-out submodules if some "autoUpdateSubmodules"
is set.

> How about doing something like this instead?

Works like a charm.

On Sat, Aug 04, 2007 at 01:41:06PM +0200, Lars Hjemli wrote:
> Btw: I've applied your patch to rc-4 and tested the result in my cgit
> repo: very nice, and very ack'd ;-)

If it matters, a definite

Acked-by: Sven Verdoolaege <skimo@kotnet.org>

too.

skimo

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Sven Verdoolaege]

On Sat, Aug 04, 2007 at 12:03:28PM -0400, Eran Tromer wrote:
> Works here: it silences the check and allows switching branches. Still,
> leaving the working tree dirty can inadvertently affect subsequent
> commits. Consider the most ordinary of sequences:
> 
> $ git checkout experimental-death-ray
> $ git submodules update
> (return a week later, woozy from the vacation.)
> $ git checkout master

Here, it'll warn that your submodule isn't up-to-date.

> (hack hack hack)
> $ git commit -a -m "fixed typos"

And if you run "git status" first, it'll tell you that the submodule
(still) isn't up-to-date.

> $ git push
> (Oops. You've just accidentally committed the wrong submodule heads.)

You always have to be careful when doing "git commit -a".

> Another approach is for pull, checkout etc. to automatically update the
> submodule' head ref, but no more.

Then everything, including "git submodule update", would assume
that the submodule is up-to-date.

skimo

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Eran Tromer]

On 2007-08-05 10:46, Sven Verdoolaege wrote:
>> $ git checkout experimental-death-ray
>> $ git submodules update
>> (return a week later, woozy from the vacation.)
>> $ git checkout master
> 
> Here, it'll warn that your submodule isn't up-to-date.
> 
>> (hack hack hack)
>> $ git commit -a -m "fixed typos"
> 
> And if you run "git status" first, it'll tell you that the submodule
> (still) isn't up-to-date.
> 
>> $ git push
>> (Oops. You've just accidentally committed the wrong submodule heads.)
> 
> You always have to be careful when doing "git commit -a".

Exactly. You now have to be very careful, whereas previously
$ git checkout master && vi foo && git commit -a -m "fixed typos"
was perfectly safe.

Worse yet, it could also be a script making similar assumptions. For
example, consider the tree filter in git-filter-branch. It used to be
fine, but will now corrupt the rewritten trees when submodules are
involved. Here's the relevant code from git-filter-branch.sh:

-----------------------------------------------------------------
while read commit parents; do
...
		git read-tree -i -m $commit
...
		git checkout-index -f -u -a ||
			die "Could not checkout the index"
...
		eval "$filter_tree" < /dev/null ||
			die "tree filter failed: $filter_tree"

		git diff-index -r $commit | cut -f 2- | tr '\n' '\0' | \
			xargs -0 git update-index --add --replace --remove
...
	sh -c "$filter_commit" "git commit-tree" \
		$(git write-tree) $parentstr < ../message > ../map/$commit
done <../revs
-----------------------------------------------------------------


>> Another approach is for pull, checkout etc. to automatically update the
>> submodule' head ref, but no more.
> 
> Then everything, including "git submodule update", would assume
> that the submodule is up-to-date.

With that approach, "git submodule update" would fetch the submodule's
head commit (which could be missing), and then check it against the
submodule's index (and maybe its work tree).

  Eran

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Sven Verdoolaege]

On Mon, Aug 06, 2007 at 02:42:20PM -0400, Eran Tromer wrote:
> On 2007-08-05 10:46, Sven Verdoolaege wrote:
> > You always have to be careful when doing "git commit -a".
> 
> Exactly. You now have to be very careful, whereas previously
> $ git checkout master && vi foo && git commit -a -m "fixed typos"
> was perfectly safe.

I don't see the difference.  If you forgot you changed something
(be it a submodule or a file) you will commit something you
didn't plan to commit.

    bash-3.00$ git init; touch a b c; git add .; git commit  -m 1
    Initialized empty Git repository in .git/
    Created initial commit 4e6da45: 1
     0 files changed, 0 insertions(+), 0 deletions(-)
     create mode 100644 a
     create mode 100644 b
     create mode 100644 c
    bash-3.00$ git checkout -b branch
    Switched to a new branch "branch"
    bash-3.00$ echo "foo" > a; git add a; git commit -m 2
    Created commit fe87123: 2
     1 files changed, 1 insertions(+), 0 deletions(-)
    bash-3.00$ echo "bar" > c
    bash-3.00$ git checkout master && echo "test" > b && git commit -a -m 'change b'
    M       c
    Switched to branch "master"
    Created commit 657c5b1: change b
     2 files changed, 2 insertions(+), 0 deletions(-)

> >> Another approach is for pull, checkout etc. to automatically update the
> >> submodule' head ref, but no more.
> > 
> > Then everything, including "git submodule update", would assume
> > that the submodule is up-to-date.
> 
> With that approach, "git submodule update" would fetch the submodule's
> head commit (which could be missing), and then check it against the
> submodule's index (and maybe its work tree).

And how is anyone supposed to figure out what HEAD the submodule's
index and working tree correspond to?
I can only hope that "git submodule update" would never blindly assume
that the submodule is clean and so the user would have to manually
sync the HEAD and the working tree.

skimo

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Eran Tromer]

On 2007-08-06 15:03, Sven Verdoolaege wrote:
> I don't see the difference.  If you forgot you changed something
> (be it a submodule or a file) you will commit something you
> didn't plan to commit.
...
>     bash-3.00$ git checkout master && echo "test" > b && git commit -a -m 'change b'
>     M       c
>     Switched to branch "master"
>     Created commit 657c5b1: change b
>      2 files changed, 2 insertions(+), 0 deletions(-)

Yes, you're right. (When I tried this, checkout complained about the
dirty working tree because a *merge* was needed.)

So let's try to explicitly reset the index and work tree:

$ git reset --hard master
$ vi foo
$ git commit -a -m 'fixed typos'

Oops, still a corrupt commit.


>>>> Another approach is for pull, checkout etc. to automatically update the
>>>> submodule' head ref, but no more.
>>> Then everything, including "git submodule update", would assume
>>> that the submodule is up-to-date.
>> With that approach, "git submodule update" would fetch the submodule's
>> head commit (which could be missing), and then check it against the
>> submodule's index (and maybe its work tree).
> And how is anyone supposed to figure out what HEAD the submodule's
> index and working tree correspond to?

What HEAD corresponds to any other dirty index or dirty working tree?
It's irrelevant and may not exist. You just have some random dirty state.

If it's the yet-to-exist submodule merging you're worried about, the
submodule's old head can be saved in ORIG_HEAD or some such during the
supermodule checkout.


> I can only hope that "git submodule update" would never blindly assume
> that the submodule is clean and so the user would have to manually
> sync the HEAD and the working tree.

Why would it assume that? In this approach, and ignoring submodule
merging for now, "git submodule update" should mean roughly "cd
submodule && git fetch HEAD && git reset --hard HEAD". After all, this
is really the only way to end up with the prescribed commit sha1.

I agree that for safety it makes sense to warn or abort if the index
doesn't match ORIG_HEAD (saved by the supermodule checkout) or if the
index doesn't match the work tree.

  Eran

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Sven Verdoolaege]

On Mon, Aug 06, 2007 at 11:24:46PM -0400, Eran Tromer wrote:
> On 2007-08-06 15:03, Sven Verdoolaege wrote:
> >>>> Another approach is for pull, checkout etc. to automatically update the
> >>>> submodule' head ref, but no more.
> >>> Then everything, including "git submodule update", would assume
> >>> that the submodule is up-to-date.
> >> With that approach, "git submodule update" would fetch the submodule's
> >> head commit (which could be missing), and then check it against the
> >> submodule's index (and maybe its work tree).
> > And how is anyone supposed to figure out what HEAD the submodule's
> > index and working tree correspond to?
> 
> What HEAD corresponds to any other dirty index or dirty working tree?
> It's irrelevant and may not exist. You just have some random dirty state.

The only way to know that it's dirty is if you know the HEAD.
How can that not be relevant.

> > I can only hope that "git submodule update" would never blindly assume
> > that the submodule is clean and so the user would have to manually
> > sync the HEAD and the working tree.
> 
> Why would it assume that? In this approach, and ignoring submodule
> merging for now, "git submodule update" should mean roughly "cd
> submodule && git fetch HEAD && git reset --hard HEAD".

If you're doing that, then that is exactly what you are assuming.

> After all, this
> is really the only way to end up with the prescribed commit sha1.

That's the best way of losing all you precious changes in the submodule.
And there is no way to get them back!
Surely this is a lot worse than occasionally committing something you
didn't plan to commit, and only if you are performing a known "dangerous"
operation.

> I agree that for safety it makes sense to warn or abort if the index
> doesn't match ORIG_HEAD (saved by the supermodule checkout) or if the
> index doesn't match the work tree.

You may have done several supermodule checkouts since you last changed
the submodule.

skimo

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Eran Tromer]

On 2007-08-07 04:51, Sven Verdoolaege wrote:
> Surely this is a lot worse than occasionally committing something you
> didn't plan to commit, and only if you are performing a known "dangerous"
> operation.
> 
Are you saying that
$ git reset --hard HEAD && vi foo && git commit -a
is a "known dangerous" operation that can record corrupted content even
though you didn't touch it? This is very bad news indeed! I don't see
any such warnings in the documentation.

So, when I'm sure all the edits I did in the work tree are fine, how
*do* I safely make a commit without manually inspecting the changed
files list, or manually listing the changed files for "git add", or
manually running "git submodule update", or manually checking whether
there happens to be some submodules in this project, some other such
cumbersome measure?


> You may have done several supermodule checkouts since you last changed
> the submodule.

True, that approach won't work. I can imagine some logic to
conditionally update ORIG_HEAD, but it gets messy and fragile. Looks
like brokenness is just inevitable when you let the state get stale and
then merrily read it out as if it's fresh.


So.... Maybe we can tackle this head-on? Let index entries be explicitly
marked as "adrift", meaning we just don't touch the work tree for these
entries -- neither reads nor writes. It's used when the piece of
content, say a submodule, is allowed to drift arbitrarily in the work
tree in a way that doesn't represent meaningful edits that should be
reflected in commits, diffs, etc.

For example:
- "git checkout" sets the "adrift" flag on all (modified?) submodules
- "git submodule update" undrifts ("moores?") the submodules
- "git commit -a" skips files that are adrift, and likewise "git add .",
  "git diff" etc. (perhaps with some warning?)
- "git add <path>" undrifts <path> and proceeds as usual
- "git status" reports drifting files as such and doesn't bother to
  check them in the work tree
- When merging into the work tree, drifting files are left as such

And why stop at submodules? If there's a large blob you don't want to
check out, just "git drift <path>" it. To set whole *directories*
adrift, we can piggybacking on the empty-directory support (i.e., add a
directory entry to the index and set it adrift). So this could be the
basis of partial-checkout support.

Does this sound reasonable?

  Eran

Re: [PATCH] unpack-trees.c: assume submodules are clean during check-out

[Sven Verdoolaege]

On Tue, Aug 07, 2007 at 09:41:34PM -0400, Eran Tromer wrote:
> On 2007-08-07 04:51, Sven Verdoolaege wrote:
> > Surely this is a lot worse than occasionally committing something you
> > didn't plan to commit, and only if you are performing a known "dangerous"
> > operation.
> > 
> Are you saying that
> $ git reset --hard HEAD && vi foo && git commit -a
> is a "known dangerous" operation that can record corrupted content even
> though you didn't touch it?

I'm only saying that "git commit -a" will commit anything that has been
modified, so you have to be careful when using it and it just so happens
that git reset may leave submodules modified.  This should probably
be documented.
And I agree with you that this is not ideal (personally, I'd want
all checked-out submodules to get updated automatically), but it's
certainly better than your earlier proposal.

> So, when I'm sure all the edits I did in the work tree are fine, how
> *do* I safely make a commit without manually inspecting the changed
> files list, or manually listing the changed files for "git add", or
> manually running "git submodule update", or manually checking whether
> there happens to be some submodules in this project, some other such
> cumbersome measure?

If you've ever done a "git submodule update" in the project, you should
know that there are submodules and if you haven't then there are no
checked-out submodules that can get out of sync.
If you're talking about tools, then they should indeed be extra careful.

[another proposal]
> 
> Does this sound reasonable?

I'll leave it to others to comment on that one.

skimo