[PATCH 4/4] save another call to git-update-index

[PATCH 4/4] save another call to git-update-index

[Alex Riesen]

and a small cleanup
---
 merge-recursive.c |   41 ++++++++++++++++++-----------------------
 1 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/merge-recursive.c b/merge-recursive.c
index 9a18e06..f3f8e7d 100644
--- a/merge-recursive.c
+++ b/merge-recursive.c
@@ -21,7 +21,7 @@ #include "graph.h"
 #include "path-list.h"
 
 #define HAVE_ALLOCA
-#define DEBUG
+/*#define DEBUG*/
 
 #ifdef DEBUG
 #define debug(args, ...) fprintf(stderr, args, ## __VA_ARGS__)
@@ -629,8 +629,10 @@ static struct rename_entry *get_renames(
 					struct tree *bTree,
 					struct index_entry **entries)
 {
+#ifdef DEBUG
 	time_t t = time(0);
 	debug("getRenames ...\n");
+#endif
 	struct rename_entry *renames = NULL;
 	struct rename_entry **rptr = &renames;
 	struct diff_options opts;
@@ -691,22 +693,16 @@ static int update_stages(FILE *up_index,
 {
 	if ( !up_index )
 		return -1;
-	if ( clear ) {
+	if ( clear )
 		fprintf(up_index, "0 %s\t%s", sha1_to_hex(null_sha1), path);
-		fputc('\0', up_index);
-	}
-	if ( omode ) {
+	if ( omode )
 		fprintf(up_index, "0%o %s 1\t%s", omode, sha1_to_hex(osha), path);
-		fputc('\0', up_index);
-	}
-	if ( amode ) {
+	if ( amode )
 		fprintf(up_index, "0%o %s 2\t%s", amode, sha1_to_hex(asha), path);
-		fputc('\0', up_index);
-	}
-	if ( bmode ) {
+	if ( bmode )
 		fprintf(up_index, "0%o %s 3\t%s", bmode, sha1_to_hex(bsha), path);
+	if ( clear || omode || amode || bmode )
 		fputc('\0', up_index);
-	}
 	return 0;
 }
 
@@ -1080,7 +1076,8 @@ static void conflict_rename_rename_2(FIL
 	free(newPath1);
 }
 
-static int process_renames(struct rename_entry *renamesA,
+static int process_renames(FILE* fp,
+			   struct rename_entry *renamesA,
 			   struct rename_entry *renamesB,
 			   const char *branchNameA,
 			   const char *branchNameB)
@@ -1097,7 +1094,6 @@ static int process_renames(struct rename
 	for (sre = renamesB; sre; sre = sre->next)
 		path_list_insert(sre->pair->one->path, &srcNames);
 
-	FILE *fp = git_update_index_pipe();
 	for_each_path(src,&srcNames) {
 		struct rename_entry *renames1, *renames2, *ren1, *ren2;
 		const char *branchName1, *branchName2;
@@ -1282,9 +1278,6 @@ static int process_renames(struct rename
 		}
 	}
 	path_list_clear(&srcNames, 0);
-	if (pclose(fp)) {
-		die("git update-index --index-info failed");
-	}
 	debug("  processRenames done\n");
 	return cleanMerge;
 }
@@ -1467,24 +1460,26 @@ static struct merge_tree_result merge_tr
 		struct rename_entry *re_head, *re_merge;
 		re_head  = get_renames(head, common, head, merge, &entries);
 		re_merge = get_renames(merge, common, head, merge, &entries);
-		result.clean = process_renames(re_head, re_merge,
+		FILE *up_index = git_update_index_pipe();
+		result.clean = process_renames(up_index,
+					       re_head, re_merge,
 					       branch1Name, branch2Name);
 		debug("\tprocessing entries...\n");
-		FILE *fp = git_update_index_pipe();
 		struct index_entry *e;
 		for (e = entries; e; e = e->next) {
 			if (e->processed)
 				continue;
-			if (!process_entry(fp, e, branch1Name, branch2Name))
+			if (!process_entry(up_index, e,
+					   branch1Name, branch2Name))
 				result.clean = 0;
 		}
+		if (pclose(up_index))
+			die("updating entry failed in git update-index");
+
 		free_rename_entries(&re_merge);
 		free_rename_entries(&re_head);
 		free_index_entries(&entries);
 
-		if (pclose(fp))
-			die("updating entry failed in git update-index");
-
 		if (result.clean || index_only)
 			result.tree = git_write_tree();
 		else
-- 
1.4.1.rc1.g17dc

Re: [PATCH 4/4] save another call to git-update-index

[Johannes Schindelin]

Hi,

FYI I've been just battling this pipe for a couple of hours. The first 
steps were easy, but since I wanted to do it incrementally, the index has 
to be written every so often, and I seem not to be able to get that right.

But now it's time to go to bed, so this will wait for tomorrow.

Ciao,
Dscho

Re: [PATCH 4/4] save another call to git-update-index

[Alex Riesen]

On 6/30/06, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
>
> FYI I've been just battling this pipe for a couple of hours. The first
> steps were easy, but since I wanted to do it incrementally, the index has
> to be written every so often, and I seem not to be able to get that right.
>
Are you sure? Does something use the index between starting update-index
pipe and pclose?

Re: [PATCH 4/4] save another call to git-update-index

[Johannes Schindelin]

Hi,

On Fri, 30 Jun 2006, Alex Riesen wrote:

> On 6/30/06, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> > 
> > FYI I've been just battling this pipe for a couple of hours. The first
> > steps were easy, but since I wanted to do it incrementally, the index has
> > to be written every so often, and I seem not to be able to get that right.
> > 
> Are you sure? Does something use the index between starting update-index
> pipe and pclose?

Yes, I am sure you have to write the cache before git-read-tree and 
git-write-tree ;-) I must have done something severely wrong, though, 
since there are not too many places where they are called. The problem is 
more likely that the cache has to be _read_, and _before_ the first 
substituted call to git-update-index.

Ciao,
Dscho

Re: [PATCH 4/4] save another call to git-update-index

[Alex Riesen]

On 6/30/06, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> > >
> > > FYI I've been just battling this pipe for a couple of hours. The first
> > > steps were easy, but since I wanted to do it incrementally, the index has
> > > to be written every so often, and I seem not to be able to get that right.
> > >
> > Are you sure? Does something use the index between starting update-index
> > pipe and pclose?
>
> Yes, I am sure you have to write the cache before git-read-tree and
> git-write-tree ;-)

Indeed :) That's what pclose before git-write-tree is for (which was not the
case as I published this optimization. Noticed it later. The last patch series
has pclose before git-write-tree)

> I must have done something severely wrong, though,
> since there are not too many places where they are called. The problem is
> more likely that the cache has to be _read_, and _before_ the first
> substituted call to git-update-index.

git-write-tree change index?

Re: [PATCH 4/4] save another call to git-update-index

[Johannes Schindelin]

Hi,

On Fri, 30 Jun 2006, Johannes Schindelin wrote:

> FYI I've been just battling this pipe for a couple of hours. The first 
> steps were easy, but since I wanted to do it incrementally, the index has 
> to be written every so often, and I seem not to be able to get that right.

I just finished it. See my upcoming series of three patches. These apply 
on top of your last cumulative patch (sometimes yesterday), although I am 
certain we can merge our efforts.

Ciao,
Dscho

[PATCH 1/3] Add read_cache_from() and discard_cache()

[Johannes Schindelin]

These functions will be useful for the inbuilt merge-recursive.

Signed-off-by: Johannes Schindelin <Johannes.Schindelin@gmx.de>
---
 cache.h      |    2 ++
 read-cache.c |   57 +++++++++++++++++++++++++++++++++++++++------------------
 2 files changed, 41 insertions(+), 18 deletions(-)

diff --git a/cache.h b/cache.h
index 8719939..a5343db 100644
--- a/cache.h
+++ b/cache.h
@@ -142,7 +142,9 @@ #define alloc_nr(x) (((x)+16)*3/2)
 
 /* Initialize and use the cache information */
 extern int read_cache(void);
+extern int read_cache_from(const char *path);
 extern int write_cache(int newfd, struct cache_entry **cache, int entries);
+extern int discard_cache(void);
 extern int verify_path(const char *path);
 extern int cache_name_pos(const char *name, int namelen);
 #define ADD_CACHE_OK_TO_ADD 1		/* Ok to add */
diff --git a/read-cache.c b/read-cache.c
index 3c32aae..04d2ec7 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -24,6 +24,9 @@ unsigned int active_nr = 0, active_alloc
 
 struct cache_tree *active_cache_tree = NULL;
 
+static void *cache_mmap = NULL;
+static size_t cache_mmap_size = 0;
+
 /*
  * This only updates the "non-critical" parts of the directory
  * cache, ie the parts that aren't tracked by GIT, and only used
@@ -729,39 +732,43 @@ static int read_index_extension(const ch
 
 int read_cache(void)
 {
+	return read_cache_from(get_index_file());
+}
+
+/* remember to discard_cache() before reading a different cache! */
+int read_cache_from(const char *path)
+{
 	int fd, i;
 	struct stat st;
-	unsigned long size, offset;
-	void *map;
+	unsigned long offset;
 	struct cache_header *hdr;
 
 	errno = EBUSY;
-	if (active_cache)
+	if (cache_mmap)
 		return active_nr;
 
 	errno = ENOENT;
 	index_file_timestamp = 0;
-	fd = open(get_index_file(), O_RDONLY);
+	fd = open(path, O_RDONLY);
 	if (fd < 0) {
 		if (errno == ENOENT)
 			return 0;
 		die("index file open failed (%s)", strerror(errno));
 	}
 
-	size = 0; // avoid gcc warning
-	map = MAP_FAILED;
+	cache_mmap = MAP_FAILED;
 	if (!fstat(fd, &st)) {
-		size = st.st_size;
+		cache_mmap_size = st.st_size;
 		errno = EINVAL;
-		if (size >= sizeof(struct cache_header) + 20)
-			map = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
+		if (cache_mmap_size >= sizeof(struct cache_header) + 20)
+			cache_mmap = mmap(NULL, cache_mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 	}
 	close(fd);
-	if (map == MAP_FAILED)
+	if (cache_mmap == MAP_FAILED)
 		die("index file mmap failed (%s)", strerror(errno));
 
-	hdr = map;
-	if (verify_hdr(hdr, size) < 0)
+	hdr = cache_mmap;
+	if (verify_hdr(hdr, cache_mmap_size) < 0)
 		goto unmap;
 
 	active_nr = ntohl(hdr->hdr_entries);
@@ -770,12 +777,12 @@ int read_cache(void)
 
 	offset = sizeof(*hdr);
 	for (i = 0; i < active_nr; i++) {
-		struct cache_entry *ce = (struct cache_entry *) ((char *) map + offset);
+		struct cache_entry *ce = (struct cache_entry *) ((char *) cache_mmap + offset);
 		offset = offset + ce_size(ce);
 		active_cache[i] = ce;
 	}
 	index_file_timestamp = st.st_mtime;
-	while (offset <= size - 20 - 8) {
+	while (offset <= cache_mmap_size - 20 - 8) {
 		/* After an array of active_nr index entries,
 		 * there can be arbitrary number of extended
 		 * sections, each of which is prefixed with
@@ -783,10 +790,10 @@ int read_cache(void)
 		 * in 4-byte network byte order.
 		 */
 		unsigned long extsize;
-		memcpy(&extsize, (char *) map + offset + 4, 4);
+		memcpy(&extsize, (char *) cache_mmap + offset + 4, 4);
 		extsize = ntohl(extsize);
-		if (read_index_extension(((const char *) map) + offset,
-					 (char *) map + offset + 8,
+		if (read_index_extension(((const char *) cache_mmap) + offset,
+					 (char *) cache_mmap + offset + 8,
 					 extsize) < 0)
 			goto unmap;
 		offset += 8;
@@ -795,11 +802,25 @@ int read_cache(void)
 	return active_nr;
 
 unmap:
-	munmap(map, size);
+	munmap(cache_mmap, cache_mmap_size);
 	errno = EINVAL;
 	die("index file corrupt");
 }
 
+int discard_cache()
+{
+	int ret;
+	
+	if (cache_mmap == NULL)
+		return 0;
+	ret = munmap(cache_mmap, cache_mmap_size);
+	cache_mmap = NULL;
+	cache_mmap_size = 0;
+	active_nr = active_cache_changed = 0;
+	/* no need to throw away allocated active_cache */
+	return ret;
+}
+
 #define WRITE_BUFFER_SIZE 8192
 static unsigned char write_buffer[WRITE_BUFFER_SIZE];
 static unsigned long write_buffer_len;
-- 
1.4.1.rc1.gb2d14

Re: [PATCH 1/3] Add read_cache_from() and discard_cache()

[Junio C Hamano]

Johannes Schindelin <Johannes.Schindelin@gmx.de> writes:

> +int discard_cache()
> +{
> +	int ret;
> +	
> +	if (cache_mmap == NULL)
> +		return 0;
> +	ret = munmap(cache_mmap, cache_mmap_size);
> +	cache_mmap = NULL;
> +	cache_mmap_size = 0;
> +	active_nr = active_cache_changed = 0;
> +	/* no need to throw away allocated active_cache */
> +	return ret;
> +}
> +

I haven't been following the details of the patches in this
thread while they are being cooked actively, but two things to
look out for are:

 - I am guessing you run discard_cache() because you want to
   read in a new cache (or start from a clean slate).  I am not
   sure what you are doing with the old cache tree data
   structure.  If you are starting from a clean slate
   (i.e. there is no read_cache_from() after you call
   discard_cache), you would probably need to discard the old
   cache tree; otherwise your next write-tree may produce an
   incorrect index file.  If you keep the old one and later
   swap it in, the problem might be even more severe.

 - index_timestamp is left as the old value in this patch when
   you switch cache using read_cache_from() directly.  I have a
   suspicion you may be bitten by "Racy Git" problem, especially
   because the operations are supposed to happen quickly thanks
   to the effort of you two ;-) increasing the risks that the
   file timestamp of the working tree file and the cached entry
   match.

Re: [PATCH 1/3] Add read_cache_from() and discard_cache()

[Johannes Schindelin]

Hi,

On Fri, 30 Jun 2006, Junio C Hamano wrote:

> Johannes Schindelin <Johannes.Schindelin@gmx.de> writes:
> 
> > +int discard_cache()
> > +{
> > +	int ret;
> > +	
> > +	if (cache_mmap == NULL)
> > +		return 0;
> > +	ret = munmap(cache_mmap, cache_mmap_size);
> > +	cache_mmap = NULL;
> > +	cache_mmap_size = 0;
> > +	active_nr = active_cache_changed = 0;
> > +	/* no need to throw away allocated active_cache */
> > +	return ret;
> > +}
> > +
> 
> I haven't been following the details of the patches in this
> thread while they are being cooked actively, but two things to
> look out for are:
> 
>  - I am guessing you run discard_cache() because you want to
>    read in a new cache (or start from a clean slate).  I am not
>    sure what you are doing with the old cache tree data
>    structure.  If you are starting from a clean slate
>    (i.e. there is no read_cache_from() after you call
>    discard_cache), you would probably need to discard the old
>    cache tree; otherwise your next write-tree may produce an
>    incorrect index file.  If you keep the old one and later
>    swap it in, the problem might be even more severe.

True, I missed that one. But it is just a call to 
cache_tree_free(active_cache_tree); in discard_cache(), right?

>  - index_timestamp is left as the old value in this patch when
>    you switch cache using read_cache_from() directly.  I have a
>    suspicion you may be bitten by "Racy Git" problem, especially
>    because the operations are supposed to happen quickly thanks
>    to the effort of you two ;-) increasing the risks that the
>    file timestamp of the working tree file and the cached entry
>    match.

Yes. Again, just one line to discard_cache(), right?

	index_file_timestamp = 0;

If there is more to it, please don't let me die dumb.

Ciao,
Dscho

Re: [PATCH 1/3] Add read_cache_from() and discard_cache()

[Junio C Hamano]

Johannes Schindelin <Johannes.Schindelin@gmx.de> writes:

> True, I missed that one. But it is just a call to 
> cache_tree_free(active_cache_tree); in discard_cache(), right?

On the codepath to write out the new index file, calling
cache_free_tree(&active_cache_tree) before write_cache() is all
that should be needed.  When "active_cache_tree == NULL",
write_cache() would write out an index file without the cached
tree information.

Currently not many things take advantage of cached tree
information to optimize its operation.  But I'd like to change
that.  For example, tree merges by read-tree should be able to
take advantage of the fact that a cached tree read from the
index and three trees being read all match for a subdirectory
and do the merge of the directory without descending into it.

>>  - index_timestamp is left as the old value in this patch when
>>    you switch cache using read_cache_from() directly.  I have a
>>    suspicion you may be bitten by "Racy Git" problem, especially
>>    because the operations are supposed to happen quickly thanks
>>    to the effort of you two ;-) increasing the risks that the
>>    file timestamp of the working tree file and the cached entry
>>    match.
>
> Yes. Again, just one line to discard_cache(), right?
>
> 	index_file_timestamp = 0;

This one I am not sure.  Read the comment in ce_match_stat() and
see the problematic sequence of events that this variable tries
to help resolve applies to your use.

Re: [PATCH 1/3] Add read_cache_from() and discard_cache()

[Johannes Schindelin]

Hi,

On Sat, 1 Jul 2006, Junio C Hamano wrote:

> Johannes Schindelin <Johannes.Schindelin@gmx.de> writes:
> 
> > True, I missed that one. But it is just a call to 
> > cache_tree_free(active_cache_tree); in discard_cache(), right?
> 
> On the codepath to write out the new index file, calling 
> cache_free_tree(&active_cache_tree) before write_cache() is all that 
> should be needed.  When "active_cache_tree == NULL", write_cache() would 
> write out an index file without the cached tree information.
> 
> Currently not many things take advantage of cached tree information to 
> optimize its operation.  But I'd like to change that.  For example, tree 
> merges by read-tree should be able to take advantage of the fact that a 
> cached tree read from the index and three trees being read all match for 
> a subdirectory and do the merge of the directory without descending into 
> it.

Together with my argument, that a library function should make life easier 
for you, not harder, you are making a fine point that the _clean() version 
of get_merge_bases() should clean the active_cache_tree, too.

Besides, is it not better to clean up now-bogus memory anyway?

> >>  - index_timestamp is left as the old value in this patch when
> >>    you switch cache using read_cache_from() directly.  I have a
> >>    suspicion you may be bitten by "Racy Git" problem, especially
> >>    because the operations are supposed to happen quickly thanks
> >>    to the effort of you two ;-) increasing the risks that the
> >>    file timestamp of the working tree file and the cached entry
> >>    match.
> >
> > Yes. Again, just one line to discard_cache(), right?
> >
> > 	index_file_timestamp = 0;
> 
> This one I am not sure.  Read the comment in ce_match_stat() and
> see the problematic sequence of events that this variable tries
> to help resolve applies to your use.

Okay. After reading the comment, I am quite certain we can just set the 
index_file_timestamp to 0.

Either we start with an empty cache; In that case it is obviously correct 
to set the timestamp to 0.

Or, after we discard the cache, we have to read a new cache before working 
on it. Now, reading the cache means calling read_cache() (or 
read_cache_from()), and -- alas -- line number 12 in the body of that 
function sets the timestamp to 0 _anyway_, to be reset to the correct 
value later.

So, I still think that these two lines should be in the cleanup part of 
get_merge_bases().

BTW personally, I prefer the one-function approach, i.e. a flag which says 
if it is okay not to clean up.

Ciao,
Dscho

Re: [PATCH 1/3] Add read_cache_from() and discard_cache()

[Junio C Hamano]

Johannes Schindelin <Johannes.Schindelin@gmx.de> writes:

> Okay. After reading the comment, I am quite certain we can just set the 
> index_file_timestamp to 0.

Thanks.

> So, I still think that these two lines should be in the cleanup part of 
> get_merge_bases().

I think that is sane -- please make it so.

> BTW personally, I prefer the one-function approach, i.e. a flag which says 
> if it is okay not to clean up.

Yup. Agreed.

Re: [PATCH 1/3] Add read_cache_from() and discard_cache()

[Johannes Schindelin]

Hi,

On Mon, 3 Jul 2006, Junio C Hamano wrote:

> Johannes Schindelin <Johannes.Schindelin@gmx.de> writes:
> 
> > Okay. After reading the comment, I am quite certain we can just set the 
> > index_file_timestamp to 0.
> 
> Thanks.
> 
> > So, I still think that these two lines should be in the cleanup part of 
> > get_merge_bases().
> 
> I think that is sane -- please make it so.

Ummm. After reading my original mail again: bad tobacco that was. Of 
course, I meant discard_cache(), not get_merge_bases(). Setting 
index_file_timestamp in get_merge_bases() would be wrong.

Sorry for the noise,
Dscho

Re: [PATCH 1/3] Add read_cache_from() and discard_cache()

[Junio C Hamano]

Johannes Schindelin <Johannes.Schindelin@gmx.de> writes:

> Ummm. After reading my original mail again: bad tobacco that was. Of 
> course, I meant discard_cache(), not get_merge_bases(). Setting 
> index_file_timestamp in get_merge_bases() would be wrong.

I was well aware you were talking about discard_cache().  I
think these two clean-ups are good to have there.

[PATCH 2/3] Make refresh_cache_entry() public

[Johannes Schindelin]

This renames refresh_entry() to refresh_cache_entry(), to make clashes
more unlikely, and makes it public. It also rethinks the rather awkward
way to pass an error: this is done by returning a NULL pointer and setting
cache_errno now.

Signed-off-by: Johannes Schindelin <Johannes.Schindelin@gmx.de>
---
 cache.h      |    2 ++
 read-cache.c |   44 +++++++++++++++++---------------------------
 2 files changed, 19 insertions(+), 27 deletions(-)

diff --git a/cache.h b/cache.h
index a5343db..a59b319 100644
--- a/cache.h
+++ b/cache.h
@@ -115,6 +115,7 @@ #define cache_entry_size(len) ((offsetof
 extern struct cache_entry **active_cache;
 extern unsigned int active_nr, active_alloc, active_cache_changed;
 extern struct cache_tree *active_cache_tree;
+extern int cache_errno;
 
 #define GIT_DIR_ENVIRONMENT "GIT_DIR"
 #define DEFAULT_GIT_DIR_ENVIRONMENT ".git"
@@ -151,6 +152,7 @@ #define ADD_CACHE_OK_TO_ADD 1		/* Ok to 
 #define ADD_CACHE_OK_TO_REPLACE 2	/* Ok to replace file/directory */
 #define ADD_CACHE_SKIP_DFCHECK 4	/* Ok to skip DF conflict checks */
 extern int add_cache_entry(struct cache_entry *ce, int option);
+extern struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really);
 extern int remove_cache_entry_at(int pos);
 extern int remove_file_from_cache(const char *path);
 extern int ce_same_name(struct cache_entry *a, struct cache_entry *b);
diff --git a/read-cache.c b/read-cache.c
index 04d2ec7..1c0fc8b 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -24,6 +24,8 @@ unsigned int active_nr = 0, active_alloc
 
 struct cache_tree *active_cache_tree = NULL;
 
+int cache_errno = 0;
+
 static void *cache_mmap = NULL;
 static size_t cache_mmap_size = 0;
 
@@ -580,22 +582,6 @@ int add_cache_entry(struct cache_entry *
 	return 0;
 }
 
-/* Three functions to allow overloaded pointer return; see linux/err.h */
-static inline void *ERR_PTR(long error)
-{
-	return (void *) error;
-}
-
-static inline long PTR_ERR(const void *ptr)
-{
-	return (long) ptr;
-}
-
-static inline long IS_ERR(const void *ptr)
-{
-	return (unsigned long)ptr > (unsigned long)-1000L;
-}
-
 /*
  * "refresh" does not calculate a new sha1 file or bring the
  * cache up-to-date for mode/content changes. But what it
@@ -607,14 +593,16 @@ static inline long IS_ERR(const void *pt
  * For example, you'd want to do this after doing a "git-read-tree",
  * to link up the stat cache details with the proper files.
  */
-static struct cache_entry *refresh_entry(struct cache_entry *ce, int really)
+struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really)
 {
 	struct stat st;
 	struct cache_entry *updated;
 	int changed, size;
 
-	if (lstat(ce->name, &st) < 0)
-		return ERR_PTR(-errno);
+	if (lstat(ce->name, &st) < 0) {
+		cache_errno = errno;
+		return NULL;
+	}
 
 	changed = ce_match_stat(ce, &st, really);
 	if (!changed) {
@@ -622,11 +610,13 @@ static struct cache_entry *refresh_entry
 		    !(ce->ce_flags & htons(CE_VALID)))
 			; /* mark this one VALID again */
 		else
-			return NULL;
+			return ce;
 	}
 
-	if (ce_modified(ce, &st, really))
-		return ERR_PTR(-EINVAL);
+	if (ce_modified(ce, &st, really)) {
+		cache_errno = EINVAL;
+		return NULL;
+	}
 
 	size = ce_size(ce);
 	updated = xmalloc(size);
@@ -669,13 +659,13 @@ int refresh_cache(unsigned int flags)
 			continue;
 		}
 
-		new = refresh_entry(ce, really);
-		if (!new)
+		new = refresh_cache_entry(ce, really);
+		if (new == ce)
 			continue;
-		if (IS_ERR(new)) {
-			if (not_new && PTR_ERR(new) == -ENOENT)
+		if (!new) {
+			if (not_new && cache_errno == ENOENT)
 				continue;
-			if (really && PTR_ERR(new) == -EINVAL) {
+			if (really && cache_errno == EINVAL) {
 				/* If we are doing --really-refresh that
 				 * means the index is not valid anymore.
 				 */
-- 
1.4.1.rc1.gb2d14

[PATCH 3/3] merge-recursive: avoid the pipe to update-index

[Johannes Schindelin]

Instead of a fork, we can use the plumbing ;-)

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
---
 graph.c           |   26 -----
 merge-recursive.c |  270 +++++++++++++++++++++++++++++++++++++----------------
 2 files changed, 191 insertions(+), 105 deletions(-)

diff --git a/graph.c b/graph.c
index fa2bfee..b784ea2 100644
--- a/graph.c
+++ b/graph.c
@@ -5,32 +5,6 @@ #include "cache.h"
 #include "commit.h"
 #include "graph.h"
 
-// does not belong here
-struct tree *git_write_tree()
-{
-#if 0
-	fprintf(stderr, "GIT_INDEX_FILE='%s' git-write-tree\n",
-		getenv("GIT_INDEX_FILE"));
-#endif
-	FILE *fp = popen("git-write-tree 2>/dev/null", "r");
-	char buf[41];
-	unsigned char sha1[20];
-	int ch;
-	unsigned i = 0;
-	while ( (ch = fgetc(fp)) != EOF )
-		if ( i < sizeof(buf)-1 && ch >= '0' && ch <= 'f' )
-			buf[i++] = ch;
-		else
-			break;
-	int rc = pclose(fp);
-	if ( rc == -1 || WEXITSTATUS(rc) )
-		return NULL;
-	buf[i] = '\0';
-	if ( get_sha1(buf, sha1) != 0 )
-		return NULL;
-	return lookup_tree(sha1);
-}
-
 const char *node_title(struct node *node, int *len)
 {
 	const char *s = "(null commit)";
diff --git a/merge-recursive.c b/merge-recursive.c
index 9bbb426..6d4e797 100644
--- a/merge-recursive.c
+++ b/merge-recursive.c
@@ -9,6 +9,7 @@ #include <sys/wait.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include "cache.h"
+#include "cache-tree.h"
 #include "commit.h"
 #include "blob.h"
 #include "tree-walk.h"
@@ -19,6 +20,47 @@ #include "run-command.h"
 #include "graph.h"
 #include "path-list.h"
 
+#ifdef DEBUG
+#include "quote.h"
+static void show_ce_entry(const char *tag, struct cache_entry *ce)
+{
+	if (tag && *tag &&
+	    (ce->ce_flags & htons(CE_VALID))) {
+		static char alttag[4];
+		memcpy(alttag, tag, 3);
+		if (isalpha(tag[0]))
+			alttag[0] = tolower(tag[0]);
+		else if (tag[0] == '?')
+			alttag[0] = '!';
+		else {
+			alttag[0] = 'v';
+			alttag[1] = tag[0];
+			alttag[2] = ' ';
+			alttag[3] = 0;
+		}
+		tag = alttag;
+	}
+
+	fprintf(stderr,"%s%06o %s %d\t",
+			tag,
+			ntohl(ce->ce_mode),
+			sha1_to_hex(ce->sha1),
+			ce_stage(ce));
+	write_name_quoted("", 0, ce->name,
+			'\n', stderr);
+	fputc('\n', stderr);
+}
+
+static void ls_files() {
+	int i;
+	for (i = 0; i < active_nr; i++) {
+		struct cache_entry *ce = active_cache[i];
+		show_ce_entry("", ce);
+	}
+	fprintf(stderr, "---\n");
+}
+#endif
+
 #define for_each_commit(p,list) for ( p = (list); p; p = p->next )
 
 struct merge_result
@@ -94,12 +136,68 @@ static void output(const char *fmt, ...)
 
 static const char *original_index_file;
 static const char *temporary_index_file;
+static int cache_dirty = 0;
+
+static int flush_cache()
+{
+	/* flush temporary index */
+	struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
+	int fd = hold_lock_file_for_update(lock, getenv("GIT_INDEX_FILE"));
+	if (fd < 0)
+		die("could not lock %s", temporary_index_file);
+	if (write_cache(fd, active_cache, active_nr) ||
+			commit_lock_file(lock))
+		die ("unable to write %s", temporary_index_file);
+	discard_cache();
+	cache_dirty = 0;
+	return 0;
+}
 
 static void setup_index(int temp)
 {
 	const char *idx = temp ? temporary_index_file: original_index_file;
+	if (cache_dirty)
+		die("fatal: cache changed flush_cache();");
 	unlink(temporary_index_file);
 	setenv("GIT_INDEX_FILE", idx, 1);
+	discard_cache();
+}
+
+static struct cache_entry *make_cache_entry(unsigned int mode,
+		const unsigned char *sha1, const char *path, int stage, int refresh)
+{
+	int size, len;
+	struct cache_entry *ce;
+
+	if (!verify_path(path))
+		return NULL;
+
+	len = strlen(path);
+	size = cache_entry_size(len);
+	ce = xcalloc(1, size);
+
+	memcpy(ce->sha1, sha1, 20);
+	memcpy(ce->name, path, len);
+	ce->ce_flags = create_ce_flags(len, stage);
+	ce->ce_mode = create_ce_mode(mode);
+
+	if (refresh)
+		return refresh_cache_entry(ce, 0);
+
+	return ce;
+}
+
+static int add_cacheinfo(unsigned int mode, const unsigned char *sha1,
+		const char *path, int stage, int refresh, int options)
+{
+	struct cache_entry *ce;
+	if (!cache_dirty)
+		read_cache_from(getenv("GIT_INDEX_FILE"));
+	cache_dirty++;
+	ce = make_cache_entry(mode, sha1 ? sha1 : null_sha1, path, stage, refresh);
+	if (!ce)
+		return error("cache_addinfo failed: %s", strerror(cache_errno));
+	return add_cache_entry(ce, options);
 }
 
 // This is a global variable which is used in a number of places but
@@ -119,6 +217,8 @@ #if 0
 		sha1_to_hex(tree->object.sha1));
 #endif
 	const char *argv[] = { "git-read-tree", NULL, NULL, };
+	if (cache_dirty)
+		die("read-tree with dirty cache");
 	argv[1] = sha1_to_hex(tree->object.sha1);
 	int rc = run_command_v(2, argv);
 	return rc < 0 ? -1: rc;
@@ -141,6 +241,8 @@ #endif
 		"git-read-tree", NULL, "-m", NULL, NULL, NULL,
 		NULL,
 	};
+	if (cache_dirty)
+		flush_cache();
 	argv[1] = update_arg;
 	argv[3] = sha1_to_hex(common->object.sha1);
 	argv[4] = sha1_to_hex(head->object.sha1);
@@ -149,6 +251,33 @@ #endif
 	return rc < 0 ? -1: rc;
 }
 
+struct tree *git_write_tree()
+{
+#if 0
+	fprintf(stderr, "GIT_INDEX_FILE='%s' git-write-tree\n",
+		getenv("GIT_INDEX_FILE"));
+#endif
+	if (cache_dirty)
+		flush_cache();
+	FILE *fp = popen("git-write-tree 2>/dev/null", "r");
+	char buf[41];
+	unsigned char sha1[20];
+	int ch;
+	unsigned i = 0;
+	while ( (ch = fgetc(fp)) != EOF )
+		if ( i < sizeof(buf)-1 && ch >= '0' && ch <= 'f' )
+			buf[i++] = ch;
+		else
+			break;
+	int rc = pclose(fp);
+	if ( rc == -1 || WEXITSTATUS(rc) )
+		return NULL;
+	buf[i] = '\0';
+	if ( get_sha1(buf, sha1) != 0 )
+		return NULL;
+	return lookup_tree(sha1);
+}
+
 struct merge_tree_result merge_trees(struct tree *head,
 				     struct tree *merge,
 				     struct tree *common,
@@ -640,36 +769,25 @@ struct rename_entry *getRenames(struct t
 	return renames;
 }
 
-static FILE *git_update_index_pipe()
-{
-	return popen("git-update-index -z --index-info", "w");
-}
-
-int setIndexStages(FILE *fp,
-		   const char *path,
+int setIndexStages(const char *path,
 		   unsigned char *osha, unsigned omode,
 		   unsigned char *asha, unsigned amode,
 		   unsigned char *bsha, unsigned bmode,
 		   int clear /* =True */)
 {
-	if ( !fp )
-		return -1;
-	if ( clear ) {
-		fprintf(fp, "0 %s\t%s", sha1_to_hex(null_sha1), path);
-		fputc('\0', fp);
-	}
-	if ( omode ) {
-		fprintf(fp, "0%o %s 1\t%s", omode, sha1_to_hex(osha), path);
-		fputc('\0', fp);
-	}
-	if ( amode ) {
-		fprintf(fp, "0%o %s 2\t%s", amode, sha1_to_hex(asha), path);
-		fputc('\0', fp);
-	}
-	if ( bmode ) {
-		fprintf(fp, "0%o %s 3\t%s", bmode, sha1_to_hex(bsha), path);
-		fputc('\0', fp);
-	}
+	int options = ADD_CACHE_OK_TO_ADD | ADD_CACHE_OK_TO_REPLACE;
+	if ( clear ) 
+		if (add_cacheinfo(0, null_sha1, path, 0, 0, options))
+			return -1;
+	if ( omode )
+		if (add_cacheinfo(omode, osha, path, 1, 0, options))
+			return -1;
+	if ( amode )
+		if (add_cacheinfo(omode, osha, path, 2, 0, options))
+			return -1;
+	if ( bmode )
+		if (add_cacheinfo(omode, osha, path, 3, 0, options))
+			return -1;
 	return 0;
 }
 
@@ -695,17 +813,17 @@ static int remove_path(const char *name)
 	return ret;
 }
 
-int removeFile(FILE *fp, int clean, const char *path)
+int removeFile(int clean, const char *path)
 {
 	int updateCache = index_only || clean;
 	int updateWd = !index_only;
 
 	if ( updateCache ) {
-		if ( !fp )
+		if (!cache_dirty)
+			read_cache_from(getenv("GIT_INDEX_FILE"));
+		cache_dirty++;
+		if (remove_file_from_cache(path))
 			return -1;
-		fprintf(fp, "0 %s\t%s", sha1_to_hex(null_sha1), path);
-		fputc('\0', fp);
-		return 0;
 	}
 	if ( updateWd )
 	{
@@ -785,8 +903,7 @@ static void flush_buffer(int fd, const c
 	}
 }
 
-void updateFileExt(FILE *fp,
-		   const unsigned char *sha,
+void updateFileExt(const unsigned char *sha,
 		   unsigned mode,
 		   const char *path,
 		   int updateCache,
@@ -830,20 +947,15 @@ void updateFileExt(FILE *fp,
 			    mode, sha1_to_hex(sha), path);
 	}
 	if ( updateCache )
-	{
-		// XXX just always use "git update-index --index-info"?
-		fprintf(fp, "%06o %s\t%s", mode, sha1_to_hex(sha), path);
-		fputc('\0', fp);
-	}
+		add_cacheinfo(mode, sha, path, 0, updateWd, ADD_CACHE_OK_TO_ADD);
 }
 
-void updateFile(FILE *fp,
-		int clean,
+void updateFile(int clean,
 		const unsigned char *sha,
 		unsigned mode,
 		const char *path)
 {
-	updateFileExt(fp, sha, mode, path, index_only || clean, !index_only);
+	updateFileExt(sha, mode, path, index_only || clean, !index_only);
 }
 
 // Low level file merging, update and removal
@@ -1004,7 +1116,6 @@ int processRenames(struct rename_entry *
 	for (sre = renamesB; sre; sre = sre->next)
 		path_list_insert(sre->src, &srcNames);
 
-	FILE *fp = git_update_index_pipe();
 	for_each_path(src,&srcNames) {
 		struct rename_entry *renames1, *renames2, *ren1, *ren2;
 		const char *branchName1, *branchName2;
@@ -1050,26 +1161,26 @@ int processRenames(struct rename_entry *
 					dstName1 = uniquePath(ren1->dst, branchName1);
 					output("%s is a directory in %s adding as %s instead",
 					       ren1->dst, branchName2, dstName1);
-					removeFile(fp, 0, ren1->dst);
+					removeFile(0, ren1->dst);
 				}
 				if ( path_list_has_path(&currentDirectorySet, ren2->dst) ) {
 					dstName2 = uniquePath(ren2->dst, branchName2);
 					output("%s is a directory in %s adding as %s instead",
 					       ren2->dst, branchName1, dstName2);
-					removeFile(fp, 0, ren2->dst);
+					removeFile(0, ren2->dst);
 				}
-				setIndexStages(fp, dstName1,
+				setIndexStages(dstName1,
 					       NULL, 0,
 					       ren1->dst_sha, ren1->dst_mode,
 					       NULL, 0,
 					       1 /* clear */);
-				setIndexStages(fp, dstName2,
+				setIndexStages(dstName2,
 					       NULL, 0,
 					       NULL, 0,
 					       ren2->dst_sha, ren2->dst_mode,
 					       1 /* clear */);
 			} else {
-				removeFile(fp, 1, ren1->src);
+				removeFile(1, ren1->src);
 				struct merge_file_info mfi;
 				mfi = mergeFile(ren1->src, ren1->src_sha, ren1->src_mode,
 						ren1->dst, ren1->dst_sha, ren1->dst_mode,
@@ -1087,18 +1198,17 @@ int processRenames(struct rename_entry *
 					cleanMerge = 0;
 
 					if ( !index_only )
-						setIndexStages(fp,
-							       ren1->dst,
+						setIndexStages(ren1->dst,
 							       ren1->src_sha, ren1->src_mode,
 							       ren1->dst_sha, ren1->dst_mode,
 							       ren2->dst_sha, ren2->dst_mode,
 							       1 /* clear */);
 				}
-				updateFile(fp, mfi.clean, mfi.sha, mfi.mode, ren1->dst);
+				updateFile(mfi.clean, mfi.sha, mfi.mode, ren1->dst);
 			}
 		} else {
 			// Renamed in 1, maybe changed in 2
-			removeFile(fp, 1, ren1->src);
+			removeFile(1, ren1->src);
 
 			unsigned char srcShaOtherBranch[20], dstShaOtherBranch[20];
 			unsigned srcModeOtherBranch, dstModeOtherBranch;
@@ -1123,15 +1233,15 @@ int processRenames(struct rename_entry *
 				       ren1->dst, branchName2);
 				output("Renaming %s to %s instead", ren1->src, newPath);
 				cleanMerge = 0;
-				removeFile(fp, 0, ren1->dst);
-				updateFile(fp, 0, ren1->dst_sha, ren1->dst_mode, newPath);
+				removeFile(0, ren1->dst);
+				updateFile(0, ren1->dst_sha, ren1->dst_mode, newPath);
 			} else if ( memcmp(srcShaOtherBranch, null_sha1, 20) == 0 ) {
 				output("CONFLICT (rename/delete): Rename %s->%s in %s "
 				       "and deleted in %s",
 				       ren1->src, ren1->dst, branchName1,
 				       branchName2);
 				cleanMerge = 0;
-				updateFile(fp, 0, ren1->dst_sha, ren1->dst_mode, ren1->dst);
+				updateFile(0, ren1->dst_sha, ren1->dst_mode, ren1->dst);
 			} else if ( memcmp(dstShaOtherBranch, null_sha1, 20) != 0 ) {
 				newPath = uniquePath(ren1->dst, branchName2);
 				output("CONFLICT (rename/add): Rename %s->%s in %s. "
@@ -1139,7 +1249,7 @@ int processRenames(struct rename_entry *
 				       ren1->src, ren1->dst, branchName1,
 				       ren1->dst, branchName2);
 				output("Adding as %s instead", newPath);
-				updateFile(fp, 0, dstShaOtherBranch, dstModeOtherBranch, newPath);
+				updateFile(0, dstShaOtherBranch, dstModeOtherBranch, newPath);
 				cleanMerge = 0;
 				tryMerge = 1;
 			} else if ( (dst2 = find_rename_bydst(renames2, ren1->dst)) ) {
@@ -1151,9 +1261,9 @@ int processRenames(struct rename_entry *
 				       dst2->src, dst2->dst, branchName2);
 				output("Renaming %s to %s and %s to %s instead",
 				       ren1->src, newPath1, dst2->src, newPath2);
-				removeFile(fp, 0, ren1->dst);
-				updateFile(fp, 0, ren1->dst_sha, ren1->dst_mode, newPath1);
-				updateFile(fp, 0, dst2->dst_sha, dst2->dst_mode, newPath2);
+				removeFile(0, ren1->dst);
+				updateFile(0, ren1->dst_sha, ren1->dst_mode, newPath1);
+				updateFile(0, dst2->dst_sha, dst2->dst_mode, newPath2);
 				dst2->processed = 1;
 				cleanMerge = 0;
 			} else
@@ -1194,21 +1304,19 @@ int processRenames(struct rename_entry *
 					cleanMerge = 0;
 
 					if ( !index_only )
-						setIndexStages(fp,
-							       ren1->dst,
+						setIndexStages(ren1->dst,
 							       osha, omode,
 							       asha, amode,
 							       bsha, bmode,
 							       1 /* clear */);
 				}
-				updateFile(fp, mfi.clean, mfi.sha, mfi.mode, ren1->dst);
+				updateFile(mfi.clean, mfi.sha, mfi.mode, ren1->dst);
 			}
 		}
 	}
 	path_list_clear(&srcNames, 0);
-	if (pclose(fp)) {
-		die("git update-index --index-info failed");
-	}
+	if (cache_dirty)
+		flush_cache();
 	return cleanMerge;
 }
 
@@ -1241,7 +1349,6 @@ int processEntry(struct index_entry *ent
 	unsigned oMode = entry->stages[1].mode;
 	unsigned aMode = entry->stages[2].mode;
 	unsigned bMode = entry->stages[3].mode;
-	FILE *fp = git_update_index_pipe();
 
 	if ( oSha && (!aSha || !bSha) ) {
 		//
@@ -1253,7 +1360,7 @@ int processEntry(struct index_entry *ent
 			// Deleted in both or deleted in one and unchanged in the other
 			if ( aSha )
 				output("Removing %s", path);
-			removeFile(fp, 1, path);
+			removeFile(1, path);
 		} else {
 			// Deleted in one and changed in the other
 			cleanMerge = 0;
@@ -1262,13 +1369,13 @@ int processEntry(struct index_entry *ent
 				       "and modified in %s. Version %s of %s left in tree.",
 				       path, branch1Name,
 				       branch2Name, branch2Name, path);
-				updateFile(fp, 0, bSha, bMode, path);
+				updateFile(0, bSha, bMode, path);
 			} else {
 				output("CONFLICT (delete/modify): %s deleted in %s "
 				       "and modified in %s. Version %s of %s left in tree.",
 				       path, branch2Name,
 				       branch1Name, branch1Name, path);
-				updateFile(fp, 0, aSha, aMode, path);
+				updateFile(0, aSha, aMode, path);
 			}
 		}
 
@@ -1302,11 +1409,11 @@ int processEntry(struct index_entry *ent
 			output("CONFLICT (%s): There is a directory with name %s in %s. "
 			       "Adding %s as %s",
 			       conf, path, otherBranch, path, newPath);
-			removeFile(fp, 0, path);
-			updateFile(fp, 0, sha, mode, newPath);
+			removeFile(0, path);
+			updateFile(0, sha, mode, newPath);
 		} else {
 			output("Adding %s", path);
-			updateFile(fp, 1, sha, mode, path);
+			updateFile(1, sha, mode, path);
 		}
 	} else if ( !oSha && aSha && bSha ) {
 		//
@@ -1319,7 +1426,7 @@ int processEntry(struct index_entry *ent
 				       "but permissions conflict %06o->%06o",
 				       path, aMode, bMode);
 				output("CONFLICT: adding with permission: %06o", aMode);
-				updateFile(fp, 0, aSha, aMode, path);
+				updateFile(0, aSha, aMode, path);
 			} else {
 				// This case is handled by git-read-tree
 				assert(0 && "This case must be handled by git-read-tree");
@@ -1331,9 +1438,9 @@ int processEntry(struct index_entry *ent
 			output("CONFLICT (add/add): File %s added non-identically "
 			       "in both branches. Adding as %s and %s instead.",
 			       path, newPath1, newPath2);
-			removeFile(fp, 0, path);
-			updateFile(fp, 0, aSha, aMode, newPath1);
-			updateFile(fp, 0, bSha, bMode, newPath2);
+			removeFile(0, path);
+			updateFile(0, aSha, aMode, newPath1);
+			updateFile(0, bSha, bMode, newPath2);
 		}
 
 	} else if ( oSha && aSha && bSha ) {
@@ -1348,22 +1455,23 @@ int processEntry(struct index_entry *ent
 				branch1Name, branch2Name);
 
 		if ( mfi.clean )
-			updateFile(fp, 1, mfi.sha, mfi.mode, path);
+			updateFile(1, mfi.sha, mfi.mode, path);
 		else {
 			cleanMerge = 0;
 			output("CONFLICT (content): Merge conflict in %s", path);
 
 			if ( index_only )
-				updateFile(fp, 0, mfi.sha, mfi.mode, path);
+				updateFile(0, mfi.sha, mfi.mode, path);
 			else
-				updateFileExt(fp, mfi.sha, mfi.mode, path,
+				updateFileExt(mfi.sha, mfi.mode, path,
 					      0 /* updateCache */, 1 /* updateWd */);
 		}
 	} else
 		die("Fatal merge failure, shouldn't happen.");
 
-	if (pclose(fp))
-		die("updating entry failed in git update-index");
+	if (cache_dirty)
+		flush_cache();
+
 	return cleanMerge;
 }
 
@@ -1570,6 +1678,10 @@ int main(int argc, char *argv[])
 		struct graph *graph = graph_build(commits);
 		result = merge(h1, h2, branch1, branch2, graph, 0, NULL);
 	}
+
+	if (cache_dirty)
+		flush_cache();
+
 	return result.clean ? 0: 1;
 }
 
-- 
1.4.1.rc1.gb2d14