[Bug] Segfault in git status

[Bug] Segfault in git status

[Dane Jensen]

Hi all-

Somehow I've managed to create a repository that causes git status to  
segfault. To be honest, I'm not entirely sure how just yet. This comes  
after a weekend of splitting a repository into two separate  
repositories, filtering out a directory from one, and then creating  
grafts in those repositories to skip commits that now change nothing.

This was in OS X 10.5.2 with git compiled from cf5c51e. Falling back  
to 1.5.4.2 cleared it up, so with the help of git bisect I've narrowed  
the bug down to cf55870.

I'll spend some time this week trying to find the exact point in my  
filtering that git status went south and report back if I can  
replicate it in a more-controlled test.

-Dane

Re: [Bug] Segfault in git status

[Junio C Hamano]

Dane Jensen <careo@fastmail.fm> writes:

> This was in OS X 10.5.2 with git compiled from cf5c51e. Falling back
> to 1.5.4.2 cleared it up, so with the help of git bisect I've narrowed
> the bug down to cf55870.

Do you have this issue only on OSX?  Do you have the issue on
any commit, or a particular commit?

What does the change between HEAD, index and the work tree
involve in the commit that causes the problem?  Can you make
repository available to the public for diagnosis (if it is "Only
on OSX", I may not be able to help much with, but never hurt to
ask)?

Just to double check, can you see if reverting the commit (and
another commit that immediately follows) fixes the issue for
you?

A patch to do the revert for cf5c51e is attached.

---

 cache.h      |    7 ---
 dir.c        |    2 +-
 read-cache.c |  118 +++++-----------------------------------------------------
 3 files changed, 11 insertions(+), 116 deletions(-)

diff --git a/cache.h b/cache.h
index e1000bc..fd7716c 100644
--- a/cache.h
+++ b/cache.h
@@ -3,7 +3,6 @@
 
 #include "git-compat-util.h"
 #include "strbuf.h"
-#include "hash.h"
 
 #include SHA1_HEADER
 #include <zlib.h>
@@ -110,7 +109,6 @@ struct ondisk_cache_entry {
 };
 
 struct cache_entry {
-	struct cache_entry *next;
 	unsigned int ce_ctime;
 	unsigned int ce_mtime;
 	unsigned int ce_dev;
@@ -133,7 +131,6 @@ struct cache_entry {
 #define CE_UPDATE    (0x10000)
 #define CE_REMOVE    (0x20000)
 #define CE_UPTODATE  (0x40000)
-#define CE_UNHASHED  (0x80000)
 
 static inline unsigned create_ce_flags(size_t len, unsigned stage)
 {
@@ -203,8 +200,6 @@ struct index_state {
 	struct cache_tree *cache_tree;
 	time_t timestamp;
 	void *alloc;
-	unsigned name_hash_initialized : 1;
-	struct hash_table name_hash;
 };
 
 extern struct index_state the_index;
@@ -228,7 +223,6 @@ extern struct index_state the_index;
 #define refresh_cache(flags) refresh_index(&the_index, (flags), NULL, NULL)
 #define ce_match_stat(ce, st, options) ie_match_stat(&the_index, (ce), (st), (options))
 #define ce_modified(ce, st, options) ie_modified(&the_index, (ce), (st), (options))
-#define cache_name_exists(name, namelen) index_name_exists(&the_index, (name), (namelen))
 #endif
 
 enum object_type {
@@ -315,7 +309,6 @@ extern int read_index_from(struct index_state *, const char *path);
 extern int write_index(struct index_state *, int newfd);
 extern int discard_index(struct index_state *);
 extern int verify_path(const char *path);
-extern int index_name_exists(struct index_state *istate, const char *name, int namelen);
 extern int index_name_pos(struct index_state *, const char *name, int namelen);
 #define ADD_CACHE_OK_TO_ADD 1		/* Ok to add */
 #define ADD_CACHE_OK_TO_REPLACE 2	/* Ok to replace file/directory */
diff --git a/dir.c b/dir.c
index 1f507da..4d20871 100644
--- a/dir.c
+++ b/dir.c
@@ -371,7 +371,7 @@ static struct dir_entry *dir_entry_new(const char *pathname, int len)
 
 struct dir_entry *dir_add_name(struct dir_struct *dir, const char *pathname, int len)
 {
-	if (cache_name_exists(pathname, len))
+	if (cache_name_pos(pathname, len) >= 0)
 		return NULL;
 
 	ALLOC_GROW(dir->entries, dir->nr+1, dir->alloc);
diff --git a/read-cache.c b/read-cache.c
index e45f4b3..07abd5d 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -23,90 +23,6 @@
 
 struct index_state the_index;
 
-static unsigned int hash_name(const char *name, int namelen)
-{
-	unsigned int hash = 0x123;
-
-	do {
-		unsigned char c = *name++;
-		hash = hash*101 + c;
-	} while (--namelen);
-	return hash;
-}
-
-static void hash_index_entry(struct index_state *istate, struct cache_entry *ce)
-{
-	void **pos;
-	unsigned int hash = hash_name(ce->name, ce_namelen(ce));
-
-	pos = insert_hash(hash, ce, &istate->name_hash);
-	if (pos) {
-		ce->next = *pos;
-		*pos = ce;
-	}
-}
-
-static void lazy_init_name_hash(struct index_state *istate)
-{
-	int nr;
-
-	if (istate->name_hash_initialized)
-		return;
-	for (nr = 0; nr < istate->cache_nr; nr++)
-		hash_index_entry(istate, istate->cache[nr]);
-	istate->name_hash_initialized = 1;
-}
-
-static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
-{
-	istate->cache[nr] = ce;
-	if (istate->name_hash_initialized)
-		hash_index_entry(istate, ce);
-}
-
-/*
- * We don't actually *remove* it, we can just mark it invalid so that
- * we won't find it in lookups.
- *
- * Not only would we have to search the lists (simple enough), but
- * we'd also have to rehash other hash buckets in case this makes the
- * hash bucket empty (common). So it's much better to just mark
- * it.
- */
-static void remove_hash_entry(struct index_state *istate, struct cache_entry *ce)
-{
-	ce->ce_flags |= CE_UNHASHED;
-}
-
-static void replace_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
-{
-	struct cache_entry *old = istate->cache[nr];
-
-	if (ce != old) {
-		remove_hash_entry(istate, old);
-		set_index_entry(istate, nr, ce);
-	}
-	istate->cache_changed = 1;
-}
-
-int index_name_exists(struct index_state *istate, const char *name, int namelen)
-{
-	unsigned int hash = hash_name(name, namelen);
-	struct cache_entry *ce;
-
-	lazy_init_name_hash(istate);
-	ce = lookup_hash(hash, &istate->name_hash);
-
-	while (ce) {
-		if (!(ce->ce_flags & CE_UNHASHED)) {
-			if (!cache_name_compare(name, namelen, ce->name, ce->ce_flags))
-				return 1;
-		}
-		ce = ce->next;
-	}
-	return 0;
-}
-
 /*
  * This only updates the "non-critical" parts of the directory
  * cache, ie the parts that aren't tracked by GIT, and only used
@@ -411,9 +327,6 @@ int index_name_pos(struct index_state *istate, const char *name, int namelen)
 /* Remove entry, return true if there are more entries to go.. */
 int remove_index_entry_at(struct index_state *istate, int pos)
 {
-	struct cache_entry *ce = istate->cache[pos];
-
-	remove_hash_entry(istate, ce);
 	istate->cache_changed = 1;
 	istate->cache_nr--;
 	if (pos >= istate->cache_nr)
@@ -789,7 +702,8 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e
 
 	/* existing match? Just replace it. */
 	if (pos >= 0) {
-		replace_index_entry(istate, pos, ce);
+		istate->cache_changed = 1;
+		istate->cache[pos] = ce;
 		return 0;
 	}
 	pos = -pos-1;
@@ -849,7 +763,7 @@ int add_index_entry(struct index_state *istate, struct cache_entry *ce, int opti
 		memmove(istate->cache + pos + 1,
 			istate->cache + pos,
 			(istate->cache_nr - pos - 1) * sizeof(ce));
-	set_index_entry(istate, pos, ce);
+	istate->cache[pos] = ce;
 	istate->cache_changed = 1;
 	return 0;
 }
@@ -978,8 +892,11 @@ int refresh_index(struct index_state *istate, unsigned int flags, const char **p
 			has_errors = 1;
 			continue;
 		}
-
-		replace_index_entry(istate, i, new);
+		istate->cache_changed = 1;
+		/* You can NOT just free istate->cache[i] here, since it
+		 * might not be necessarily malloc()ed but can also come
+		 * from mmap(). */
+		istate->cache[i] = new;
 	}
 	return has_errors;
 }
@@ -1054,20 +971,6 @@ static void convert_from_disk(struct ondisk_cache_entry *ondisk, struct cache_en
 	memcpy(ce->name, ondisk->name, len + 1);
 }
 
-static inline size_t estimate_cache_size(size_t ondisk_size, unsigned int entries)
-{
-	long per_entry;
-
-	per_entry = sizeof(struct cache_entry) - sizeof(struct ondisk_cache_entry);
-
-	/*
-	 * Alignment can cause differences. This should be "alignof", but
-	 * since that's a gcc'ism, just use the size of a pointer.
-	 */
-	per_entry += sizeof(void *);
-	return ondisk_size + entries*per_entry;
-}
-
 /* remember to discard_cache() before reading a different cache! */
 int read_index_from(struct index_state *istate, const char *path)
 {
@@ -1118,7 +1021,7 @@ int read_index_from(struct index_state *istate, const char *path)
 	 * has room for a few  more flags, we can allocate using the same
 	 * index size
 	 */
-	istate->alloc = xmalloc(estimate_cache_size(mmap_size, istate->cache_nr));
+	istate->alloc = xmalloc(mmap_size);
 
 	src_offset = sizeof(*hdr);
 	dst_offset = 0;
@@ -1129,7 +1032,7 @@ int read_index_from(struct index_state *istate, const char *path)
 		disk_ce = (struct ondisk_cache_entry *)((char *)mmap + src_offset);
 		ce = (struct cache_entry *)((char *)istate->alloc + dst_offset);
 		convert_from_disk(disk_ce, ce);
-		set_index_entry(istate, i, ce);
+		istate->cache[i] = ce;
 
 		src_offset += ondisk_ce_size(ce);
 		dst_offset += ce_size(ce);
@@ -1167,7 +1070,6 @@ int discard_index(struct index_state *istate)
 	istate->cache_nr = 0;
 	istate->cache_changed = 0;
 	istate->timestamp = 0;
-	free_hash(&istate->name_hash);
 	cache_tree_free(&(istate->cache_tree));
 	free(istate->alloc);
 	istate->alloc = NULL;

Re: [Bug] Segfault in git status

[Dane Jensen]

On Feb 18, 2008, at 5:26 PM, Junio C Hamano wrote:

> Dane Jensen <careo@fastmail.fm> writes:
>
>> This was in OS X 10.5.2 with git compiled from cf5c51e. Falling back
>> to 1.5.4.2 cleared it up, so with the help of git bisect I've  
>> narrowed
>> the bug down to cf55870.
>
> Do you have this issue only on OSX?  Do you have the issue on
> any commit, or a particular commit?

It's not with any particular commit. It occurs with a fresh clone or  
at any other point, regardless of any changes I've made, what branch  
I'm on, or anything else I could think of testing.

> What does the change between HEAD, index and the work tree
> involve in the commit that causes the problem?  Can you make
> repository available to the public for diagnosis (if it is "Only
> on OSX", I may not be able to help much with, but never hurt to
> ask)?

Unfortunately not. I'll try to see if I can't cook up a test case that  
reproduces it however.

> Just to double check, can you see if reverting the commit (and
> another commit that immediately follows) fixes the issue for
> you?
>
> A patch to do the revert for cf5c51e is attached.

The patch does indeed fix it.

Additionally, now that I see the output of a successful run right  
after a failed one, it appears that git-status is dying before listing  
untracked files. It will output the "Changed but not updated:"  
section, but then die before getting to "Untracked files:".

-Dane

Re: [Bug] Segfault in git status

[Jay Soffian]

On Mon, Feb 18, 2008 at 3:36 AM, Dane Jensen <careo@fastmail.fm> wrote:
> Hi all-
>
>  Somehow I've managed to create a repository that causes git status to
>  segfault. To be honest, I'm not entirely sure how just yet. This comes
>  after a weekend of splitting a repository into two separate
>  repositories, filtering out a directory from one, and then creating
>  grafts in those repositories to skip commits that now change nothing.
>
>  This was in OS X 10.5.2 with git compiled from cf5c51e. Falling back
>  to 1.5.4.2 cleared it up, so with the help of git bisect I've narrowed
>  the bug down to cf55870.
>
>  I'll spend some time this week trying to find the exact point in my
>  filtering that git status went south and report back if I can
>  replicate it in a more-controlled test.

BTW - you should have a crash log in ~/Library/Logs/CrashReporter and
that should list the function in which the crash occurred.

j.

Re: [Bug] Segfault in git status

[Pieter de Bie]

On Feb 18, 2008, at 9:36 AM, Dane Jensen wrote:

> Hi all-
>
> Somehow I've managed to create a repository that causes git status  
> to segfault. To be honest, I'm not entirely sure how just yet. This  
> comes after a weekend of splitting a repository into two separate  
> repositories, filtering out a directory from one, and then creating  
> grafts in those repositories to skip commits that now change nothing.

I have the same problem, also on OS X.

Vienna:Project pieter$ git --version
git version 1.5.4.2.263.g5729

which is 'next' + 2 patches

Vienna:Project pieter$ gdb --args git status
GNU gdb 6.3.50-20050815 (Apple version gdb-768) (Tue Oct  2 04:07:49  
UTC 2007)
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and  
you are
welcome to change it and/or distribute copies of it under certain  
conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for  
details.
This GDB was configured as "i386-apple-darwin"...Reading symbols for  
shared libraries ........ done

(gdb) r
Starting program: /opt/pieter/bin/git status
Reading symbols for shared libraries +++++++. done
# On branch master
# Changed but not updated:
#   (use "git add <file>..." to update what will be committed)
#
#	modified:   Data/Metadata/redirects.xml
#	modified:   protege/ontologie.owl
#	modified:   protege/ontologie.pprj
#	modified:   protege/ontologie.repository
#	modified:   protege/swirl_data.owl
#	modified:   test-po2s.db
#	modified:   test-so2p.db
#	modified:   test-sp2o.db
#

Program received signal EXC_BAD_ACCESS, Could not access memory.
Reason: KERN_INVALID_ADDRESS at address: 0x004a2010
index_name_exists (istate=0xeba80, name=0xbfffdc27 "Data/Cache/ 
Template/Peter_II_of_Yugoslavia.html", namelen=47) at read-cache.c:101
101			if (!(ce->ce_flags & CE_UNHASHED)) {
(gdb) print *ce
Cannot access memory at address 0x4a1fec

This happens in at least one repository, most others are just fine.

- Pieter

Re: [Bug] Segfault in git status

[Jeff King]

On Thu, Feb 21, 2008 at 05:01:10PM +0100, Pieter de Bie wrote:

> Program received signal EXC_BAD_ACCESS, Could not access memory.
> Reason: KERN_INVALID_ADDRESS at address: 0x004a2010
> index_name_exists (istate=0xeba80, name=0xbfffdc27 "Data/Cache/ 
> Template/Peter_II_of_Yugoslavia.html", namelen=47) at read-cache.c:101
> 101			if (!(ce->ce_flags & CE_UNHASHED)) {
> (gdb) print *ce
> Cannot access memory at address 0x4a1fec

This shows up using valgrind under Linux, too. I haven't had time to
track it down yet, though.

-Peff

Re: [Bug] Segfault in git status

[Jay Soffian]

On Thu, Feb 21, 2008 at 11:01 AM, Pieter de Bie <pdebie@ai.rug.nl> wrote:
>  Vienna:Project pieter$ gdb --args git status
>  (gdb) r
>  Program received signal EXC_BAD_ACCESS, Could not access memory.
>  Reason: KERN_INVALID_ADDRESS at address: 0x004a2010
>  index_name_exists (istate=0xeba80, name=0xbfffdc27 "Data/Cache/
>  Template/Peter_II_of_Yugoslavia.html", namelen=47) at read-cache.c:101
>  101                     if (!(ce->ce_flags & CE_UNHASHED)) {
>  (gdb) print *ce
>  Cannot access memory at address 0x4a1fec

Do that again, but type "bt" at the gdb prompt after the crash please. :-)

j.

Re: [Bug] Segfault in git status

[Pieter de Bie]

On Feb 21, 2008, at 5:53 PM, Jay Soffian wrote:
>>
> Do that again, but type "bt" at the gdb prompt after the crash  
> please. :-)
>
> j.

Program received signal EXC_BAD_ACCESS, Could not access memory.
Reason: KERN_INVALID_ADDRESS at address: 0x004a2010
index_name_exists (istate=0xeba80, name=0xbfffdc27 "Data/Cache/ 
Template/Peter_II_of_Yugoslavia.html", namelen=47) at read-cache.c:101
101			if (!(ce->ce_flags & CE_UNHASHED)) {
(gdb) backtrace
#0  index_name_exists (istate=0xeba80, name=0xbfffdc27 "Data/Cache/ 
Template/Peter_II_of_Yugoslavia.html", namelen=47) at read-cache.c:101
#1  0x0006fc04 in dir_add_name (dir=0xbfffef68, pathname=0xbfffdc27  
"Data/Cache/Template/Peter_II_of_Yugoslavia.html", len=47) at dir.c:374
#2  0x00070d11 in read_directory_recursive (dir=0xbfffef68,  
path=<value temporarily unavailable, due to optimizations>,  
base=<value temporarily unavailable, due to optimizations>,  
baselen=20, check_only=0, simplify=0x8072cc) at dir.c:642
#3  0x00070baf in read_directory_recursive (dir=0xbfffef68,  
path=<value temporarily unavailable, due to optimizations>,  
base=<value temporarily unavailable, due to optimizations>,  
baselen=11, check_only=0, simplify=<value temporarily unavailable, due  
to optimizations>) at dir.c:627
#4  0x00070baf in read_directory_recursive (dir=0xbfffef68,  
path=<value temporarily unavailable, due to optimizations>,  
base=<value temporarily unavailable, due to optimizations>, baselen=5,  
check_only=0, simplify=<value temporarily unavailable, due to  
optimizations>) at dir.c:627
#5  0x00070baf in read_directory_recursive (dir=0xbfffef68,  
path=<value temporarily unavailable, due to optimizations>,  
base=<value temporarily unavailable, due to optimizations>, baselen=0,  
check_only=0, simplify=<value temporarily unavailable, due to  
optimizations>) at dir.c:627
#6  0x00070ed3 in read_directory (dir=0xbfffef68, path=0xc446c ".",  
base=0xbb080 "", baselen=4857836, pathspec=0x0) at dir.c:715
#7  0x0009d4d6 in wt_status_print (s=0xbffff5ac) at wt-status.c:282
#8  0x000167c3 in run_status (fp=0xa015f5f8, index_file=0x500300 "/ 
Users/pieter/Documents/Studie/2007-2008/AWT/Project/.git/index",  
prefix=0x0, nowarn=4857836) at builtin-commit.c:347
#9  0x000171af in cmd_status (argc=1, argv=0x0, prefix=0x500300 "/ 
Users/pieter/Documents/Studie/2007-2008/AWT/Project/.git/index") at  
builtin-commit.c:783
#10 0x00002260 in handle_internal_command (argc=1, argv=0xbffff7b8) at  
git.c:259
#11 0x0000246f in main (argc=1, argv=0xbffff7b8) at git.c:451


There ya go

- Pieter

Re: [Bug] Segfault in git status

[Dane Jensen]

On Feb 21, 2008, at 8:53 AM, Jay Soffian wrote:

> On Thu, Feb 21, 2008 at 11:01 AM, Pieter de Bie <pdebie@ai.rug.nl>  
> wrote:
>> Vienna:Project pieter$ gdb --args git status
>> (gdb) r
>> Program received signal EXC_BAD_ACCESS, Could not access memory.
>> Reason: KERN_INVALID_ADDRESS at address: 0x004a2010
>> index_name_exists (istate=0xeba80, name=0xbfffdc27 "Data/Cache/
>> Template/Peter_II_of_Yugoslavia.html", namelen=47) at read-cache.c: 
>> 101
>> 101                     if (!(ce->ce_flags & CE_UNHASHED)) {
>> (gdb) print *ce
>> Cannot access memory at address 0x4a1fec
>
> Do that again, but type "bt" at the gdb prompt after the crash  
> please. :-)


Same error as mine. Here's the result against my repository. This is  
with master, git --version => 1.5.4.2.184.gb23b,

(gdb) r
Starting program: /Users/djensen/bin/git status
Reading symbols for shared libraries +++++++. done
# On branch remote/device-emulator

Program received signal EXC_BAD_ACCESS, Could not access memory.
Reason: KERN_INVALID_ADDRESS at address: 0x00431018
index_name_exists (istate=0xe04a0, name=0xbfffdc87 "shared/models/spec/ 
device_spec.rb", namelen=33) at read-cache.c:101
101			if (!(ce->ce_flags & CE_UNHASHED)) {
(gdb) bt
#0  index_name_exists (istate=0xe04a0, name=0xbfffdc87 "shared/models/ 
spec/device_spec.rb", namelen=33) at read-cache.c:101
#1  0x00068624 in dir_add_name (dir=0xbfffefc0, pathname=0xbfffdc87  
"shared/models/spec/device_spec.rb", len=33) at dir.c:374
#2  0x00069731 in read_directory_recursive (dir=0xbfffefc0,  
path=<value temporarily unavailable, due to optimizations>,  
base=<value temporarily unavailable, due to optimizations>,  
baselen=19, check_only=0, simplify=0x81de38) at dir.c:642
#3  0x000695cf in read_directory_recursive (dir=0xbfffefc0,  
path=<value temporarily unavailable, due to optimizations>,  
base=<value temporarily unavailable, due to optimizations>,  
baselen=14, check_only=0, simplify=<value temporarily unavailable, due  
to optimizations>) at dir.c:627
#4  0x000695cf in read_directory_recursive (dir=0xbfffefc0,  
path=<value temporarily unavailable, due to optimizations>,  
base=<value temporarily unavailable, due to optimizations>, baselen=7,  
check_only=0, simplify=<value temporarily unavailable, due to  
optimizations>) at dir.c:627
#5  0x000695cf in read_directory_recursive (dir=0xbfffefc0,  
path=<value temporarily unavailable, due to optimizations>,  
base=<value temporarily unavailable, due to optimizations>, baselen=0,  
check_only=0, simplify=<value temporarily unavailable, due to  
optimizations>) at dir.c:627
#6  0x000698f3 in read_directory (dir=0xbfffefc0, path=0xae8a0 ".",  
base=0xae19c "", baselen=4394996, pathspec=0x0) at dir.c:715
#7  0x00094746 in wt_status_print (s=0xbffff5ec) at wt-status.c:282
#8  0x00014423 in run_status (fp=0xa01a65f8, index_file=0x5003b0 "/ 
Users/djensen/Projects/Soocial/source/.git/index", prefix=0x0,  
nowarn=4394996) at builtin-commit.c:346
#9  0x00014e0f in cmd_status (argc=1, argv=0x0, prefix=0x5003b0 "/ 
Users/djensen/Projects/Soocial/source/.git/index") at builtin-commit.c: 
782
#10 0x00001da0 in handle_internal_command (argc=1, argv=0xbffff7f8) at  
git.c:259
#11 0x00001faf in main (argc=1, argv=0xbffff7f8) at git.c:449
(gdb)