[PATCH/RFC v1 0/2] The ext4 filesystem and racy git

[PATCH/RFC v1 0/2] The ext4 filesystem and racy git

[Kjetil Barvik]

Ext4 is marked stable in Linux v2.6.28, and I have done a very simple
test on one disk-partition of my slow laptop, and the numbers look
very promising.

With default created ext3 disk-partition on my harddisk the 'git
checkout -q my-v2.6.25/27' test takes around 20 seconds real time for
the best cases.  The same test run on a ext4 partition takes around 14
seconds for the best cases.

And, since ext4 supports nanosecond timestamps, when I added patch
2/2, the checkout time is much more stable and more close to 14-15
seconds most of the time.

Conclusion: for GIT on my laptop the ext4 filesystem is a speedup!

>From '/proc/mounts' here is the mount options:

  /dev/hda10 /home ext3 rw,noatime,errors=continue,data=ordered 0 0
  /dev/hda12 /opt2 ext4 rw,noatime,barrier=1,journal_async_commit,data=ordered 0 0

/dev/hda10 is formatted with default (gentoo) ext3 parameters, and the
/dev/hda12 is formatted with:

  /sbin/mkfs -t ext4 -I 256 -G 64 -Oflex_bg,uninit_bg /dev/hda12


Kjetil Barvik (2):
  fix compile error when USE_NSEC is defined
  make USE_NSEC work as expected

 builtin-fetch-pack.c |    4 +-
 cache.h              |    6 ++--
 read-cache.c         |   70 ++++++++++++++++++++++++++++++++++++++++----------
 3 files changed, 61 insertions(+), 19 deletions(-)

[PATCH/RFC v1 1/2] fix compile error when USE_NSEC is defined

[Kjetil Barvik]

'struct cache' does not have a 'usec' member, but a 'unsigned int
nsec' member.  Simmilar 'struct stat' does not have a 'st_mtim.usec'
member, and we should instead use 'st_mtim.tv_nsec'.

Signed-off-by: Kjetil Barvik <barvik@broadpark.no>
---
 builtin-fetch-pack.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/builtin-fetch-pack.c b/builtin-fetch-pack.c
index 67fb80e..3b210c7 100644
--- a/builtin-fetch-pack.c
+++ b/builtin-fetch-pack.c
@@ -802,14 +802,14 @@ struct ref *fetch_pack(struct fetch_pack_args *my_args,
 
 		mtime.sec = st.st_mtime;
 #ifdef USE_NSEC
-		mtime.usec = st.st_mtim.usec;
+		mtime.nsec = st.st_mtim.tv_nsec;
 #endif
 		if (stat(shallow, &st)) {
 			if (mtime.sec)
 				die("shallow file was removed during fetch");
 		} else if (st.st_mtime != mtime.sec
 #ifdef USE_NSEC
-				|| st.st_mtim.usec != mtime.usec
+				|| st.st_mtim.tv_nsec != mtime.nsec
 #endif
 			  )
 			die("shallow file was changed during fetch");
-- 
1.6.1.349.g99fa5

[PATCH/RFC v1 2/2] make USE_NSEC work as expected

[Kjetil Barvik]

Since the filesystem ext4 is now defined as stable in Linux v2.6.28,
and ext4 supports nanonsecond resolution timestamps natively, it is
time to make USE_NSEC work as expected.

This will make racy git situations less likely to happen.  For 'git
checkout' this means it will be less likely that we have to open, read
the contents of the file into RAM, and check if file is really
modified or not.  The result sould be a litle less used CPU time, less
pagefaults and a litle faster program, at least for 'git checkout'.

Since the number of possible racy git situations would increase when
disks gets faster, this patch would be more and more helpfull as times
go by.  For a fast Solid State Disk, this patch should be helpfull.

Note that, when file operations starts to take less than 1 nanosecond,
one would again start to get more racy git situations.

For more info on racy git, see Documentation/technical/racy-git.txt
For more info on ext4, see http://kernelnewbies.org/Ext4

Signed-off-by: Kjetil Barvik <barvik@broadpark.no>
---
 cache.h      |    6 ++--
 read-cache.c |   70 ++++++++++++++++++++++++++++++++++++++++++++++-----------
 2 files changed, 59 insertions(+), 17 deletions(-)

diff --git a/cache.h b/cache.h
index 37dfb1c..309053d 100644
--- a/cache.h
+++ b/cache.h
@@ -140,8 +140,8 @@ struct ondisk_cache_entry_extended {
 };
 
 struct cache_entry {
-	unsigned int ce_ctime;
-	unsigned int ce_mtime;
+	struct cache_time ce_ctime;
+	struct cache_time ce_mtime;
 	unsigned int ce_dev;
 	unsigned int ce_ino;
 	unsigned int ce_mode;
@@ -282,7 +282,7 @@ struct index_state {
 	struct cache_entry **cache;
 	unsigned int cache_nr, cache_alloc, cache_changed;
 	struct cache_tree *cache_tree;
-	time_t timestamp;
+	struct cache_time timestamp;
 	void *alloc;
 	unsigned name_hash_initialized : 1,
 		 initialized : 1;
diff --git a/read-cache.c b/read-cache.c
index 940ec76..ca4bec2 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -67,8 +67,15 @@ void rename_index_entry_at(struct index_state *istate, int nr, const char *new_n
  */
 void fill_stat_cache_info(struct cache_entry *ce, struct stat *st)
 {
-	ce->ce_ctime = st->st_ctime;
-	ce->ce_mtime = st->st_mtime;
+	ce->ce_ctime.sec = (unsigned int)st->st_ctime;
+	ce->ce_mtime.sec = (unsigned int)st->st_mtime;
+#ifdef USE_NSEC
+	ce->ce_ctime.nsec = (unsigned int)st->st_ctim.tv_nsec;
+	ce->ce_mtime.nsec = (unsigned int)st->st_mtim.tv_nsec;
+#else
+	ce->ce_ctime.nsec = 0;
+	ce->ce_mtime.nsec = 0;
+#endif
 	ce->ce_dev = st->st_dev;
 	ce->ce_ino = st->st_ino;
 	ce->ce_uid = st->st_uid;
@@ -196,11 +203,18 @@ static int ce_match_stat_basic(struct cache_entry *ce, struct stat *st)
 	default:
 		die("internal error: ce_mode is %o", ce->ce_mode);
 	}
-	if (ce->ce_mtime != (unsigned int) st->st_mtime)
+	if (ce->ce_mtime.sec != (unsigned int)st->st_mtime)
 		changed |= MTIME_CHANGED;
-	if (trust_ctime && ce->ce_ctime != (unsigned int) st->st_ctime)
+	if (trust_ctime && ce->ce_ctime.sec != (unsigned int)st->st_ctime)
 		changed |= CTIME_CHANGED;
 
+#ifdef USE_NSEC
+	if (ce->ce_mtime.nsec != (unsigned int)st->st_mtim.tv_nsec)
+		changed |= MTIME_CHANGED;
+	if (trust_ctime && ce->ce_ctime.nsec != (unsigned int)st->st_ctim.tv_nsec)
+		changed |= CTIME_CHANGED;
+#endif
+
 	if (ce->ce_uid != (unsigned int) st->st_uid ||
 	    ce->ce_gid != (unsigned int) st->st_gid)
 		changed |= OWNER_CHANGED;
@@ -232,8 +246,16 @@ static int ce_match_stat_basic(struct cache_entry *ce, struct stat *st)
 static int is_racy_timestamp(const struct index_state *istate, struct cache_entry *ce)
 {
 	return (!S_ISGITLINK(ce->ce_mode) &&
-		istate->timestamp &&
-		((unsigned int)istate->timestamp) <= ce->ce_mtime);
+		istate->timestamp.sec &&
+#ifdef USE_NSEC
+		 /* nanosecond timestamped files can also be racy! */
+		(istate->timestamp.sec < ce->ce_mtime.sec ||
+		 (istate->timestamp.sec == ce->ce_mtime.sec &&
+		  istate->timestamp.nsec <= ce->ce_mtime.nsec))
+#else
+		istate->timestamp.sec <= ce->ce_mtime.sec
+#endif
+		 );
 }
 
 int ie_match_stat(const struct index_state *istate,
@@ -1139,8 +1161,15 @@ static void convert_from_disk(struct ondisk_cache_entry *ondisk, struct cache_en
 	size_t len;
 	const char *name;
 
-	ce->ce_ctime = ntohl(ondisk->ctime.sec);
-	ce->ce_mtime = ntohl(ondisk->mtime.sec);
+	ce->ce_ctime.sec = ntohl(ondisk->ctime.sec);
+	ce->ce_mtime.sec = ntohl(ondisk->mtime.sec);
+#ifdef USE_NSEC
+	ce->ce_ctime.nsec = ntohl(ondisk->ctime.nsec);
+	ce->ce_mtime.nsec = ntohl(ondisk->mtime.nsec);
+#else
+	ce->ce_ctime.nsec = 0;
+	ce->ce_mtime.nsec = 0;
+#endif
 	ce->ce_dev   = ntohl(ondisk->dev);
 	ce->ce_ino   = ntohl(ondisk->ino);
 	ce->ce_mode  = ntohl(ondisk->mode);
@@ -1206,7 +1235,8 @@ int read_index_from(struct index_state *istate, const char *path)
 		return istate->cache_nr;
 
 	errno = ENOENT;
-	istate->timestamp = 0;
+	istate->timestamp.sec = 0;
+	istate->timestamp.nsec = 0;
 	fd = open(path, O_RDONLY);
 	if (fd < 0) {
 		if (errno == ENOENT)
@@ -1258,7 +1288,13 @@ int read_index_from(struct index_state *istate, const char *path)
 		src_offset += ondisk_ce_size(ce);
 		dst_offset += ce_size(ce);
 	}
-	istate->timestamp = st.st_mtime;
+	istate->timestamp.sec = st.st_mtime;
+#ifdef USE_NSEC
+	istate->timestamp.nsec = (unsigned int)st.st_mtim.tv_nsec;
+#else
+	istate->timestamp.nsec = 0;
+#endif
+
 	while (src_offset <= mmap_size - 20 - 8) {
 		/* After an array of active_nr index entries,
 		 * there can be arbitrary number of extended
@@ -1288,14 +1324,15 @@ unmap:
 
 int is_index_unborn(struct index_state *istate)
 {
-	return (!istate->cache_nr && !istate->alloc && !istate->timestamp);
+	return (!istate->cache_nr && !istate->alloc && !istate->timestamp.sec);
 }
 
 int discard_index(struct index_state *istate)
 {
 	istate->cache_nr = 0;
 	istate->cache_changed = 0;
-	istate->timestamp = 0;
+	istate->timestamp.sec = 0;
+	istate->timestamp.nsec = 0;
 	istate->name_hash_initialized = 0;
 	free_hash(&istate->name_hash);
 	cache_tree_free(&(istate->cache_tree));
@@ -1441,10 +1478,15 @@ static int ce_write_entry(git_SHA_CTX *c, int fd, struct cache_entry *ce)
 	struct ondisk_cache_entry *ondisk = xcalloc(1, size);
 	char *name;
 
-	ondisk->ctime.sec = htonl(ce->ce_ctime);
+	ondisk->ctime.sec = htonl(ce->ce_ctime.sec);
+	ondisk->mtime.sec = htonl(ce->ce_mtime.sec);
+#ifdef USE_NSEC
+	ondisk->ctime.nsec = htonl(ce->ce_ctime.nsec);
+	ondisk->mtime.nsec = htonl(ce->ce_mtime.nsec);
+#else
 	ondisk->ctime.nsec = 0;
-	ondisk->mtime.sec = htonl(ce->ce_mtime);
 	ondisk->mtime.nsec = 0;
+#endif
 	ondisk->dev  = htonl(ce->ce_dev);
 	ondisk->ino  = htonl(ce->ce_ino);
 	ondisk->mode = htonl(ce->ce_mode);
-- 
1.6.1.349.g99fa5