* [PATCH/RFC v4 01/13] Move index v2 specific functions to their own file
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 02/13] read-cache.c: Re-read index if index file changed Thomas Gummerer
` (11 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Move index version 2 specific functions to their own file,
to prepare for the addition of a new index file format. With
the split into two files we have the non-index specific
functions in read-cache.c and the index-v2 specific functions
in read-cache-v2.c
Helped-by: Nguyen Thai Ngoc Duy <pclouds@gmail.com>
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
Makefile | 2 +
cache.h | 13 +-
read-cache-v2.c | 581 ++++++++++++++++++++++++++++++++++++++++++++++++
read-cache.c | 612 +++------------------------------------------------
read-cache.h | 57 +++++
test-index-version.c | 7 +-
6 files changed, 682 insertions(+), 590 deletions(-)
create mode 100644 read-cache-v2.c
create mode 100644 read-cache.h
diff --git a/Makefile b/Makefile
index 6b0c961..3ccd3a8 100644
--- a/Makefile
+++ b/Makefile
@@ -645,6 +645,7 @@ LIB_H += progress.h
LIB_H += prompt.h
LIB_H += quote.h
LIB_H += reachable.h
+LIB_H += read-cache.h
LIB_H += reflog-walk.h
LIB_H += refs.h
LIB_H += remote.h
@@ -768,6 +769,7 @@ LIB_OBJS += prompt.o
LIB_OBJS += quote.o
LIB_OBJS += reachable.o
LIB_OBJS += read-cache.o
+LIB_OBJS += read-cache-v2.o
LIB_OBJS += reflog-walk.o
LIB_OBJS += refs.o
LIB_OBJS += remote.o
diff --git a/cache.h b/cache.h
index 67f28b4..c77cdbe 100644
--- a/cache.h
+++ b/cache.h
@@ -94,16 +94,8 @@ unsigned long git_deflate_bound(git_zstream *, unsigned long);
*/
#define DEFAULT_GIT_PORT 9418
-/*
- * Basic data structures for the directory cache
- */
#define CACHE_SIGNATURE 0x44495243 /* "DIRC" */
-struct cache_header {
- unsigned int hdr_signature;
- unsigned int hdr_version;
- unsigned int hdr_entries;
-};
#define INDEX_FORMAT_LB 2
#define INDEX_FORMAT_UB 4
@@ -267,6 +259,7 @@ struct index_state {
unsigned name_hash_initialized : 1,
initialized : 1;
struct hash_table name_hash;
+ struct index_ops *ops;
};
extern struct index_state the_index;
@@ -471,8 +464,8 @@ extern int index_name_is_other(const struct index_state *, const char *, int);
#define CE_MATCH_RACY_IS_DIRTY 02
/* do stat comparison even if CE_SKIP_WORKTREE is true */
#define CE_MATCH_IGNORE_SKIP_WORKTREE 04
-extern int ie_match_stat(const struct index_state *, struct cache_entry *, struct stat *, unsigned int);
-extern int ie_modified(const struct index_state *, struct cache_entry *, struct stat *, unsigned int);
+extern int ie_match_stat(struct index_state *, struct cache_entry *, struct stat *, unsigned int);
+extern int ie_modified(struct index_state *, struct cache_entry *, struct stat *, unsigned int);
struct pathspec {
const char **raw; /* get_pathspec() result, not freed by free_pathspec() */
diff --git a/read-cache-v2.c b/read-cache-v2.c
new file mode 100644
index 0000000..2c5e78b
--- /dev/null
+++ b/read-cache-v2.c
@@ -0,0 +1,581 @@
+#include "cache.h"
+#include "read-cache.h"
+#include "resolve-undo.h"
+#include "cache-tree.h"
+#include "varint.h"
+
+/* Mask for the name length in ce_flags in the on-disk index */
+#define CE_NAMEMASK (0x0fff)
+
+struct cache_header {
+ unsigned int hdr_entries;
+};
+
+/*****************************************************************
+ * Index File I/O
+ *****************************************************************/
+
+/*
+ * dev/ino/uid/gid/size are also just tracked to the low 32 bits
+ * Again - this is just a (very strong in practice) heuristic that
+ * the inode hasn't changed.
+ *
+ * We save the fields in big-endian order to allow using the
+ * index file over NFS transparently.
+ */
+struct ondisk_cache_entry {
+ struct cache_time ctime;
+ struct cache_time mtime;
+ unsigned int dev;
+ unsigned int ino;
+ unsigned int mode;
+ unsigned int uid;
+ unsigned int gid;
+ unsigned int size;
+ unsigned char sha1[20];
+ unsigned short flags;
+ char name[FLEX_ARRAY]; /* more */
+};
+
+/*
+ * This struct is used when CE_EXTENDED bit is 1
+ * The struct must match ondisk_cache_entry exactly from
+ * ctime till flags
+ */
+struct ondisk_cache_entry_extended {
+ struct cache_time ctime;
+ struct cache_time mtime;
+ unsigned int dev;
+ unsigned int ino;
+ unsigned int mode;
+ unsigned int uid;
+ unsigned int gid;
+ unsigned int size;
+ unsigned char sha1[20];
+ unsigned short flags;
+ unsigned short flags2;
+ char name[FLEX_ARRAY]; /* more */
+};
+
+/* These are only used for v3 or lower */
+#define align_flex_name(STRUCT,len) ((offsetof(struct STRUCT,name) + (len) + 8) & ~7)
+#define ondisk_cache_entry_size(len) align_flex_name(ondisk_cache_entry,len)
+#define ondisk_cache_entry_extended_size(len) align_flex_name(ondisk_cache_entry_extended,len)
+#define ondisk_ce_size(ce) (((ce)->ce_flags & CE_EXTENDED) ? \
+ ondisk_cache_entry_extended_size(ce_namelen(ce)) : \
+ ondisk_cache_entry_size(ce_namelen(ce)))
+
+static int verify_hdr(void *mmap, unsigned long size)
+{
+ git_SHA_CTX c;
+ unsigned char sha1[20];
+
+ if (size < sizeof(struct cache_version_header)
+ + sizeof(struct cache_header) + 20)
+ die("index file smaller than expected");
+
+ git_SHA1_Init(&c);
+ git_SHA1_Update(&c, mmap, size - 20);
+ git_SHA1_Final(sha1, &c);
+ if (hashcmp(sha1, (unsigned char *)mmap + size - 20))
+ return error("bad index file sha1 signature");
+ return 0;
+}
+
+static int match_stat_basic(struct cache_entry *ce,
+ struct stat *st,
+ int changed)
+{
+ if (ce->ce_mtime.sec != (unsigned int)st->st_mtime)
+ changed |= MTIME_CHANGED;
+ if (trust_ctime && ce->ce_ctime.sec != (unsigned int)st->st_ctime)
+ changed |= CTIME_CHANGED;
+
+#ifdef USE_NSEC
+ if (ce->ce_mtime.nsec != ST_MTIME_NSEC(*st))
+ changed |= MTIME_CHANGED;
+ if (trust_ctime && ce->ce_ctime.nsec != ST_CTIME_NSEC(*st))
+ changed |= CTIME_CHANGED;
+#endif
+
+ if (ce->ce_uid != (unsigned int) st->st_uid ||
+ ce->ce_gid != (unsigned int) st->st_gid)
+ changed |= OWNER_CHANGED;
+ if (ce->ce_ino != (unsigned int) st->st_ino)
+ changed |= INODE_CHANGED;
+
+#ifdef USE_STDEV
+ /*
+ * st_dev breaks on network filesystems where different
+ * clients will have different views of what "device"
+ * the filesystem is on
+ */
+ if (ce->ce_dev != (unsigned int) st->st_dev)
+ changed |= INODE_CHANGED;
+#endif
+
+ if (ce->ce_size != (unsigned int) st->st_size)
+ changed |= DATA_CHANGED;
+
+ /* Racily smudged entry? */
+ if (!ce->ce_size) {
+ if (!is_empty_blob_sha1(ce->sha1))
+ changed |= DATA_CHANGED;
+ }
+
+ return changed;
+}
+
+static struct cache_entry *cache_entry_from_ondisk(struct ondisk_cache_entry *ondisk,
+ unsigned int flags,
+ const char *name,
+ size_t len)
+{
+ struct cache_entry *ce = xmalloc(cache_entry_size(len));
+
+ ce->ce_ctime.sec = ntoh_l(ondisk->ctime.sec);
+ ce->ce_mtime.sec = ntoh_l(ondisk->mtime.sec);
+ ce->ce_ctime.nsec = ntoh_l(ondisk->ctime.nsec);
+ ce->ce_mtime.nsec = ntoh_l(ondisk->mtime.nsec);
+ ce->ce_dev = ntoh_l(ondisk->dev);
+ ce->ce_ino = ntoh_l(ondisk->ino);
+ ce->ce_mode = ntoh_l(ondisk->mode);
+ ce->ce_uid = ntoh_l(ondisk->uid);
+ ce->ce_gid = ntoh_l(ondisk->gid);
+ ce->ce_size = ntoh_l(ondisk->size);
+ ce->ce_flags = flags & ~CE_NAMEMASK;
+ ce->ce_namelen = len;
+ hashcpy(ce->sha1, ondisk->sha1);
+ memcpy(ce->name, name, len);
+ ce->name[len] = '\0';
+ return ce;
+}
+
+/*
+ * Adjacent cache entries tend to share the leading paths, so it makes
+ * sense to only store the differences in later entries. In the v4
+ * on-disk format of the index, each on-disk cache entry stores the
+ * number of bytes to be stripped from the end of the previous name,
+ * and the bytes to append to the result, to come up with its name.
+ */
+static unsigned long expand_name_field(struct strbuf *name, const char *cp_)
+{
+ const unsigned char *ep, *cp = (const unsigned char *)cp_;
+ size_t len = decode_varint(&cp);
+
+ if (name->len < len)
+ die("malformed name field in the index");
+ strbuf_remove(name, name->len - len, len);
+ for (ep = cp; *ep; ep++)
+ ; /* find the end */
+ strbuf_add(name, cp, ep - cp);
+ return (const char *)ep + 1 - cp_;
+}
+
+static struct cache_entry *create_from_disk(struct ondisk_cache_entry *ondisk,
+ unsigned long *ent_size,
+ struct strbuf *previous_name)
+{
+ struct cache_entry *ce;
+ size_t len;
+ const char *name;
+ unsigned int flags;
+
+ /* On-disk flags are just 16 bits */
+ flags = ntoh_s(ondisk->flags);
+ len = flags & CE_NAMEMASK;
+
+ if (flags & CE_EXTENDED) {
+ struct ondisk_cache_entry_extended *ondisk2;
+ int extended_flags;
+ ondisk2 = (struct ondisk_cache_entry_extended *)ondisk;
+ extended_flags = ntoh_s(ondisk2->flags2) << 16;
+ /* We do not yet understand any bit out of CE_EXTENDED_FLAGS */
+ if (extended_flags & ~CE_EXTENDED_FLAGS)
+ die("Unknown index entry format %08x", extended_flags);
+ flags |= extended_flags;
+ name = ondisk2->name;
+ }
+ else
+ name = ondisk->name;
+
+ if (!previous_name) {
+ /* v3 and earlier */
+ if (len == CE_NAMEMASK)
+ len = strlen(name);
+ ce = cache_entry_from_ondisk(ondisk, flags, name, len);
+
+ *ent_size = ondisk_ce_size(ce);
+ } else {
+ unsigned long consumed;
+ consumed = expand_name_field(previous_name, name);
+ ce = cache_entry_from_ondisk(ondisk, flags,
+ previous_name->buf,
+ previous_name->len);
+
+ *ent_size = (name - ((char *)ondisk)) + consumed;
+ }
+ return ce;
+}
+
+static int read_index_extension(struct index_state *istate,
+ const char *ext, void *data, unsigned long sz)
+{
+ switch (CACHE_EXT(ext)) {
+ case CACHE_EXT_TREE:
+ istate->cache_tree = cache_tree_read(data, sz);
+ break;
+ case CACHE_EXT_RESOLVE_UNDO:
+ istate->resolve_undo = resolve_undo_read(data, sz);
+ break;
+ default:
+ if (*ext < 'A' || 'Z' < *ext)
+ return error("index uses %.4s extension, which we do not understand",
+ ext);
+ fprintf(stderr, "ignoring %.4s extension\n", ext);
+ break;
+ }
+ return 0;
+}
+
+static int read_index_v2(struct index_state *istate, void *mmap, int mmap_size)
+{
+ int i;
+ unsigned long src_offset;
+ struct cache_version_header *hdr;
+ struct cache_header *hdr_v2;
+ struct strbuf previous_name_buf = STRBUF_INIT, *previous_name;
+
+ hdr = mmap;
+ hdr_v2 = (struct cache_header *)((char *)mmap + sizeof(*hdr));
+ istate->version = ntohl(hdr->hdr_version);
+ istate->cache_nr = ntohl(hdr_v2->hdr_entries);
+ istate->cache_alloc = alloc_nr(istate->cache_nr);
+ istate->cache = xcalloc(istate->cache_alloc, sizeof(struct cache_entry *));
+ istate->initialized = 1;
+
+ if (istate->version == 4)
+ previous_name = &previous_name_buf;
+ else
+ previous_name = NULL;
+
+ src_offset = sizeof(*hdr) + sizeof(*hdr_v2);
+ for (i = 0; i < istate->cache_nr; i++) {
+ struct ondisk_cache_entry *disk_ce;
+ struct cache_entry *ce;
+ unsigned long consumed;
+
+ disk_ce = (struct ondisk_cache_entry *)((char *)mmap + src_offset);
+ ce = create_from_disk(disk_ce, &consumed, previous_name);
+ set_index_entry(istate, i, ce);
+
+ src_offset += consumed;
+ }
+ strbuf_release(&previous_name_buf);
+
+ while (src_offset <= mmap_size - 20 - 8) {
+ /* After an array of active_nr index entries,
+ * there can be arbitrary number of extended
+ * sections, each of which is prefixed with
+ * extension name (4-byte) and section length
+ * in 4-byte network byte order.
+ */
+ uint32_t extsize;
+ memcpy(&extsize, (char *)mmap + src_offset + 4, 4);
+ extsize = ntohl(extsize);
+ if (read_index_extension(istate,
+ (const char *) mmap + src_offset,
+ (char *) mmap + src_offset + 8,
+ extsize) < 0)
+ goto unmap;
+ src_offset += 8;
+ src_offset += extsize;
+ }
+ return 0;
+unmap:
+ munmap(mmap, mmap_size);
+ die("index file corrupt");
+}
+
+#define WRITE_BUFFER_SIZE 8192
+static unsigned char write_buffer[WRITE_BUFFER_SIZE];
+static unsigned long write_buffer_len;
+
+static int ce_write_flush(git_SHA_CTX *context, int fd)
+{
+ unsigned int buffered = write_buffer_len;
+ if (buffered) {
+ git_SHA1_Update(context, write_buffer, buffered);
+ if (write_in_full(fd, write_buffer, buffered) != buffered)
+ return -1;
+ write_buffer_len = 0;
+ }
+ return 0;
+}
+
+static int ce_write(git_SHA_CTX *context, int fd, void *data, unsigned int len)
+{
+ while (len) {
+ unsigned int buffered = write_buffer_len;
+ unsigned int partial = WRITE_BUFFER_SIZE - buffered;
+ if (partial > len)
+ partial = len;
+ memcpy(write_buffer + buffered, data, partial);
+ buffered += partial;
+ if (buffered == WRITE_BUFFER_SIZE) {
+ write_buffer_len = buffered;
+ if (ce_write_flush(context, fd))
+ return -1;
+ buffered = 0;
+ }
+ write_buffer_len = buffered;
+ len -= partial;
+ data = (char *) data + partial;
+ }
+ return 0;
+}
+
+static int write_index_ext_header(git_SHA_CTX *context, int fd,
+ unsigned int ext, unsigned int sz)
+{
+ ext = htonl(ext);
+ sz = htonl(sz);
+ return ((ce_write(context, fd, &ext, 4) < 0) ||
+ (ce_write(context, fd, &sz, 4) < 0)) ? -1 : 0;
+}
+
+static int ce_flush(git_SHA_CTX *context, int fd)
+{
+ unsigned int left = write_buffer_len;
+
+ if (left) {
+ write_buffer_len = 0;
+ git_SHA1_Update(context, write_buffer, left);
+ }
+
+ /* Flush first if not enough space for SHA1 signature */
+ if (left + 20 > WRITE_BUFFER_SIZE) {
+ if (write_in_full(fd, write_buffer, left) != left)
+ return -1;
+ left = 0;
+ }
+
+ /* Append the SHA1 signature at the end */
+ git_SHA1_Final(write_buffer + left, context);
+ left += 20;
+ return (write_in_full(fd, write_buffer, left) != left) ? -1 : 0;
+}
+
+static void ce_smudge_racily_clean_entry(struct index_state *istate, struct cache_entry *ce)
+{
+ /*
+ * The only thing we care about in this function is to smudge the
+ * falsely clean entry due to touch-update-touch race, so we leave
+ * everything else as they are. We are called for entries whose
+ * ce_mtime match the index file mtime.
+ *
+ * Note that this actually does not do much for gitlinks, for
+ * which ce_match_stat_basic() always goes to the actual
+ * contents. The caller checks with is_racy_timestamp() which
+ * always says "no" for gitlinks, so we are not called for them ;-)
+ */
+ struct stat st;
+
+ if (lstat(ce->name, &st) < 0)
+ return;
+ if (ce_match_stat_basic(istate, ce, &st))
+ return;
+ if (ce_modified_check_fs(ce, &st)) {
+ /* This is "racily clean"; smudge it. Note that this
+ * is a tricky code. At first glance, it may appear
+ * that it can break with this sequence:
+ *
+ * $ echo xyzzy >frotz
+ * $ git-update-index --add frotz
+ * $ : >frotz
+ * $ sleep 3
+ * $ echo filfre >nitfol
+ * $ git-update-index --add nitfol
+ *
+ * but it does not. When the second update-index runs,
+ * it notices that the entry "frotz" has the same timestamp
+ * as index, and if we were to smudge it by resetting its
+ * size to zero here, then the object name recorded
+ * in index is the 6-byte file but the cached stat information
+ * becomes zero --- which would then match what we would
+ * obtain from the filesystem next time we stat("frotz").
+ *
+ * However, the second update-index, before calling
+ * this function, notices that the cached size is 6
+ * bytes and what is on the filesystem is an empty
+ * file, and never calls us, so the cached size information
+ * for "frotz" stays 6 which does not match the filesystem.
+ */
+ ce->ce_size = 0;
+ }
+}
+
+/* Copy miscellaneous fields but not the name */
+static char *copy_cache_entry_to_ondisk(struct ondisk_cache_entry *ondisk,
+ struct cache_entry *ce)
+{
+ short flags;
+
+ ondisk->ctime.sec = htonl(ce->ce_ctime.sec);
+ ondisk->mtime.sec = htonl(ce->ce_mtime.sec);
+ ondisk->ctime.nsec = htonl(ce->ce_ctime.nsec);
+ ondisk->mtime.nsec = htonl(ce->ce_mtime.nsec);
+ ondisk->dev = htonl(ce->ce_dev);
+ ondisk->ino = htonl(ce->ce_ino);
+ ondisk->mode = htonl(ce->ce_mode);
+ ondisk->uid = htonl(ce->ce_uid);
+ ondisk->gid = htonl(ce->ce_gid);
+ ondisk->size = htonl(ce->ce_size);
+ hashcpy(ondisk->sha1, ce->sha1);
+
+ flags = ce->ce_flags;
+ flags |= (ce_namelen(ce) >= CE_NAMEMASK ? CE_NAMEMASK : ce_namelen(ce));
+ ondisk->flags = htons(flags);
+ if (ce->ce_flags & CE_EXTENDED) {
+ struct ondisk_cache_entry_extended *ondisk2;
+ ondisk2 = (struct ondisk_cache_entry_extended *)ondisk;
+ ondisk2->flags2 = htons((ce->ce_flags & CE_EXTENDED_FLAGS) >> 16);
+ return ondisk2->name;
+ }
+ else {
+ return ondisk->name;
+ }
+}
+
+static int ce_write_entry(git_SHA_CTX *c, int fd, struct cache_entry *ce,
+ struct strbuf *previous_name)
+{
+ int size;
+ struct ondisk_cache_entry *ondisk;
+ char *name;
+ int result;
+
+ if (!previous_name) {
+ size = ondisk_ce_size(ce);
+ ondisk = xcalloc(1, size);
+ name = copy_cache_entry_to_ondisk(ondisk, ce);
+ memcpy(name, ce->name, ce_namelen(ce));
+ } else {
+ int common, to_remove, prefix_size;
+ unsigned char to_remove_vi[16];
+ for (common = 0;
+ (ce->name[common] &&
+ common < previous_name->len &&
+ ce->name[common] == previous_name->buf[common]);
+ common++)
+ ; /* still matching */
+ to_remove = previous_name->len - common;
+ prefix_size = encode_varint(to_remove, to_remove_vi);
+
+ if (ce->ce_flags & CE_EXTENDED)
+ size = offsetof(struct ondisk_cache_entry_extended, name);
+ else
+ size = offsetof(struct ondisk_cache_entry, name);
+ size += prefix_size + (ce_namelen(ce) - common + 1);
+
+ ondisk = xcalloc(1, size);
+ name = copy_cache_entry_to_ondisk(ondisk, ce);
+ memcpy(name, to_remove_vi, prefix_size);
+ memcpy(name + prefix_size, ce->name + common, ce_namelen(ce) - common);
+
+ strbuf_splice(previous_name, common, to_remove,
+ ce->name + common, ce_namelen(ce) - common);
+ }
+
+ result = ce_write(c, fd, ondisk, size);
+ free(ondisk);
+ return result;
+}
+
+static int write_index_v2(struct index_state *istate, int newfd)
+{
+ git_SHA_CTX c;
+ struct cache_version_header hdr;
+ struct cache_header hdr_v2;
+ int i, err, removed, extended, hdr_version;
+ struct cache_entry **cache = istate->cache;
+ int entries = istate->cache_nr;
+ struct stat st;
+ struct strbuf previous_name_buf = STRBUF_INIT, *previous_name;
+
+ for (i = removed = extended = 0; i < entries; i++) {
+ if (cache[i]->ce_flags & CE_REMOVE)
+ removed++;
+
+ /* reduce extended entries if possible */
+ cache[i]->ce_flags &= ~CE_EXTENDED;
+ if (cache[i]->ce_flags & CE_EXTENDED_FLAGS) {
+ extended++;
+ cache[i]->ce_flags |= CE_EXTENDED;
+ }
+ }
+
+ /* demote version 3 to version 2 when the latter suffices */
+ if (istate->version == 3 || istate->version == 2)
+ istate->version = extended ? 3 : 2;
+
+ hdr_version = istate->version;
+
+ hdr.hdr_signature = htonl(CACHE_SIGNATURE);
+ hdr.hdr_version = htonl(hdr_version);
+ hdr_v2.hdr_entries = htonl(entries - removed);
+
+ git_SHA1_Init(&c);
+ if (ce_write(&c, newfd, &hdr, sizeof(hdr)) < 0)
+ return -1;
+ if (ce_write(&c, newfd, &hdr_v2, sizeof(hdr_v2)) < 0)
+ return -1;
+
+ previous_name = (hdr_version == 4) ? &previous_name_buf : NULL;
+ for (i = 0; i < entries; i++) {
+ struct cache_entry *ce = cache[i];
+ if (ce->ce_flags & CE_REMOVE)
+ continue;
+ if (!ce_uptodate(ce) && is_racy_timestamp(istate, ce))
+ ce_smudge_racily_clean_entry(istate, ce);
+ if (ce_write_entry(&c, newfd, ce, previous_name) < 0)
+ return -1;
+ }
+ strbuf_release(&previous_name_buf);
+
+ /* Write extension data here */
+ if (istate->cache_tree) {
+ struct strbuf sb = STRBUF_INIT;
+
+ cache_tree_write(&sb, istate->cache_tree);
+ err = write_index_ext_header(&c, newfd, CACHE_EXT_TREE, sb.len) < 0
+ || ce_write(&c, newfd, sb.buf, sb.len) < 0;
+ strbuf_release(&sb);
+ if (err)
+ return -1;
+ }
+ if (istate->resolve_undo) {
+ struct strbuf sb = STRBUF_INIT;
+
+ resolve_undo_write(&sb, istate->resolve_undo);
+ err = write_index_ext_header(&c, newfd, CACHE_EXT_RESOLVE_UNDO,
+ sb.len) < 0
+ || ce_write(&c, newfd, sb.buf, sb.len) < 0;
+ strbuf_release(&sb);
+ if (err)
+ return -1;
+ }
+
+ if (ce_flush(&c, newfd) || fstat(newfd, &st))
+ return -1;
+ istate->timestamp.sec = (unsigned int)st.st_mtime;
+ istate->timestamp.nsec = ST_MTIME_NSEC(st);
+ return 0;
+}
+
+struct index_ops v2_ops = {
+ match_stat_basic,
+ verify_hdr,
+ read_index_v2,
+ write_index_v2
+};
diff --git a/read-cache.c b/read-cache.c
index 2f8159f..6a8b4b1 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -5,6 +5,7 @@
*/
#define NO_THE_INDEX_COMPATIBILITY_MACROS
#include "cache.h"
+#include "read-cache.h"
#include "cache-tree.h"
#include "refs.h"
#include "dir.h"
@@ -17,26 +18,9 @@
static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really);
-/* Mask for the name length in ce_flags in the on-disk index */
-
-#define CE_NAMEMASK (0x0fff)
-
-/* Index extensions.
- *
- * The first letter should be 'A'..'Z' for extensions that are not
- * necessary for a correct operation (i.e. optimization data).
- * When new extensions are added that _needs_ to be understood in
- * order to correctly interpret the index file, pick character that
- * is outside the range, to cause the reader to abort.
- */
-
-#define CACHE_EXT(s) ( (s[0]<<24)|(s[1]<<16)|(s[2]<<8)|(s[3]) )
-#define CACHE_EXT_TREE 0x54524545 /* "TREE" */
-#define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */
-
struct index_state the_index;
-static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
+void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
{
istate->cache[nr] = ce;
add_name_hash(istate, ce);
@@ -143,7 +127,7 @@ static int ce_compare_gitlink(struct cache_entry *ce)
return hashcmp(sha1, ce->sha1);
}
-static int ce_modified_check_fs(struct cache_entry *ce, struct stat *st)
+int ce_modified_check_fs(struct cache_entry *ce, struct stat *st)
{
switch (st->st_mode & S_IFMT) {
case S_IFREG:
@@ -163,7 +147,17 @@ static int ce_modified_check_fs(struct cache_entry *ce, struct stat *st)
return 0;
}
-static int ce_match_stat_basic(struct cache_entry *ce, struct stat *st)
+static void check_set_istate_ops(struct index_state *istate)
+{
+ if (!istate->version)
+ istate->version = INDEX_FORMAT_DEFAULT;
+ if (!istate->ops)
+ if (istate->version >= 2 && istate->version <=4)
+ istate->ops = &v2_ops;
+}
+
+int ce_match_stat_basic(struct index_state *istate,
+ struct cache_entry *ce, struct stat *st)
{
unsigned int changed = 0;
@@ -195,47 +189,13 @@ static int ce_match_stat_basic(struct cache_entry *ce, struct stat *st)
default:
die("internal error: ce_mode is %o", ce->ce_mode);
}
- if (ce->ce_mtime.sec != (unsigned int)st->st_mtime)
- changed |= MTIME_CHANGED;
- if (trust_ctime && ce->ce_ctime.sec != (unsigned int)st->st_ctime)
- changed |= CTIME_CHANGED;
-
-#ifdef USE_NSEC
- if (ce->ce_mtime.nsec != ST_MTIME_NSEC(*st))
- changed |= MTIME_CHANGED;
- if (trust_ctime && ce->ce_ctime.nsec != ST_CTIME_NSEC(*st))
- changed |= CTIME_CHANGED;
-#endif
-
- if (ce->ce_uid != (unsigned int) st->st_uid ||
- ce->ce_gid != (unsigned int) st->st_gid)
- changed |= OWNER_CHANGED;
- if (ce->ce_ino != (unsigned int) st->st_ino)
- changed |= INODE_CHANGED;
-
-#ifdef USE_STDEV
- /*
- * st_dev breaks on network filesystems where different
- * clients will have different views of what "device"
- * the filesystem is on
- */
- if (ce->ce_dev != (unsigned int) st->st_dev)
- changed |= INODE_CHANGED;
-#endif
-
- if (ce->ce_size != (unsigned int) st->st_size)
- changed |= DATA_CHANGED;
-
- /* Racily smudged entry? */
- if (!ce->ce_size) {
- if (!is_empty_blob_sha1(ce->sha1))
- changed |= DATA_CHANGED;
- }
+ check_set_istate_ops(istate);
+ changed = istate->ops->match_stat_basic(ce, st, changed);
return changed;
}
-static int is_racy_timestamp(const struct index_state *istate, struct cache_entry *ce)
+int is_racy_timestamp(const struct index_state *istate, struct cache_entry *ce)
{
return (!S_ISGITLINK(ce->ce_mode) &&
istate->timestamp.sec &&
@@ -250,7 +210,7 @@ static int is_racy_timestamp(const struct index_state *istate, struct cache_entr
);
}
-int ie_match_stat(const struct index_state *istate,
+int ie_match_stat(struct index_state *istate,
struct cache_entry *ce, struct stat *st,
unsigned int options)
{
@@ -278,7 +238,7 @@ int ie_match_stat(const struct index_state *istate,
if (ce->ce_flags & CE_INTENT_TO_ADD)
return DATA_CHANGED | TYPE_CHANGED | MODE_CHANGED;
- changed = ce_match_stat_basic(ce, st);
+ changed = ce_match_stat_basic(istate, ce, st);
/*
* Within 1 second of this sequence:
@@ -306,7 +266,7 @@ int ie_match_stat(const struct index_state *istate,
return changed;
}
-int ie_modified(const struct index_state *istate,
+int ie_modified(struct index_state *istate,
struct cache_entry *ce, struct stat *st, unsigned int options)
{
int changed, changed_fs;
@@ -1190,99 +1150,18 @@ static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int reall
return refresh_cache_ent(&the_index, ce, really, NULL, NULL);
}
-
-/*****************************************************************
- * Index File I/O
- *****************************************************************/
-
-#define INDEX_FORMAT_DEFAULT 3
-
-/*
- * dev/ino/uid/gid/size are also just tracked to the low 32 bits
- * Again - this is just a (very strong in practice) heuristic that
- * the inode hasn't changed.
- *
- * We save the fields in big-endian order to allow using the
- * index file over NFS transparently.
- */
-struct ondisk_cache_entry {
- struct cache_time ctime;
- struct cache_time mtime;
- unsigned int dev;
- unsigned int ino;
- unsigned int mode;
- unsigned int uid;
- unsigned int gid;
- unsigned int size;
- unsigned char sha1[20];
- unsigned short flags;
- char name[FLEX_ARRAY]; /* more */
-};
-
-/*
- * This struct is used when CE_EXTENDED bit is 1
- * The struct must match ondisk_cache_entry exactly from
- * ctime till flags
- */
-struct ondisk_cache_entry_extended {
- struct cache_time ctime;
- struct cache_time mtime;
- unsigned int dev;
- unsigned int ino;
- unsigned int mode;
- unsigned int uid;
- unsigned int gid;
- unsigned int size;
- unsigned char sha1[20];
- unsigned short flags;
- unsigned short flags2;
- char name[FLEX_ARRAY]; /* more */
-};
-
-/* These are only used for v3 or lower */
-#define align_flex_name(STRUCT,len) ((offsetof(struct STRUCT,name) + (len) + 8) & ~7)
-#define ondisk_cache_entry_size(len) align_flex_name(ondisk_cache_entry,len)
-#define ondisk_cache_entry_extended_size(len) align_flex_name(ondisk_cache_entry_extended,len)
-#define ondisk_ce_size(ce) (((ce)->ce_flags & CE_EXTENDED) ? \
- ondisk_cache_entry_extended_size(ce_namelen(ce)) : \
- ondisk_cache_entry_size(ce_namelen(ce)))
-
-static int verify_hdr(struct cache_header *hdr, unsigned long size)
+static int verify_hdr_version(struct index_state *istate,
+ struct cache_version_header *hdr, unsigned long size)
{
- git_SHA_CTX c;
- unsigned char sha1[20];
int hdr_version;
if (hdr->hdr_signature != htonl(CACHE_SIGNATURE))
return error("bad signature");
hdr_version = ntohl(hdr->hdr_version);
- if (hdr_version < 2 || 4 < hdr_version)
+ if (hdr_version >= 2 && hdr_version <= 4)
+ istate->ops = &v2_ops;
+ else
return error("bad index version %d", hdr_version);
- git_SHA1_Init(&c);
- git_SHA1_Update(&c, hdr, size - 20);
- git_SHA1_Final(sha1, &c);
- if (hashcmp(sha1, (unsigned char *)hdr + size - 20))
- return error("bad index file sha1 signature");
- return 0;
-}
-
-static int read_index_extension(struct index_state *istate,
- const char *ext, void *data, unsigned long sz)
-{
- switch (CACHE_EXT(ext)) {
- case CACHE_EXT_TREE:
- istate->cache_tree = cache_tree_read(data, sz);
- break;
- case CACHE_EXT_RESOLVE_UNDO:
- istate->resolve_undo = resolve_undo_read(data, sz);
- break;
- default:
- if (*ext < 'A' || 'Z' < *ext)
- return error("index uses %.4s extension, which we do not understand",
- ext);
- fprintf(stderr, "ignoring %.4s extension\n", ext);
- break;
- }
return 0;
}
@@ -1291,128 +1170,14 @@ int read_index(struct index_state *istate)
return read_index_from(istate, get_index_file());
}
-#ifndef NEEDS_ALIGNED_ACCESS
-#define ntoh_s(var) ntohs(var)
-#define ntoh_l(var) ntohl(var)
-#else
-static inline uint16_t ntoh_s_force_align(void *p)
-{
- uint16_t x;
- memcpy(&x, p, sizeof(x));
- return ntohs(x);
-}
-static inline uint32_t ntoh_l_force_align(void *p)
-{
- uint32_t x;
- memcpy(&x, p, sizeof(x));
- return ntohl(x);
-}
-#define ntoh_s(var) ntoh_s_force_align(&(var))
-#define ntoh_l(var) ntoh_l_force_align(&(var))
-#endif
-
-static struct cache_entry *cache_entry_from_ondisk(struct ondisk_cache_entry *ondisk,
- unsigned int flags,
- const char *name,
- size_t len)
-{
- struct cache_entry *ce = xmalloc(cache_entry_size(len));
-
- ce->ce_ctime.sec = ntoh_l(ondisk->ctime.sec);
- ce->ce_mtime.sec = ntoh_l(ondisk->mtime.sec);
- ce->ce_ctime.nsec = ntoh_l(ondisk->ctime.nsec);
- ce->ce_mtime.nsec = ntoh_l(ondisk->mtime.nsec);
- ce->ce_dev = ntoh_l(ondisk->dev);
- ce->ce_ino = ntoh_l(ondisk->ino);
- ce->ce_mode = ntoh_l(ondisk->mode);
- ce->ce_uid = ntoh_l(ondisk->uid);
- ce->ce_gid = ntoh_l(ondisk->gid);
- ce->ce_size = ntoh_l(ondisk->size);
- ce->ce_flags = flags & ~CE_NAMEMASK;
- ce->ce_namelen = len;
- hashcpy(ce->sha1, ondisk->sha1);
- memcpy(ce->name, name, len);
- ce->name[len] = '\0';
- return ce;
-}
-
-/*
- * Adjacent cache entries tend to share the leading paths, so it makes
- * sense to only store the differences in later entries. In the v4
- * on-disk format of the index, each on-disk cache entry stores the
- * number of bytes to be stripped from the end of the previous name,
- * and the bytes to append to the result, to come up with its name.
- */
-static unsigned long expand_name_field(struct strbuf *name, const char *cp_)
-{
- const unsigned char *ep, *cp = (const unsigned char *)cp_;
- size_t len = decode_varint(&cp);
-
- if (name->len < len)
- die("malformed name field in the index");
- strbuf_remove(name, name->len - len, len);
- for (ep = cp; *ep; ep++)
- ; /* find the end */
- strbuf_add(name, cp, ep - cp);
- return (const char *)ep + 1 - cp_;
-}
-
-static struct cache_entry *create_from_disk(struct ondisk_cache_entry *ondisk,
- unsigned long *ent_size,
- struct strbuf *previous_name)
-{
- struct cache_entry *ce;
- size_t len;
- const char *name;
- unsigned int flags;
-
- /* On-disk flags are just 16 bits */
- flags = ntoh_s(ondisk->flags);
- len = flags & CE_NAMEMASK;
-
- if (flags & CE_EXTENDED) {
- struct ondisk_cache_entry_extended *ondisk2;
- int extended_flags;
- ondisk2 = (struct ondisk_cache_entry_extended *)ondisk;
- extended_flags = ntoh_s(ondisk2->flags2) << 16;
- /* We do not yet understand any bit out of CE_EXTENDED_FLAGS */
- if (extended_flags & ~CE_EXTENDED_FLAGS)
- die("Unknown index entry format %08x", extended_flags);
- flags |= extended_flags;
- name = ondisk2->name;
- }
- else
- name = ondisk->name;
-
- if (!previous_name) {
- /* v3 and earlier */
- if (len == CE_NAMEMASK)
- len = strlen(name);
- ce = cache_entry_from_ondisk(ondisk, flags, name, len);
-
- *ent_size = ondisk_ce_size(ce);
- } else {
- unsigned long consumed;
- consumed = expand_name_field(previous_name, name);
- ce = cache_entry_from_ondisk(ondisk, flags,
- previous_name->buf,
- previous_name->len);
-
- *ent_size = (name - ((char *)ondisk)) + consumed;
- }
- return ce;
-}
-
/* remember to discard_cache() before reading a different cache! */
int read_index_from(struct index_state *istate, const char *path)
{
- int fd, i;
+ int fd;
struct stat st;
- unsigned long src_offset;
- struct cache_header *hdr;
+ struct cache_version_header *hdr;
void *mmap;
size_t mmap_size;
- struct strbuf previous_name_buf = STRBUF_INIT, *previous_name;
errno = EBUSY;
if (istate->initialized)
@@ -1433,69 +1198,27 @@ int read_index_from(struct index_state *istate, const char *path)
errno = EINVAL;
mmap_size = xsize_t(st.st_size);
- if (mmap_size < sizeof(struct cache_header) + 20)
- die("index file smaller than expected");
-
mmap = xmmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
close(fd);
if (mmap == MAP_FAILED)
die_errno("unable to map index file");
hdr = mmap;
- if (verify_hdr(hdr, mmap_size) < 0)
+ if (verify_hdr_version(istate, hdr, mmap_size) < 0)
goto unmap;
- istate->version = ntohl(hdr->hdr_version);
- istate->cache_nr = ntohl(hdr->hdr_entries);
- istate->cache_alloc = alloc_nr(istate->cache_nr);
- istate->cache = xcalloc(istate->cache_alloc, sizeof(struct cache_entry *));
- istate->initialized = 1;
-
- if (istate->version == 4)
- previous_name = &previous_name_buf;
- else
- previous_name = NULL;
-
- src_offset = sizeof(*hdr);
- for (i = 0; i < istate->cache_nr; i++) {
- struct ondisk_cache_entry *disk_ce;
- struct cache_entry *ce;
- unsigned long consumed;
-
- disk_ce = (struct ondisk_cache_entry *)((char *)mmap + src_offset);
- ce = create_from_disk(disk_ce, &consumed, previous_name);
- set_index_entry(istate, i, ce);
+ if (istate->ops->verify_hdr(mmap, mmap_size) < 0)
+ goto unmap;
- src_offset += consumed;
- }
- strbuf_release(&previous_name_buf);
+ istate->ops->read_index(istate, mmap, mmap_size);
istate->timestamp.sec = st.st_mtime;
istate->timestamp.nsec = ST_MTIME_NSEC(st);
- while (src_offset <= mmap_size - 20 - 8) {
- /* After an array of active_nr index entries,
- * there can be arbitrary number of extended
- * sections, each of which is prefixed with
- * extension name (4-byte) and section length
- * in 4-byte network byte order.
- */
- uint32_t extsize;
- memcpy(&extsize, (char *)mmap + src_offset + 4, 4);
- extsize = ntohl(extsize);
- if (read_index_extension(istate,
- (const char *) mmap + src_offset,
- (char *) mmap + src_offset + 8,
- extsize) < 0)
- goto unmap;
- src_offset += 8;
- src_offset += extsize;
- }
munmap(mmap, mmap_size);
return istate->cache_nr;
unmap:
munmap(mmap, mmap_size);
- errno = EINVAL;
die("index file corrupt");
}
@@ -1534,201 +1257,6 @@ int unmerged_index(const struct index_state *istate)
return 0;
}
-#define WRITE_BUFFER_SIZE 8192
-static unsigned char write_buffer[WRITE_BUFFER_SIZE];
-static unsigned long write_buffer_len;
-
-static int ce_write_flush(git_SHA_CTX *context, int fd)
-{
- unsigned int buffered = write_buffer_len;
- if (buffered) {
- git_SHA1_Update(context, write_buffer, buffered);
- if (write_in_full(fd, write_buffer, buffered) != buffered)
- return -1;
- write_buffer_len = 0;
- }
- return 0;
-}
-
-static int ce_write(git_SHA_CTX *context, int fd, void *data, unsigned int len)
-{
- while (len) {
- unsigned int buffered = write_buffer_len;
- unsigned int partial = WRITE_BUFFER_SIZE - buffered;
- if (partial > len)
- partial = len;
- memcpy(write_buffer + buffered, data, partial);
- buffered += partial;
- if (buffered == WRITE_BUFFER_SIZE) {
- write_buffer_len = buffered;
- if (ce_write_flush(context, fd))
- return -1;
- buffered = 0;
- }
- write_buffer_len = buffered;
- len -= partial;
- data = (char *) data + partial;
- }
- return 0;
-}
-
-static int write_index_ext_header(git_SHA_CTX *context, int fd,
- unsigned int ext, unsigned int sz)
-{
- ext = htonl(ext);
- sz = htonl(sz);
- return ((ce_write(context, fd, &ext, 4) < 0) ||
- (ce_write(context, fd, &sz, 4) < 0)) ? -1 : 0;
-}
-
-static int ce_flush(git_SHA_CTX *context, int fd)
-{
- unsigned int left = write_buffer_len;
-
- if (left) {
- write_buffer_len = 0;
- git_SHA1_Update(context, write_buffer, left);
- }
-
- /* Flush first if not enough space for SHA1 signature */
- if (left + 20 > WRITE_BUFFER_SIZE) {
- if (write_in_full(fd, write_buffer, left) != left)
- return -1;
- left = 0;
- }
-
- /* Append the SHA1 signature at the end */
- git_SHA1_Final(write_buffer + left, context);
- left += 20;
- return (write_in_full(fd, write_buffer, left) != left) ? -1 : 0;
-}
-
-static void ce_smudge_racily_clean_entry(struct cache_entry *ce)
-{
- /*
- * The only thing we care about in this function is to smudge the
- * falsely clean entry due to touch-update-touch race, so we leave
- * everything else as they are. We are called for entries whose
- * ce_mtime match the index file mtime.
- *
- * Note that this actually does not do much for gitlinks, for
- * which ce_match_stat_basic() always goes to the actual
- * contents. The caller checks with is_racy_timestamp() which
- * always says "no" for gitlinks, so we are not called for them ;-)
- */
- struct stat st;
-
- if (lstat(ce->name, &st) < 0)
- return;
- if (ce_match_stat_basic(ce, &st))
- return;
- if (ce_modified_check_fs(ce, &st)) {
- /* This is "racily clean"; smudge it. Note that this
- * is a tricky code. At first glance, it may appear
- * that it can break with this sequence:
- *
- * $ echo xyzzy >frotz
- * $ git-update-index --add frotz
- * $ : >frotz
- * $ sleep 3
- * $ echo filfre >nitfol
- * $ git-update-index --add nitfol
- *
- * but it does not. When the second update-index runs,
- * it notices that the entry "frotz" has the same timestamp
- * as index, and if we were to smudge it by resetting its
- * size to zero here, then the object name recorded
- * in index is the 6-byte file but the cached stat information
- * becomes zero --- which would then match what we would
- * obtain from the filesystem next time we stat("frotz").
- *
- * However, the second update-index, before calling
- * this function, notices that the cached size is 6
- * bytes and what is on the filesystem is an empty
- * file, and never calls us, so the cached size information
- * for "frotz" stays 6 which does not match the filesystem.
- */
- ce->ce_size = 0;
- }
-}
-
-/* Copy miscellaneous fields but not the name */
-static char *copy_cache_entry_to_ondisk(struct ondisk_cache_entry *ondisk,
- struct cache_entry *ce)
-{
- short flags;
-
- ondisk->ctime.sec = htonl(ce->ce_ctime.sec);
- ondisk->mtime.sec = htonl(ce->ce_mtime.sec);
- ondisk->ctime.nsec = htonl(ce->ce_ctime.nsec);
- ondisk->mtime.nsec = htonl(ce->ce_mtime.nsec);
- ondisk->dev = htonl(ce->ce_dev);
- ondisk->ino = htonl(ce->ce_ino);
- ondisk->mode = htonl(ce->ce_mode);
- ondisk->uid = htonl(ce->ce_uid);
- ondisk->gid = htonl(ce->ce_gid);
- ondisk->size = htonl(ce->ce_size);
- hashcpy(ondisk->sha1, ce->sha1);
-
- flags = ce->ce_flags;
- flags |= (ce_namelen(ce) >= CE_NAMEMASK ? CE_NAMEMASK : ce_namelen(ce));
- ondisk->flags = htons(flags);
- if (ce->ce_flags & CE_EXTENDED) {
- struct ondisk_cache_entry_extended *ondisk2;
- ondisk2 = (struct ondisk_cache_entry_extended *)ondisk;
- ondisk2->flags2 = htons((ce->ce_flags & CE_EXTENDED_FLAGS) >> 16);
- return ondisk2->name;
- }
- else {
- return ondisk->name;
- }
-}
-
-static int ce_write_entry(git_SHA_CTX *c, int fd, struct cache_entry *ce,
- struct strbuf *previous_name)
-{
- int size;
- struct ondisk_cache_entry *ondisk;
- char *name;
- int result;
-
- if (!previous_name) {
- size = ondisk_ce_size(ce);
- ondisk = xcalloc(1, size);
- name = copy_cache_entry_to_ondisk(ondisk, ce);
- memcpy(name, ce->name, ce_namelen(ce));
- } else {
- int common, to_remove, prefix_size;
- unsigned char to_remove_vi[16];
- for (common = 0;
- (ce->name[common] &&
- common < previous_name->len &&
- ce->name[common] == previous_name->buf[common]);
- common++)
- ; /* still matching */
- to_remove = previous_name->len - common;
- prefix_size = encode_varint(to_remove, to_remove_vi);
-
- if (ce->ce_flags & CE_EXTENDED)
- size = offsetof(struct ondisk_cache_entry_extended, name);
- else
- size = offsetof(struct ondisk_cache_entry, name);
- size += prefix_size + (ce_namelen(ce) - common + 1);
-
- ondisk = xcalloc(1, size);
- name = copy_cache_entry_to_ondisk(ondisk, ce);
- memcpy(name, to_remove_vi, prefix_size);
- memcpy(name + prefix_size, ce->name + common, ce_namelen(ce) - common);
-
- strbuf_splice(previous_name, common, to_remove,
- ce->name + common, ce_namelen(ce) - common);
- }
-
- result = ce_write(c, fd, ondisk, size);
- free(ondisk);
- return result;
-}
-
static int has_racy_timestamp(struct index_state *istate)
{
int entries = istate->cache_nr;
@@ -1756,83 +1284,9 @@ void update_index_if_able(struct index_state *istate, struct lock_file *lockfile
int write_index(struct index_state *istate, int newfd)
{
- git_SHA_CTX c;
- struct cache_header hdr;
- int i, err, removed, extended, hdr_version;
- struct cache_entry **cache = istate->cache;
- int entries = istate->cache_nr;
- struct stat st;
- struct strbuf previous_name_buf = STRBUF_INIT, *previous_name;
-
- for (i = removed = extended = 0; i < entries; i++) {
- if (cache[i]->ce_flags & CE_REMOVE)
- removed++;
-
- /* reduce extended entries if possible */
- cache[i]->ce_flags &= ~CE_EXTENDED;
- if (cache[i]->ce_flags & CE_EXTENDED_FLAGS) {
- extended++;
- cache[i]->ce_flags |= CE_EXTENDED;
- }
- }
-
- if (!istate->version)
- istate->version = INDEX_FORMAT_DEFAULT;
-
- /* demote version 3 to version 2 when the latter suffices */
- if (istate->version == 3 || istate->version == 2)
- istate->version = extended ? 3 : 2;
-
- hdr_version = istate->version;
-
- hdr.hdr_signature = htonl(CACHE_SIGNATURE);
- hdr.hdr_version = htonl(hdr_version);
- hdr.hdr_entries = htonl(entries - removed);
-
- git_SHA1_Init(&c);
- if (ce_write(&c, newfd, &hdr, sizeof(hdr)) < 0)
- return -1;
-
- previous_name = (hdr_version == 4) ? &previous_name_buf : NULL;
- for (i = 0; i < entries; i++) {
- struct cache_entry *ce = cache[i];
- if (ce->ce_flags & CE_REMOVE)
- continue;
- if (!ce_uptodate(ce) && is_racy_timestamp(istate, ce))
- ce_smudge_racily_clean_entry(ce);
- if (ce_write_entry(&c, newfd, ce, previous_name) < 0)
- return -1;
- }
- strbuf_release(&previous_name_buf);
+ check_set_istate_ops(istate);
- /* Write extension data here */
- if (istate->cache_tree) {
- struct strbuf sb = STRBUF_INIT;
-
- cache_tree_write(&sb, istate->cache_tree);
- err = write_index_ext_header(&c, newfd, CACHE_EXT_TREE, sb.len) < 0
- || ce_write(&c, newfd, sb.buf, sb.len) < 0;
- strbuf_release(&sb);
- if (err)
- return -1;
- }
- if (istate->resolve_undo) {
- struct strbuf sb = STRBUF_INIT;
-
- resolve_undo_write(&sb, istate->resolve_undo);
- err = write_index_ext_header(&c, newfd, CACHE_EXT_RESOLVE_UNDO,
- sb.len) < 0
- || ce_write(&c, newfd, sb.buf, sb.len) < 0;
- strbuf_release(&sb);
- if (err)
- return -1;
- }
-
- if (ce_flush(&c, newfd) || fstat(newfd, &st))
- return -1;
- istate->timestamp.sec = (unsigned int)st.st_mtime;
- istate->timestamp.nsec = ST_MTIME_NSEC(st);
- return 0;
+ return istate->ops->write_index(istate, newfd);
}
/*
diff --git a/read-cache.h b/read-cache.h
new file mode 100644
index 0000000..f41ba75
--- /dev/null
+++ b/read-cache.h
@@ -0,0 +1,57 @@
+/* Index extensions.
+ *
+ * The first letter should be 'A'..'Z' for extensions that are not
+ * necessary for a correct operation (i.e. optimization data).
+ * When new extensions are added that _needs_ to be understood in
+ * order to correctly interpret the index file, pick character that
+ * is outside the range, to cause the reader to abort.
+ */
+
+#define CACHE_EXT(s) ( (s[0]<<24)|(s[1]<<16)|(s[2]<<8)|(s[3]) )
+#define CACHE_EXT_TREE 0x54524545 /* "TREE" */
+#define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */
+
+#define INDEX_FORMAT_DEFAULT 3
+
+/*
+ * Basic data structures for the directory cache
+ */
+struct cache_version_header {
+ unsigned int hdr_signature;
+ unsigned int hdr_version;
+};
+
+struct index_ops {
+ int (*match_stat_basic)(struct cache_entry *ce, struct stat *st, int changed);
+ int (*verify_hdr)(void *mmap, unsigned long size);
+ int (*read_index)(struct index_state *istate, void *mmap, int mmap_size);
+ int (*write_index)(struct index_state *istate, int newfd);
+};
+
+extern struct index_ops v2_ops;
+
+#ifndef NEEDS_ALIGNED_ACCESS
+#define ntoh_s(var) ntohs(var)
+#define ntoh_l(var) ntohl(var)
+#else
+static inline uint16_t ntoh_s_force_align(void *p)
+{
+ uint16_t x;
+ memcpy(&x, p, sizeof(x));
+ return ntohs(x);
+}
+static inline uint32_t ntoh_l_force_align(void *p)
+{
+ uint32_t x;
+ memcpy(&x, p, sizeof(x));
+ return ntohl(x);
+}
+#define ntoh_s(var) ntoh_s_force_align(&(var))
+#define ntoh_l(var) ntoh_l_force_align(&(var))
+#endif
+
+extern int ce_modified_check_fs(struct cache_entry *ce, struct stat *st);
+extern int ce_match_stat_basic(struct index_state *istate,
+ struct cache_entry *ce, struct stat *st);
+extern int is_racy_timestamp(const struct index_state *istate, struct cache_entry *ce);
+extern void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce);
diff --git a/test-index-version.c b/test-index-version.c
index bfaad9e..3899a2f 100644
--- a/test-index-version.c
+++ b/test-index-version.c
@@ -1,8 +1,13 @@
#include "cache.h"
+struct cache_version_header {
+ unsigned int hdr_signature;
+ unsigned int hdr_version;
+};
+
int main(int argc, const char **argv)
{
- struct cache_header hdr;
+ struct cache_version_header hdr;
int version;
memset(&hdr,0,sizeof(hdr));
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 02/13] read-cache.c: Re-read index if index file changed
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 01/13] Move index v2 specific functions to their own file Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 03/13] t2104: Don't fail for index versions other than [23] Thomas Gummerer
` (10 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Add the possibility of re-reading the index file, if it changed
while reading.
The index file might change during the read, causing outdated
information to be displayed. We check if the index file changed
by using its stat data as heuristic.
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
read-cache.c | 87 +++++++++++++++++++++++++++++++++++++++++-------------------
1 file changed, 60 insertions(+), 27 deletions(-)
diff --git a/read-cache.c b/read-cache.c
index 6a8b4b1..cdd8480 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -1170,11 +1170,34 @@ int read_index(struct index_state *istate)
return read_index_from(istate, get_index_file());
}
+static int index_changed(struct stat st_old, struct stat st_new)
+{
+ int changed = 0;
+
+ if (st_old.st_mtime != st_new.st_mtime ||
+ st_old.st_uid != st_new.st_uid ||
+ st_old.st_gid != st_new.st_gid ||
+ st_old.st_ino != st_new.st_ino ||
+ st_old.st_size != st_new.st_size)
+ changed = 1;
+#ifdef USE_NSEC
+ if (ST_MTIME_NSEC(st_old) != ST_MTIME_NSEC(st_new))
+ changed = 1;
+#endif
+
+#ifdef USE_STDEV
+ if (st_old.st_dev != st_new.st_dev)
+ changed = 1;
+#endif
+
+ return changed;
+}
+
/* remember to discard_cache() before reading a different cache! */
int read_index_from(struct index_state *istate, const char *path)
{
- int fd;
- struct stat st;
+ int fd, err, i = 0;
+ struct stat st_old, st_new;
struct cache_version_header *hdr;
void *mmap;
size_t mmap_size;
@@ -1186,38 +1209,48 @@ int read_index_from(struct index_state *istate, const char *path)
errno = ENOENT;
istate->timestamp.sec = 0;
istate->timestamp.nsec = 0;
- fd = open(path, O_RDONLY);
- if (fd < 0) {
- if (errno == ENOENT)
- return 0;
- die_errno("index file open failed");
- }
+ do {
+ err = 0;
+ fd = open(path, O_RDONLY);
+ if (fd < 0) {
+ if (errno == ENOENT)
+ return 0;
+ die_errno("index file open failed");
+ }
- if (fstat(fd, &st))
- die_errno("cannot stat the open index");
+ if (fstat(fd, &st_old))
+ die_errno("cannot stat the open index");
- errno = EINVAL;
- mmap_size = xsize_t(st.st_size);
- mmap = xmmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
- close(fd);
- if (mmap == MAP_FAILED)
- die_errno("unable to map index file");
+ errno = EINVAL;
+ mmap_size = xsize_t(st_old.st_size);
+ mmap = xmmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
+ close(fd);
+ if (mmap == MAP_FAILED)
+ die_errno("unable to map index file");
- hdr = mmap;
- if (verify_hdr_version(istate, hdr, mmap_size) < 0)
- goto unmap;
+ hdr = mmap;
+ if (verify_hdr_version(istate, hdr, mmap_size) < 0)
+ err = 1;
- if (istate->ops->verify_hdr(mmap, mmap_size) < 0)
- goto unmap;
+ if (istate->ops->verify_hdr(mmap, mmap_size) < 0)
+ err = 1;
- istate->ops->read_index(istate, mmap, mmap_size);
- istate->timestamp.sec = st.st_mtime;
- istate->timestamp.nsec = ST_MTIME_NSEC(st);
+ if (istate->ops->read_index(istate, mmap, mmap_size) < 0)
+ err = 1;
+ istate->timestamp.sec = st_old.st_mtime;
+ istate->timestamp.nsec = ST_MTIME_NSEC(st_old);
+ if (lstat(path, &st_new))
+ die_errno("cannot stat the open index");
- munmap(mmap, mmap_size);
- return istate->cache_nr;
+ munmap(mmap, mmap_size);
+
+ if (!index_changed(st_old, st_new) && !err)
+ return istate->cache_nr;
+
+ usleep(10*1000);
+ i++;
+ } while ((err || index_changed(st_old, st_new)) && i < 50);
-unmap:
munmap(mmap, mmap_size);
die("index file corrupt");
}
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 03/13] t2104: Don't fail for index versions other than [23]
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 01/13] Move index v2 specific functions to their own file Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 02/13] read-cache.c: Re-read index if index file changed Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 04/13] Add documentation of the index-v5 file format Thomas Gummerer
` (9 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
t2104 currently checks for the exact index version 2 or 3,
depending if there is a skip-worktree flag or not. Other
index versions do not use extended flags and thus cannot
be tested for version changes.
Make this test update the index to version 2 at the beginning
of the test. Testing the skip-worktree flags for the default
index format is still covered by t7011 and t7012.
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
t/t2104-update-index-skip-worktree.sh | 1 +
1 file changed, 1 insertion(+)
diff --git a/t/t2104-update-index-skip-worktree.sh b/t/t2104-update-index-skip-worktree.sh
index 1d0879b..bd9644f 100755
--- a/t/t2104-update-index-skip-worktree.sh
+++ b/t/t2104-update-index-skip-worktree.sh
@@ -22,6 +22,7 @@ H sub/2
EOF
test_expect_success 'setup' '
+ git update-index --index-version=2 &&
mkdir sub &&
touch ./1 ./2 sub/1 sub/2 &&
git add 1 2 sub/1 sub/2 &&
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 04/13] Add documentation of the index-v5 file format
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
` (2 preceding siblings ...)
2012-08-16 9:58 ` [PATCH/RFC v4 03/13] t2104: Don't fail for index versions other than [23] Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 05/13] Make in-memory format aware of stat_crc Thomas Gummerer
` (8 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Add a documentation of the index file format version 5 to
Documentation/technical.
Helped-by: Michael Haggerty <mhagger@alum.mit.edu>
Helped-by: Junio C Hamano <gitster@pobox.com>
Helped-by: Thomas Rast <trast@student.ethz.ch>
Helped-by: Nguyen Thai Ngoc Duy <pclouds@gmail.com>
Helped-by: Robin Rosenberg <robin.rosenberg@dewire.com>
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
Documentation/technical/index-file-format-v5.txt | 296 +++++++++++++++++++++++
1 file changed, 296 insertions(+)
create mode 100644 Documentation/technical/index-file-format-v5.txt
diff --git a/Documentation/technical/index-file-format-v5.txt b/Documentation/technical/index-file-format-v5.txt
new file mode 100644
index 0000000..563d830
--- /dev/null
+++ b/Documentation/technical/index-file-format-v5.txt
@@ -0,0 +1,296 @@
+GIT index format
+================
+
+== The git index
+
+ The git index file (.git/index) documents the status of the files
+ in the git staging area.
+
+ The staging area is used for preparing commits, merging, etc.
+
+== The git index file format
+
+ All binary numbers are in network byte order. Version 5 is described
+ here. The index file consists of various sections. They appear in
+ the following order in the file.
+
+ - header: the description of the index format, including it's signature,
+ version and various other fields that are used internally.
+
+ - diroffsets (ndir entries of "direcotry offset"): A 4-byte offset
+ relative to the beginning of the "direntries block" (see below)
+ for each of the ndir directories in the index, sorted by pathname
+ (of the directory it's pointing to). [1]
+
+ - direntries (ndir entries of "directory offset"): A directory entry
+ for each of the ndir directories in the index, sorted by pathname
+ (see below). [2]
+
+ - fileoffsets (nfile entries of "file offset"): A 4-byte offset
+ relative to the beginning of the fileentries block (see below)
+ for each of the nfile files in the index. [1]
+
+ - fileentries (nfile entries of "file entry"): A file entry for
+ each of the nfile files in the index (see below).
+
+ - crdata: A number of entries for conflicted data/resolved conflicts
+ (see below).
+
+ - Extensions (Currently none, see below in the future)
+
+ Extensions are identified by signature. Optional extensions can
+ be ignored if GIT does not understand them.
+
+ GIT supports an arbitrary number of extension, but currently none
+ is implemented. [3]
+
+ extsig (32-bits): extension signature. If the first byte is 'A'..'Z'
+ the extension is optional and can be ignored.
+
+ extsize (32-bits): size of the extension, excluding the header
+ (extsig, extsize, extchecksum).
+
+ extchecksum (32-bits): crc32 checksum of the extension signature
+ and size.
+
+ - Extension data.
+
+== Header
+ sig (32-bits): Signature:
+ The signature is { 'D', 'I', 'R', 'C' } (stands for "dircache")
+
+ vnr (32-bits): Version number:
+ The current supported versions are 2, 3, 4 and 5.
+
+ ndir (32-bits): number of directories in the index.
+
+ nfile (32-bits): number of file entries in the index.
+
+ fblockoffset (32-bits): offset to the file block, relative to the
+ beginning of the file.
+
+ - Offset to the extensions.
+
+ nextensions (32-bits): number of extensions.
+
+ extoffset (32-bits): offset to the extension. (Possibly none, as
+ many as indicated in the 4-byte number of extensions)
+
+ headercrc (32-bits): crc checksum including the header and the
+ offsets to the extensions.
+
+
+== Directory offsets (diroffsets)
+
+ diroffset (32-bits): offset to the directory relative to the beginning
+ of the index file. There are ndir + 1 offsets in the diroffset table,
+ the last is pointing to the end of the last direntry. With this last
+ entry, we are able to replace the strlen of when reading the directory
+ name, by calculating it from diroffset[n+1]-diroffset[n]-61. 61 is the
+ size of the directory data, which follows each each directory + the
+ crc sum + the NUL byte.
+
+ This part is needed for making the directory entries bisectable and
+ thus allowing a binary search.
+
+== Directory entry (direntries)
+
+ Directory entries are sorted in lexicographic order by the name
+ of their path starting with the root.
+
+ pathname (variable length, nul terminated): relative to top level
+ directory (without the leading slash). '/' is used as path
+ separator. A string of length 0 ('') indicates the root directory.
+ The special path components ".", and ".." (without quotes) are
+ disallowed. The path also includes a trailing slash. [9]
+
+ foffset (32-bits): offset to the lexicographically first file in
+ the file offsets (fileoffsets), relative to the beginning of
+ the fileoffset block.
+
+ cr (32-bits): offset to conflicted/resolved data at the end of the
+ index. 0 if there is no such data. [4]
+
+ ncr (32-bits): number of conflicted/resolved data entries at the
+ end of the index if the offset is non 0. If cr is 0, ncr is
+ also 0.
+
+ nsubtrees (32-bits): number of subtrees this tree has in the index.
+
+ nfiles (32-bits): number of files in the directory, that are in
+ the index.
+
+ nentries (32-bits): number of entries in the index that is covered
+ by the tree this entry represents. (-1 if the entry is invalid).
+ This number includes all the files in this tree, recursively.
+
+ objname (160-bits): object name for the object that would result
+ from writing this span of index as a tree. This is only valid
+ if nentries is valid, meaning the cache-tree is valid.
+
+ flags (16-bits): 'flags' field split into (high to low bits) (For
+ D/F conflicts)
+
+ stage (2-bits): stage of the directory during merge
+
+ 14-bit unused
+
+ dircrc (32-bits): crc32 checksum for each directory entry.
+
+ The last 24 bytes (4-byte number of entries + 160-bit object name) are
+ for the cache tree. An entry can be in an invalidated state which is
+ represented by having -1 in the entry_count field.
+
+ The entries are written out in the top-down, depth-first order. The
+ first entry represents the root level of the repository, followed by
+ the first subtree - let's call it A - of the root level, followed by
+ the first subtree of A, ... There is no prefix compression for
+ directories.
+
+== File offsets (fileoffsets)
+
+ fileoffset (32-bits): offset to the file relative to the beginning of
+ the fileentries block.
+
+ This part is needed for making the file entries bisectable and
+ thus allowing a binary search. There are nfile + 1 offsets in the
+ fileoffset table, the last is pointing to the end of the last
+ fileentry. With this last entry, we can replace the strlen when
+ reading each filename, by calculating its length with the offsets.
+
+== File entry (fileentries)
+
+ File entries are sorted in ascending order on the name field, after the
+ respective offset given by the directory entries. All file names are
+ prefix compressed, meaning the file name is relative to the directory.
+
+ filename (variable length, nul terminated). The exact encoding is
+ undefined, but the filename cannot contain a NUL byte (iow, the same
+ encoding as a UNIX pathname).
+
+ flags (16-bits): 'flags' field split into (high to low bits)
+
+ assumevalid (1-bit): assume-valid flag
+
+ intenttoadd (1-bit): intent-to-add flag, used by "git add -N".
+ Extended flag in index v3.
+
+ stage (2-bit): stage of the file during merge
+
+ skipworktree (1-bit): skip-worktree flag, used by sparse checkout.
+ Extended flag in index v3.
+
+ smudged (1-bit): indicates if the file is racily smudged.
+
+ 10-bit unused, must be zero [6]
+
+ mode (16-bits): file mode, split into (high to low bits)
+
+ objtype (4-bits): object type
+ valid values in binary are 1000 (regular file), 1010 (symbolic
+ link) and 1110 (gitlink)
+
+ 3-bit unused
+
+ permission (9-bits): unix permission. Only 0755 and 0644 are valid
+ for regular files. Symbolic links and gitlinks have value 0 in
+ this field.
+
+ mtimes (32-bits): mtime seconds, the last time a file's data changed
+ this is stat(2) data
+
+ mtimens (32-bits): mtime nanosecond fractions
+ this is stat(2) data
+
+ file size (32-bits): The on-disk size, trucated to 32-bit.
+ this is stat(2) data
+
+ statcrc (32-bits): crc32 checksum over ctime seconds, ctime
+ nanoseconds, ino, dev, uid, gid (All stat(2) data
+ except mtime and file size). If the statcrc is 0 it will
+ be ignored. [7]
+
+ objhash (160-bits): SHA-1 for the represented object
+
+ entrycrc (32-bits): crc32 checksum for the file entry. The crc code
+ includes the offset to the offset to the file, relative to the
+ beginning of the file.
+
+== Conflict data
+
+ A conflict is represented in the index as a set of higher stage entries.
+ These entries are stored at the end of the index. When a conflict is
+ resolved (e.g. with "git add path"). A bit is flipped, to indicate that
+ the conflict is resolved, but the entries will be kept, so that
+ conflicts can be recreated (e.g. with "git checkout -m", in case users
+ want to redo a conflict resolution from scratch.
+
+ The first part of a conflict (usually stage 1) will be stored both in
+ the entries part of the index and in the conflict part. All other parts
+ will only be stored in the conflict part.
+
+ filename (variable length, nul terminated): filename of the entry,
+ relative to its containing directory).
+
+ nfileconflicts (32-bits): number of conflicts for the file [8]
+
+ flags (nfileconflicts entries of "flags") (16-bits): 'flags' field
+ split into:
+
+ conflicted (1-bit): conflicted state (conflicted/resolved) (1 if
+ conflicted)
+
+ stage (2-bits): stage during merge.
+
+ 13-bit unused
+
+ entry_mode (nfileconflicts entries of "entry mode") (16-bits):
+ octal numbers, entry mode of eache entry in the different stages.
+ (How many is defined by the 4-byte number before)
+
+ objectnames (nfileconflicts entries of "object name") (160-bits):
+ object names of the different stages.
+
+ conflictcrc (32-bits): crc32 checksum over conflict data.
+
+== Design explanations
+
+[1] The directory and file offsets are included in the index format
+ to enable bisectability of the index, for binary searches.Updating
+ a single entry and partial reading will benefit from this.
+
+[2] The directories are saved in their own block, to be able to
+ quickly search for a directory in the index. They include a
+ offset to the (lexically) first file in the directory.
+
+[3] The data of the cache-tree extension and the resolve undo
+ extension is now part of the index itself, but if other extensions
+ come up in the future, there is no need to change the index, they
+ can simply be added at the end.
+
+[4] To avoid rewrites of the whole index when there are conflicts or
+ conflicts are being resolved, conflicted data will be stored at
+ the end of the index. To mark the conflict resolved, just a bit
+ has to be flipped. The data will still be there, if a user wants
+ to redo the conflict resolution.
+
+[5] Since only 4 modes are effectively allowed in git but 32-bit are
+ used to store them, having a two bit flag for the mode is enough
+ and saves 4 byte per entry.
+
+[6] The length of the file name was dropped, since each file name is
+ nul terminated anyway.
+
+[7] Since all stat data (except mtime and ctime) is just used for
+ checking if a file has changed a checksum of the data is enough.
+ In addition to that Thomas Rast suggested ctime could be ditched
+ completely (core.trustctime=false) and thus included in the
+ checksum. This would save 24 bytes per index entry, which would
+ be about 4 MB on the Webkit index.
+ (Thanks for the suggestion to Michael Haggerty)
+
+[8] Since there can be more stage #1 entries, it is necessary to know
+ the number of conflict data entries there are.
+
+[9] As Michael Haggerty pointed out on the mailing list, storing the
+ trailing slash will simplify a few operations.
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 05/13] Make in-memory format aware of stat_crc
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
` (3 preceding siblings ...)
2012-08-16 9:58 ` [PATCH/RFC v4 04/13] Add documentation of the index-v5 file format Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 06/13] Read index-v5 Thomas Gummerer
` (7 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Make the in-memory format aware of the stat_crc used by index-v5.
It is simply ignored by index version prior to v5.
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
cache.h | 1 +
read-cache.c | 25 +++++++++++++++++++++++++
2 files changed, 26 insertions(+)
diff --git a/cache.h b/cache.h
index c77cdbe..bfe3099 100644
--- a/cache.h
+++ b/cache.h
@@ -122,6 +122,7 @@ struct cache_entry {
unsigned int ce_flags;
unsigned int ce_namelen;
unsigned char sha1[20];
+ uint32_t ce_stat_crc;
struct cache_entry *next;
struct cache_entry *dir_next;
char name[FLEX_ARRAY]; /* more */
diff --git a/read-cache.c b/read-cache.c
index cdd8480..9d2bd62 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -51,6 +51,29 @@ void rename_index_entry_at(struct index_state *istate, int nr, const char *new_n
add_index_entry(istate, new, ADD_CACHE_OK_TO_ADD|ADD_CACHE_OK_TO_REPLACE);
}
+static uint32_t calculate_stat_crc(struct cache_entry *ce)
+{
+ unsigned int ctimens = 0;
+ uint32_t stat, stat_crc;
+
+ stat = htonl(ce->ce_ctime.sec);
+ stat_crc = crc32(0, (Bytef*)&stat, 4);
+#ifdef USE_NSEC
+ ctimens = ce->ce_ctime.nsec;
+#endif
+ stat = htonl(ctimens);
+ stat_crc = crc32(stat_crc, (Bytef*)&stat, 4);
+ stat = htonl(ce->ce_ino);
+ stat_crc = crc32(stat_crc, (Bytef*)&stat, 4);
+ stat = htonl(ce->ce_dev);
+ stat_crc = crc32(stat_crc, (Bytef*)&stat, 4);
+ stat = htonl(ce->ce_uid);
+ stat_crc = crc32(stat_crc, (Bytef*)&stat, 4);
+ stat = htonl(ce->ce_gid);
+ stat_crc = crc32(stat_crc, (Bytef*)&stat, 4);
+ return stat_crc;
+}
+
/*
* This only updates the "non-critical" parts of the directory
* cache, ie the parts that aren't tracked by GIT, and only used
@@ -73,6 +96,8 @@ void fill_stat_cache_info(struct cache_entry *ce, struct stat *st)
if (S_ISREG(st->st_mode))
ce_mark_uptodate(ce);
+
+ ce->ce_stat_crc = calculate_stat_crc(ce);
}
static int ce_compare_data(struct cache_entry *ce, struct stat *st)
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 06/13] Read index-v5
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
` (4 preceding siblings ...)
2012-08-16 9:58 ` [PATCH/RFC v4 05/13] Make in-memory format aware of stat_crc Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 07/13] Read resolve-undo data Thomas Gummerer
` (6 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Make git read the index file version 5 without complaining.
This version of the reader doesn't read neither the cache-tree
nor the resolve undo data, but doesn't choke on an index that
includes such data.
Helped-by: Junio C Hamano <gitster@pobox.com>
Helped-by: Nguyen Thai Ngoc Duy <pclouds@gmail.com>
Helped-by: Thomas Rast <trast@student.ethz.ch>
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
Makefile | 1 +
cache.h | 73 ++++++++
read-cache-v5.c | 556 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 630 insertions(+)
create mode 100644 read-cache-v5.c
diff --git a/Makefile b/Makefile
index 3ccd3a8..ef55509 100644
--- a/Makefile
+++ b/Makefile
@@ -770,6 +770,7 @@ LIB_OBJS += quote.o
LIB_OBJS += reachable.o
LIB_OBJS += read-cache.o
LIB_OBJS += read-cache-v2.o
+LIB_OBJS += read-cache-v5.o
LIB_OBJS += reflog-walk.o
LIB_OBJS += refs.o
LIB_OBJS += remote.o
diff --git a/cache.h b/cache.h
index bfe3099..bec9402 100644
--- a/cache.h
+++ b/cache.h
@@ -110,6 +110,15 @@ struct cache_time {
unsigned int nsec;
};
+/*
+ * The *next pointer is used in read_entries_v5 for holding
+ * all the elements of a directory, and points to the next
+ * cache_entry in a directory.
+ *
+ * It is reset by the add_name_hash call in set_index_entry
+ * to set it to point to the next cache_entry in the
+ * correct in-memory format ordering.
+ */
struct cache_entry {
struct cache_time ce_ctime;
struct cache_time ce_mtime;
@@ -128,11 +137,59 @@ struct cache_entry {
char name[FLEX_ARRAY]; /* more */
};
+struct directory_entry {
+ struct directory_entry *next;
+ struct directory_entry *next_hash;
+ struct cache_entry *ce;
+ struct cache_entry *ce_last;
+ struct conflict_entry *conflict;
+ struct conflict_entry *conflict_last;
+ unsigned int conflict_size;
+ unsigned int de_foffset;
+ unsigned int de_cr;
+ unsigned int de_ncr;
+ unsigned int de_nsubtrees;
+ unsigned int de_nfiles;
+ unsigned int de_nentries;
+ unsigned char sha1[20];
+ unsigned short de_flags;
+ unsigned int de_pathlen;
+ char pathname[FLEX_ARRAY];
+};
+
+struct conflict_part {
+ struct conflict_part *next;
+ unsigned short flags;
+ unsigned short entry_mode;
+ unsigned char sha1[20];
+};
+
+struct conflict_entry {
+ struct conflict_entry *next;
+ unsigned int nfileconflicts;
+ struct conflict_part *entries;
+ unsigned int namelen;
+ unsigned int pathlen;
+ char name[FLEX_ARRAY];
+};
+
+struct ondisk_conflict_part {
+ unsigned short flags;
+ unsigned short entry_mode;
+ unsigned char sha1[20];
+};
+
+#define CE_NAMEMASK (0x0fff)
#define CE_STAGEMASK (0x3000)
#define CE_EXTENDED (0x4000)
#define CE_VALID (0x8000)
+#define CE_SMUDGED (0x0400) /* index v5 only flag */
#define CE_STAGESHIFT 12
+#define CONFLICT_CONFLICTED (0x8000)
+#define CONFLICT_STAGESHIFT 13
+#define CONFLICT_STAGEMASK (0x6000)
+
/*
* Range 0xFFFF0000 in ce_flags is divided into
* two parts: in-memory flags and on-disk ones.
@@ -166,6 +223,18 @@ struct cache_entry {
#define CE_EXTENDED_FLAGS (CE_INTENT_TO_ADD | CE_SKIP_WORKTREE)
/*
+ * Representation of the extended on-disk flags in the v5 format.
+ * They must not collide with the ordinary on-disk flags, and need to
+ * fit in 16 bits. Note however that v5 does not save the name
+ * length.
+ */
+#define CE_INTENT_TO_ADD_V5 (0x4000)
+#define CE_SKIP_WORKTREE_V5 (0x0800)
+#if (CE_VALID|CE_STAGEMASK) & (CE_INTENTTOADD_V5|CE_SKIPWORKTREE_V5)
+#error "v5 on-disk flags collide with ordinary on-disk flags"
+#endif
+
+/*
* Safeguard to avoid saving wrong flags:
* - CE_EXTENDED2 won't get saved until its semantic is known
* - Bits in 0x0000FFFF have been saved in ce_flags already
@@ -203,6 +272,8 @@ static inline unsigned create_ce_flags(unsigned stage)
#define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE)
#define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE)
+#define conflict_stage(c) ((CONFLICT_STAGEMASK & (c)->flags) >> CONFLICT_STAGESHIFT)
+
#define ce_permissions(mode) (((mode) & 0100) ? 0755 : 0644)
static inline unsigned int create_ce_mode(unsigned int mode)
{
@@ -249,6 +320,8 @@ static inline unsigned int canon_mode(unsigned int mode)
}
#define cache_entry_size(len) (offsetof(struct cache_entry,name) + (len) + 1)
+#define directory_entry_size(len) (offsetof(struct directory_entry,pathname) + (len) + 1)
+#define conflict_entry_size(len) (offsetof(struct conflict_entry,name) + (len) + 1)
struct index_state {
struct cache_entry **cache;
diff --git a/read-cache-v5.c b/read-cache-v5.c
new file mode 100644
index 0000000..2031969
--- /dev/null
+++ b/read-cache-v5.c
@@ -0,0 +1,556 @@
+#include "cache.h"
+#include "read-cache.h"
+#include "resolve-undo.h"
+#include "cache-tree.h"
+
+#define ptr_add(x,y) ((void *)(((char *)(x)) + (y)))
+
+struct cache_header {
+ unsigned int hdr_ndir;
+ unsigned int hdr_nfile;
+ unsigned int hdr_fblockoffset;
+ unsigned int hdr_nextension;
+};
+
+/*****************************************************************
+ * Index File I/O
+ *****************************************************************/
+
+struct ondisk_cache_entry {
+ unsigned short flags;
+ unsigned short mode;
+ struct cache_time mtime;
+ unsigned int size;
+ int stat_crc;
+ unsigned char sha1[20];
+};
+
+struct ondisk_directory_entry {
+ unsigned int foffset;
+ unsigned int cr;
+ unsigned int ncr;
+ unsigned int nsubtrees;
+ unsigned int nfiles;
+ unsigned int nentries;
+ unsigned char sha1[20];
+ unsigned short flags;
+};
+
+static int check_crc32(int initialcrc,
+ void *data,
+ size_t len,
+ unsigned int expected_crc)
+{
+ int crc;
+
+ crc = crc32(initialcrc, (Bytef*)data, len);
+ return crc == expected_crc;
+}
+
+static int match_stat_crc(struct stat *st, uint32_t expected_crc)
+{
+ uint32_t data, stat_crc = 0;
+ unsigned int ctimens = 0;
+
+ data = htonl(st->st_ctime);
+ stat_crc = crc32(0, (Bytef*)&data, 4);
+#ifdef USE_NSEC
+ ctimens = ST_CTIME_NSEC(*st);
+#endif
+ data = htonl(ctimens);
+ stat_crc = crc32(stat_crc, (Bytef*)&data, 4);
+ data = htonl(st->st_ino);
+ stat_crc = crc32(stat_crc, (Bytef*)&data, 4);
+ data = htonl(st->st_dev);
+ stat_crc = crc32(stat_crc, (Bytef*)&data, 4);
+ data = htonl(st->st_uid);
+ stat_crc = crc32(stat_crc, (Bytef*)&data, 4);
+ data = htonl(st->st_gid);
+ stat_crc = crc32(stat_crc, (Bytef*)&data, 4);
+
+ return stat_crc == expected_crc;
+}
+
+static int match_stat_basic(struct cache_entry *ce,
+ struct stat *st,
+ int changed)
+{
+
+ if (ce->ce_mtime.sec != (unsigned int)st->st_mtime)
+ changed |= MTIME_CHANGED;
+#ifdef USE_NSEC
+ if (ce->ce_mtime.nsec != ST_MTIME_NSEC(*st))
+ changed |= MTIME_CHANGED;
+#endif
+ if (ce->ce_size != (unsigned int)st->st_size)
+ changed |= DATA_CHANGED;
+
+ if (trust_ctime && ce->ce_stat_crc != 0 && !match_stat_crc(st, ce->ce_stat_crc)) {
+ changed |= OWNER_CHANGED;
+ changed |= INODE_CHANGED;
+ }
+ /* Racily smudged entry? */
+ if (ce->ce_flags & CE_SMUDGED) {
+ if (!changed && !is_empty_blob_sha1(ce->sha1) && ce_modified_check_fs(ce, st))
+ changed |= DATA_CHANGED;
+ }
+ return changed;
+}
+
+static int verify_hdr(void *mmap, unsigned long size)
+{
+ uint32_t *filecrc;
+ unsigned int header_size;
+ struct cache_version_header *hdr;
+ struct cache_header *hdr_v5;
+
+ if (size < sizeof(struct cache_version_header)
+ + sizeof (struct cache_header) + 4)
+ die("index file smaller than expected");
+
+ hdr = mmap;
+ hdr_v5 = ptr_add(mmap, sizeof(*hdr));
+ /* Size of the header + the size of the extensionoffsets */
+ header_size = sizeof(*hdr) + sizeof(*hdr_v5) + hdr_v5->hdr_nextension * 4;
+ /* Initialize crc */
+ filecrc = ptr_add(mmap, header_size);
+ if (!check_crc32(0, hdr, header_size, ntohl(*filecrc)))
+ return error("bad index file header crc signature");
+ return 0;
+}
+
+static struct cache_entry *cache_entry_from_ondisk(struct ondisk_cache_entry *ondisk,
+ struct directory_entry *de,
+ char *name,
+ size_t len,
+ size_t prefix_len)
+{
+ struct cache_entry *ce = xmalloc(cache_entry_size(len + de->de_pathlen));
+ int flags;
+
+ flags = ntoh_s(ondisk->flags);
+ ce->ce_ctime.sec = 0;
+ ce->ce_mtime.sec = ntoh_l(ondisk->mtime.sec);
+ ce->ce_ctime.nsec = 0;
+ ce->ce_mtime.nsec = ntoh_l(ondisk->mtime.nsec);
+ ce->ce_dev = 0;
+ ce->ce_ino = 0;
+ ce->ce_mode = ntoh_s(ondisk->mode);
+ ce->ce_uid = 0;
+ ce->ce_gid = 0;
+ ce->ce_size = ntoh_l(ondisk->size);
+ ce->ce_flags = flags & CE_STAGEMASK;
+ ce->ce_flags |= flags & CE_VALID;
+ ce->ce_flags |= flags & CE_SMUDGED;
+ if (flags & CE_INTENT_TO_ADD_V5)
+ ce->ce_flags |= CE_INTENT_TO_ADD;
+ if (flags & CE_SKIP_WORKTREE_V5)
+ ce->ce_flags |= CE_SKIP_WORKTREE;
+ ce->ce_stat_crc = ntoh_l(ondisk->stat_crc);
+ ce->ce_namelen = len + de->de_pathlen;
+ hashcpy(ce->sha1, ondisk->sha1);
+ memcpy(ce->name, de->pathname, de->de_pathlen);
+ memcpy(ce->name + de->de_pathlen, name, len);
+ ce->name[len + de->de_pathlen] = '\0';
+ return ce;
+}
+
+static struct directory_entry *directory_entry_from_ondisk(struct ondisk_directory_entry *ondisk,
+ const char *name,
+ size_t len)
+{
+ struct directory_entry *de = xmalloc(directory_entry_size(len));
+
+
+ memcpy(de->pathname, name, len);
+ de->pathname[len] = '\0';
+ de->de_flags = ntoh_s(ondisk->flags);
+ de->de_foffset = ntoh_l(ondisk->foffset);
+ de->de_cr = ntoh_l(ondisk->cr);
+ de->de_ncr = ntoh_l(ondisk->ncr);
+ de->de_nsubtrees = ntoh_l(ondisk->nsubtrees);
+ de->de_nfiles = ntoh_l(ondisk->nfiles);
+ de->de_nentries = ntoh_l(ondisk->nentries);
+ de->de_pathlen = len;
+ hashcpy(de->sha1, ondisk->sha1);
+ return de;
+}
+
+static struct conflict_part *conflict_part_from_ondisk(struct ondisk_conflict_part *ondisk)
+{
+ struct conflict_part *cp = xmalloc(sizeof(struct conflict_part));
+
+ cp->flags = ntoh_s(ondisk->flags);
+ cp->entry_mode = ntoh_s(ondisk->entry_mode);
+ hashcpy(cp->sha1, ondisk->sha1);
+ return cp;
+}
+
+static struct cache_entry *convert_conflict_part(struct conflict_part *cp,
+ char * name,
+ unsigned int len)
+{
+
+ struct cache_entry *ce = xmalloc(cache_entry_size(len));
+
+ ce->ce_ctime.sec = 0;
+ ce->ce_mtime.sec = 0;
+ ce->ce_ctime.nsec = 0;
+ ce->ce_mtime.nsec = 0;
+ ce->ce_dev = 0;
+ ce->ce_ino = 0;
+ ce->ce_mode = cp->entry_mode;
+ ce->ce_uid = 0;
+ ce->ce_gid = 0;
+ ce->ce_size = 0;
+ ce->ce_flags = conflict_stage(cp) << CE_STAGESHIFT;
+ ce->ce_stat_crc = 0;
+ ce->ce_namelen = len;
+ hashcpy(ce->sha1, cp->sha1);
+ memcpy(ce->name, name, len);
+ ce->name[len] = '\0';
+ return ce;
+}
+
+static struct directory_entry *read_directories(unsigned int *dir_offset,
+ unsigned int *dir_table_offset,
+ void *mmap,
+ int mmap_size)
+{
+ int i, ondisk_directory_size;
+ uint32_t *filecrc, *beginning, *end;
+ struct directory_entry *current = NULL;
+ struct ondisk_directory_entry *disk_de;
+ struct directory_entry *de;
+ unsigned int data_len, len;
+ char *name;
+
+ ondisk_directory_size = sizeof(disk_de->flags)
+ + sizeof(disk_de->foffset)
+ + sizeof(disk_de->cr)
+ + sizeof(disk_de->ncr)
+ + sizeof(disk_de->nsubtrees)
+ + sizeof(disk_de->nfiles)
+ + sizeof(disk_de->nentries)
+ + sizeof(disk_de->sha1);
+ name = ptr_add(mmap, *dir_offset);
+ beginning = ptr_add(mmap, *dir_table_offset);
+ end = ptr_add(mmap, *dir_table_offset + 4);
+ len = ntoh_l(*end) - ntoh_l(*beginning) - ondisk_directory_size - 5;
+ disk_de = ptr_add(mmap, *dir_offset + len + 1);
+ de = directory_entry_from_ondisk(disk_de, name, len);
+ de->next = NULL;
+
+ /* Length of pathname + nul byte for termination + size of
+ * members of ondisk_directory_entry. (Just using the size
+ * of the stuct doesn't work, because there may be padding
+ * bytes for the struct)
+ */
+ data_len = len + 1 + ondisk_directory_size;
+
+ filecrc = ptr_add(mmap, *dir_offset + data_len);
+ if (!check_crc32(0, ptr_add(mmap, *dir_offset), data_len, ntoh_l(*filecrc)))
+ goto unmap;
+
+ *dir_table_offset += 4;
+ *dir_offset += data_len + 4; /* crc code */
+
+ current = de;
+ for (i = 0; i < de->de_nsubtrees; i++) {
+ current->next = read_directories(dir_offset, dir_table_offset,
+ mmap, mmap_size);
+ while (current->next)
+ current = current->next;
+ }
+
+ return de;
+unmap:
+ munmap(mmap, mmap_size);
+ die("directory crc doesn't match for '%s'", de->pathname);
+}
+
+static int read_entry(struct cache_entry **ce, struct directory_entry *de,
+ unsigned long *entry_offset,
+ void **mmap, unsigned long mmap_size,
+ unsigned int *foffsetblock)
+{
+ int len, offset_to_offset;
+ char *name;
+ uint32_t foffsetblockcrc;
+ uint32_t *filecrc, *beginning, *end;
+ struct ondisk_cache_entry *disk_ce;
+
+ name = ptr_add(*mmap, *entry_offset);
+ beginning = ptr_add(*mmap, *foffsetblock);
+ end = ptr_add(*mmap, *foffsetblock + 4);
+ len = ntoh_l(*end) - ntoh_l(*beginning) - sizeof(struct ondisk_cache_entry) - 5;
+ disk_ce = ptr_add(*mmap, *entry_offset + len + 1);
+ *ce = cache_entry_from_ondisk(disk_ce, de, name, len, de->de_pathlen);
+ filecrc = ptr_add(*mmap, *entry_offset + len + 1 + sizeof(*disk_ce));
+ offset_to_offset = htonl(*foffsetblock);
+ foffsetblockcrc = crc32(0, (Bytef*)&offset_to_offset, 4);
+ if (!check_crc32(foffsetblockcrc,
+ ptr_add(*mmap, *entry_offset), len + 1 + sizeof(*disk_ce),
+ ntoh_l(*filecrc)))
+ return -1;
+
+ /* TODO: return error if crc code is wrong */
+ *entry_offset += len + 1 + sizeof(*disk_ce) + 4;
+ return 0;
+}
+
+static void ce_queue_push(struct cache_entry **head,
+ struct cache_entry **tail,
+ struct cache_entry *ce)
+{
+ if (!*head) {
+ *head = *tail = ce;
+ (*tail)->next = NULL;
+ return;
+ }
+
+ (*tail)->next = ce;
+ ce->next = NULL;
+ *tail = (*tail)->next;
+}
+
+static void conflict_entry_push(struct conflict_entry **head,
+ struct conflict_entry **tail,
+ struct conflict_entry *conflict_entry)
+{
+ if (!*head) {
+ *head = *tail = conflict_entry;
+ (*tail)->next = NULL;
+ return;
+ }
+
+ (*tail)->next = conflict_entry;
+ conflict_entry->next = NULL;
+ *tail = (*tail)->next;
+}
+
+static struct cache_entry *ce_queue_pop(struct cache_entry **head)
+{
+ struct cache_entry *ce;
+
+ ce = *head;
+ *head = (*head)->next;
+ return ce;
+}
+
+static void conflict_part_head_remove(struct conflict_part **head)
+{
+ struct conflict_part *to_free;
+
+ to_free = *head;
+ *head = (*head)->next;
+ free(to_free);
+}
+
+static void conflict_entry_head_remove(struct conflict_entry **head)
+{
+ struct conflict_entry *to_free;
+
+ to_free = *head;
+ *head = (*head)->next;
+ free(to_free);
+}
+
+struct conflict_entry *create_new_conflict(char *name, int len, int pathlen)
+{
+ struct conflict_entry *conflict_entry;
+
+ if (pathlen)
+ pathlen++;
+ conflict_entry = xmalloc(conflict_entry_size(len));
+ conflict_entry->entries = NULL;
+ conflict_entry->nfileconflicts = 0;
+ conflict_entry->namelen = len;
+ memcpy(conflict_entry->name, name, len);
+ conflict_entry->name[len] = '\0';
+ conflict_entry->pathlen = pathlen;
+ conflict_entry->next = NULL;
+
+ return conflict_entry;
+}
+
+void add_part_to_conflict_entry(struct directory_entry *de,
+ struct conflict_entry *entry,
+ struct conflict_part *conflict_part)
+{
+
+ struct conflict_part *conflict_search;
+
+ entry->nfileconflicts++;
+ de->conflict_size += sizeof(struct ondisk_conflict_part);
+ if (!entry->entries)
+ entry->entries = conflict_part;
+ else {
+ conflict_search = entry->entries;
+ while (conflict_search->next)
+ conflict_search = conflict_search->next;
+ conflict_search->next = conflict_part;
+ }
+}
+
+static int read_conflicts(struct conflict_entry **head,
+ struct directory_entry *de,
+ void **mmap, unsigned long mmap_size)
+{
+ struct conflict_entry *tail;
+ unsigned int croffset, i;
+ char *full_name;
+
+ croffset = de->de_cr;
+ tail = NULL;
+ for (i = 0; i < de->de_ncr; i++) {
+ struct conflict_entry *conflict_new;
+ unsigned int len, *nfileconflicts;
+ char *name;
+ void *crc_start;
+ int k, offset;
+ uint32_t *filecrc;
+
+ offset = croffset;
+ crc_start = ptr_add(*mmap, offset);
+ name = ptr_add(*mmap, offset);
+ len = strlen(name);
+ offset += len + 1;
+ nfileconflicts = ptr_add(*mmap, offset);
+ offset += 4;
+
+ full_name = xmalloc(sizeof(char) * (len + de->de_pathlen));
+ memcpy(full_name, de->pathname, de->de_pathlen);
+ memcpy(full_name + de->de_pathlen, name, len);
+ conflict_new = create_new_conflict(full_name,
+ len + de->de_pathlen, de->de_pathlen);
+ for (k = 0; k < ntoh_l(*nfileconflicts); k++) {
+ struct ondisk_conflict_part *ondisk;
+ struct conflict_part *cp;
+
+ ondisk = ptr_add(*mmap, offset);
+ cp = conflict_part_from_ondisk(ondisk);
+ cp->next = NULL;
+ add_part_to_conflict_entry(de, conflict_new, cp);
+ offset += sizeof(struct ondisk_conflict_part);
+ }
+ filecrc = ptr_add(*mmap, offset);
+ free(full_name);
+ if (!check_crc32(0, crc_start,
+ len + 1 + 4 + conflict_new->nfileconflicts
+ * sizeof(struct ondisk_conflict_part),
+ ntoh_l(*filecrc)))
+ return -1;
+ croffset = offset + 4;
+ conflict_entry_push(head, &tail, conflict_new);
+ }
+ return 0;
+}
+
+static int read_entries(struct index_state *istate, struct directory_entry **de,
+ unsigned long *entry_offset, void **mmap,
+ unsigned long mmap_size, int *nr,
+ unsigned int *foffsetblock)
+{
+ struct cache_entry *head = NULL, *tail = NULL;
+ struct conflict_entry *conflict_queue;
+ struct cache_entry *ce;
+ int i;
+
+ conflict_queue = NULL;
+ if (read_conflicts(&conflict_queue, *de, mmap, mmap_size) < 0)
+ return -1;
+ for (i = 0; i < (*de)->de_nfiles; i++) {
+ if (read_entry(&ce,
+ *de,
+ entry_offset,
+ mmap,
+ mmap_size,
+ foffsetblock) < 0)
+ return -1;
+ ce_queue_push(&head, &tail, ce);
+ *foffsetblock += 4;
+
+ /* Add the conflicted entries at the end of the index file
+ * to the in memory format
+ */
+ if (conflict_queue &&
+ (conflict_queue->entries->flags & CONFLICT_CONFLICTED) != 0 &&
+ !cache_name_compare(conflict_queue->name, conflict_queue->namelen,
+ ce->name, ce_namelen(ce))) {
+ struct conflict_part *cp;
+ cp = conflict_queue->entries;
+ cp = cp->next;
+ while (cp) {
+ ce = convert_conflict_part(cp,
+ conflict_queue->name,
+ conflict_queue->namelen);
+ ce_queue_push(&head, &tail, ce);
+ conflict_part_head_remove(&cp);
+ }
+ conflict_entry_head_remove(&conflict_queue);
+ }
+ }
+
+ *de = (*de)->next;
+
+ while (head) {
+ if (*de != NULL
+ && strcmp(head->name, (*de)->pathname) > 0) {
+ read_entries(istate,
+ de,
+ entry_offset,
+ mmap,
+ mmap_size,
+ nr,
+ foffsetblock);
+ } else {
+ ce = ce_queue_pop(&head);
+ set_index_entry(istate, *nr, ce);
+ (*nr)++;
+ }
+ }
+ return 0;
+}
+
+static int read_index_v5(struct index_state *istate, void *mmap, int mmap_size)
+{
+ unsigned long entry_offset;
+ unsigned int dir_offset, dir_table_offset;
+ struct cache_version_header *hdr;
+ struct cache_header *hdr_v5;
+ struct directory_entry *root_directory, *de;
+ int nr;
+ unsigned int foffsetblock;
+
+ hdr = mmap;
+ hdr_v5 = ptr_add(mmap, sizeof(*hdr));
+ istate->version = ntohl(hdr->hdr_version);
+ istate->cache_nr = ntohl(hdr_v5->hdr_nfile);
+ istate->cache_alloc = alloc_nr(istate->cache_nr);
+ istate->cache = xcalloc(istate->cache_alloc, sizeof(struct cache_entry *));
+ istate->initialized = 1;
+
+ /* Skip size of the header + crc sum + size of offsets */
+ dir_offset = sizeof(*hdr) + sizeof(*hdr_v5) + 4 + (ntohl(hdr_v5->hdr_ndir) + 1) * 4;
+ dir_table_offset = sizeof(*hdr) + sizeof(*hdr_v5) + 4;
+ root_directory = read_directories(&dir_offset, &dir_table_offset, mmap, mmap_size);
+
+ entry_offset = ntohl(hdr_v5->hdr_fblockoffset);
+
+ nr = 0;
+ foffsetblock = dir_offset;
+ de = root_directory;
+ while (de)
+ if (read_entries(istate, &de, &entry_offset,
+ &mmap, mmap_size, &nr, &foffsetblock) < 0)
+ return -1;
+ return 0;
+}
+
+struct index_ops v5_ops = {
+ match_stat_basic,
+ verify_hdr,
+ read_index_v5,
+ NULL
+};
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 07/13] Read resolve-undo data
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
` (5 preceding siblings ...)
2012-08-16 9:58 ` [PATCH/RFC v4 06/13] Read index-v5 Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 08/13] Read cache-tree in index-v5 Thomas Gummerer
` (5 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Make git read the resolve-undo data from the index.
Since the resolve-undo data is joined with the conflicts in
the ondisk format of the index file version 5, conflicts and
resolved data is read at the same time, and the resolve-undo
data is then converted to the in-memory format.
Helped-by: Thomas Rast <trast@student.ethz.ch>
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
read-cache-v5.c | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/read-cache-v5.c b/read-cache-v5.c
index 2031969..fb549de 100644
--- a/read-cache-v5.c
+++ b/read-cache-v5.c
@@ -1,5 +1,6 @@
#include "cache.h"
#include "read-cache.h"
+#include "string-list.h"
#include "resolve-undo.h"
#include "cache-tree.h"
@@ -447,6 +448,42 @@ static int read_conflicts(struct conflict_entry **head,
return 0;
}
+static void resolve_undo_convert_v5(struct index_state *istate,
+ struct conflict_entry *ce)
+{
+ int i;
+
+ while (ce) {
+ struct string_list_item *lost;
+ struct resolve_undo_info *ui;
+ struct conflict_part *cp;
+
+ if (ce->entries && (ce->entries->flags & CONFLICT_CONFLICTED) != 0) {
+ ce = ce->next;
+ continue;
+ }
+ if (!istate->resolve_undo) {
+ istate->resolve_undo = xcalloc(1, sizeof(struct string_list));
+ istate->resolve_undo->strdup_strings = 1;
+ }
+
+ lost = string_list_insert(istate->resolve_undo, ce->name);
+ if (!lost->util)
+ lost->util = xcalloc(1, sizeof(*ui));
+ ui = lost->util;
+
+ cp = ce->entries;
+ for (i = 0; i < 3; i++)
+ ui->mode[i] = 0;
+ while (cp) {
+ ui->mode[conflict_stage(cp) - 1] = cp->entry_mode;
+ hashcpy(ui->sha1[conflict_stage(cp) - 1], cp->sha1);
+ cp = cp->next;
+ }
+ ce = ce->next;
+ }
+}
+
static int read_entries(struct index_state *istate, struct directory_entry **de,
unsigned long *entry_offset, void **mmap,
unsigned long mmap_size, int *nr,
@@ -460,6 +497,7 @@ static int read_entries(struct index_state *istate, struct directory_entry **de,
conflict_queue = NULL;
if (read_conflicts(&conflict_queue, *de, mmap, mmap_size) < 0)
return -1;
+ resolve_undo_convert_v5(istate, conflict_queue);
for (i = 0; i < (*de)->de_nfiles; i++) {
if (read_entry(&ce,
*de,
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 08/13] Read cache-tree in index-v5
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
` (6 preceding siblings ...)
2012-08-16 9:58 ` [PATCH/RFC v4 07/13] Read resolve-undo data Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 09/13] Write index-v5 Thomas Gummerer
` (4 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Since the cache-tree data is saved as part of the directory data,
we already read it at the beginning of the index. The cache-tree
is only converted from this directory data.
The cache-tree data is arranged in a tree, with the children sorted by
pathlen at each node, while the ondisk format is sorted lexically.
So we have to rebuild this format from the on-disk directory list.
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
cache-tree.c | 2 +-
cache-tree.h | 6 ++++
read-cache-v5.c | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 105 insertions(+), 1 deletion(-)
diff --git a/cache-tree.c b/cache-tree.c
index 28ed657..61544d8 100644
--- a/cache-tree.c
+++ b/cache-tree.c
@@ -31,7 +31,7 @@ void cache_tree_free(struct cache_tree **it_p)
*it_p = NULL;
}
-static int subtree_name_cmp(const char *one, int onelen,
+int subtree_name_cmp(const char *one, int onelen,
const char *two, int twolen)
{
if (onelen < twolen)
diff --git a/cache-tree.h b/cache-tree.h
index d8cb2e9..7416007 100644
--- a/cache-tree.h
+++ b/cache-tree.h
@@ -20,10 +20,16 @@ struct cache_tree {
struct cache_tree_sub **down;
};
+struct directory_queue {
+ struct directory_queue *down;
+ struct directory_entry *de;
+};
+
struct cache_tree *cache_tree(void);
void cache_tree_free(struct cache_tree **);
void cache_tree_invalidate_path(struct cache_tree *, const char *);
struct cache_tree_sub *cache_tree_sub(struct cache_tree *, const char *);
+int subtree_name_cmp(const char *, int, const char *, int);
void cache_tree_write(struct strbuf *, struct cache_tree *root);
struct cache_tree *cache_tree_read(const char *buffer, unsigned long size);
diff --git a/read-cache-v5.c b/read-cache-v5.c
index fb549de..b497726 100644
--- a/read-cache-v5.c
+++ b/read-cache-v5.c
@@ -448,6 +448,103 @@ static int read_conflicts(struct conflict_entry **head,
return 0;
}
+static struct cache_tree *convert_one(struct directory_queue *queue, int dirnr)
+{
+ int i, subtree_nr;
+ struct cache_tree *it;
+ struct directory_queue *down;
+
+ it = cache_tree();
+ it->entry_count = queue[dirnr].de->de_nentries;
+ subtree_nr = queue[dirnr].de->de_nsubtrees;
+ if (0 <= it->entry_count)
+ hashcpy(it->sha1, queue[dirnr].de->sha1);
+
+ /*
+ * Just a heuristic -- we do not add directories that often but
+ * we do not want to have to extend it immediately when we do,
+ * hence +2.
+ */
+ it->subtree_alloc = subtree_nr + 2;
+ it->down = xcalloc(it->subtree_alloc, sizeof(struct cache_tree_sub *));
+ down = queue[dirnr].down;
+ for (i = 0; i < subtree_nr; i++) {
+ struct cache_tree *sub;
+ struct cache_tree_sub *subtree;
+ char *buf, *name;
+
+ name = "";
+ buf = strtok(down[i].de->pathname, "/");
+ while (buf) {
+ name = buf;
+ buf = strtok(NULL, "/");
+ }
+ sub = convert_one(down, i);
+ if(!sub)
+ goto free_return;
+ subtree = cache_tree_sub(it, name);
+ subtree->cache_tree = sub;
+ }
+ if (subtree_nr != it->subtree_nr)
+ die("cache-tree: internal error");
+ return it;
+ free_return:
+ cache_tree_free(&it);
+ return NULL;
+}
+
+static int compare_cache_tree_elements(const void *a, const void *b)
+{
+ const struct directory_entry *de1, *de2;
+
+ de1 = ((const struct directory_queue *)a)->de;
+ de2 = ((const struct directory_queue *)b)->de;
+ return subtree_name_cmp(de1->pathname, de1->de_pathlen,
+ de2->pathname, de2->de_pathlen);
+}
+
+static struct directory_entry *sort_directories(struct directory_entry *de,
+ struct directory_queue *queue)
+{
+ int i, nsubtrees;
+
+ nsubtrees = de->de_nsubtrees;
+ for (i = 0; i < nsubtrees; i++) {
+ struct directory_entry *new_de;
+ de = de->next;
+ new_de = xmalloc(directory_entry_size(de->de_pathlen));
+ memcpy(new_de, de, directory_entry_size(de->de_pathlen));
+ queue[i].de = new_de;
+ if (de->de_nsubtrees) {
+ queue[i].down = xcalloc(de->de_nsubtrees,
+ sizeof(struct directory_queue));
+ de = sort_directories(de,
+ queue[i].down);
+ }
+ }
+ qsort(queue, nsubtrees, sizeof(struct directory_queue),
+ compare_cache_tree_elements);
+ return de;
+}
+
+/*
+ * This function modifys the directory argument that is given to it.
+ * Don't use it if the directory entries are still needed after.
+ */
+static struct cache_tree *cache_tree_convert_v5(struct directory_entry *de)
+{
+ struct directory_queue *queue;
+
+ if (!de->de_nentries)
+ return NULL;
+ queue = xcalloc(1, sizeof(struct directory_queue));
+ queue[0].de = de;
+ queue[0].down = xcalloc(de->de_nsubtrees, sizeof(struct directory_queue));
+
+ sort_directories(de, queue[0].down);
+ return convert_one(queue, 0);
+}
+
static void resolve_undo_convert_v5(struct index_state *istate,
struct conflict_entry *ce)
{
@@ -583,6 +680,7 @@ static int read_index_v5(struct index_state *istate, void *mmap, int mmap_size)
if (read_entries(istate, &de, &entry_offset,
&mmap, mmap_size, &nr, &foffsetblock) < 0)
return -1;
+ istate->cache_tree = cache_tree_convert_v5(root_directory);
return 0;
}
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 09/13] Write index-v5
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
` (7 preceding siblings ...)
2012-08-16 9:58 ` [PATCH/RFC v4 08/13] Read cache-tree in index-v5 Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 10/13] Write index-v5 cache-tree data Thomas Gummerer
` (3 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Write the index version 5 file format to disk. This version doesn't
write the cache-tree data and resolve-undo data to the file.
The main work is done when filtering out the directories from the
current in-memory format, where in the same turn also the conflicts
and the file data is calculated.
Helped-by: Nguyen Thai Ngoc Duy <pclouds@gmail.com>
Helped-by: Thomas Rast <trast@student.ethz.ch>
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
cache.h | 10 +-
read-cache-v5.c | 589 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-
read-cache.c | 19 +-
read-cache.h | 3 +
4 files changed, 611 insertions(+), 10 deletions(-)
diff --git a/cache.h b/cache.h
index bec9402..72c88f1 100644
--- a/cache.h
+++ b/cache.h
@@ -98,7 +98,7 @@ unsigned long git_deflate_bound(git_zstream *, unsigned long);
#define CACHE_SIGNATURE 0x44495243 /* "DIRC" */
#define INDEX_FORMAT_LB 2
-#define INDEX_FORMAT_UB 4
+#define INDEX_FORMAT_UB 5
/*
* The "cache_time" is just the low 32 bits of the
@@ -511,6 +511,7 @@ extern int verify_path(const char *path);
extern struct cache_entry *index_name_exists(struct index_state *istate, const char *name, int namelen, int igncase);
extern int index_name_stage_pos(const struct index_state *, const char *name, int namelen, int stage);
extern int index_name_pos(const struct index_state *, const char *name, int namelen);
+extern struct directory_entry *init_directory_entry(char *pathname, int len);
#define ADD_CACHE_OK_TO_ADD 1 /* Ok to add */
#define ADD_CACHE_OK_TO_REPLACE 2 /* Ok to replace file/directory */
#define ADD_CACHE_SKIP_DFCHECK 4 /* Ok to skip DF conflict checks */
@@ -1245,6 +1246,13 @@ static inline ssize_t write_str_in_full(int fd, const char *str)
return write_in_full(fd, str, strlen(str));
}
+/* index-v5 helper functions */
+extern char *super_directory(const char *filename);
+extern void insert_directory_entry(struct directory_entry *, struct hash_table *, int *, unsigned int *, uint32_t);
+extern void add_conflict_to_directory_entry(struct directory_entry *, struct conflict_entry *);
+extern void add_part_to_conflict_entry(struct directory_entry *, struct conflict_entry *, struct conflict_part *);
+extern struct conflict_entry *create_new_conflict(char *, int, int);
+
/* pager.c */
extern void setup_pager(void);
extern const char *pager_program;
diff --git a/read-cache-v5.c b/read-cache-v5.c
index b497726..a2afc94 100644
--- a/read-cache-v5.c
+++ b/read-cache-v5.c
@@ -684,9 +684,596 @@ static int read_index_v5(struct index_state *istate, void *mmap, int mmap_size)
return 0;
}
+#define WRITE_BUFFER_SIZE 8192
+static unsigned char write_buffer[WRITE_BUFFER_SIZE];
+static unsigned long write_buffer_len;
+
+static int ce_write_flush(int fd)
+{
+ unsigned int buffered = write_buffer_len;
+ if (buffered) {
+ if (write_in_full(fd, write_buffer, buffered) != buffered)
+ return -1;
+ write_buffer_len = 0;
+ }
+ return 0;
+}
+
+static int ce_write(uint32_t *crc, int fd, void *data, unsigned int len)
+{
+ if (crc)
+ *crc = crc32(*crc, (Bytef*)data, len);
+ while (len) {
+ unsigned int buffered = write_buffer_len;
+ unsigned int partial = WRITE_BUFFER_SIZE - buffered;
+ if (partial > len)
+ partial = len;
+ memcpy(write_buffer + buffered, data, partial);
+ buffered += partial;
+ if (buffered == WRITE_BUFFER_SIZE) {
+ write_buffer_len = buffered;
+ if (ce_write_flush(fd))
+ return -1;
+ buffered = 0;
+ }
+ write_buffer_len = buffered;
+ len -= partial;
+ data = (char *) data + partial;
+ }
+ return 0;
+}
+
+static int ce_flush(int fd)
+{
+ unsigned int left = write_buffer_len;
+
+ if (left)
+ write_buffer_len = 0;
+
+ if (write_in_full(fd, write_buffer, left) != left)
+ return -1;
+
+ return 0;
+}
+
+static void ce_smudge_racily_clean_entry(struct cache_entry *ce)
+{
+ /*
+ * This method shall only be called if the timestamp of ce
+ * is racy (check with is_racy_timestamp). If the timestamp
+ * is racy, the writer will set the CE_SMUDGED flag.
+ *
+ * The reader (match_stat_basic) will then take care
+ * of checking if the entry is really changed or not, by
+ * taking into account the size and the stat_crc and if
+ * that hasn't changed checking the sha1.
+ */
+ ce->ce_flags |= CE_SMUDGED;
+}
+
+char *super_directory(const char *filename)
+{
+ char *slash;
+
+ slash = strrchr(filename, '/');
+ if (slash)
+ return xmemdupz(filename, slash-filename);
+ return NULL;
+}
+
+struct directory_entry *init_directory_entry(char *pathname, int len)
+{
+ struct directory_entry *de = xmalloc(directory_entry_size(len));
+
+ memcpy(de->pathname, pathname, len);
+ de->pathname[len] = '\0';
+ de->de_flags = 0;
+ de->de_foffset = 0;
+ de->de_cr = 0;
+ de->de_ncr = 0;
+ de->de_nsubtrees = 0;
+ de->de_nfiles = 0;
+ de->de_nentries = 0;
+ memset(de->sha1, 0, 20);
+ de->de_pathlen = len;
+ de->next = NULL;
+ de->next_hash = NULL;
+ de->ce = NULL;
+ de->ce_last = NULL;
+ de->conflict = NULL;
+ de->conflict_last = NULL;
+ de->conflict_size = 0;
+ return de;
+}
+
+static void ondisk_from_directory_entry(struct directory_entry *de,
+ struct ondisk_directory_entry *ondisk)
+{
+ ondisk->foffset = htonl(de->de_foffset);
+ ondisk->cr = htonl(de->de_cr);
+ ondisk->ncr = htonl(de->de_ncr);
+ ondisk->nsubtrees = htonl(de->de_nsubtrees);
+ ondisk->nfiles = htonl(de->de_nfiles);
+ ondisk->nentries = htonl(de->de_nentries);
+ hashcpy(ondisk->sha1, de->sha1);
+ ondisk->flags = htons(de->de_flags);
+}
+
+static struct conflict_part *conflict_part_from_inmemory(struct cache_entry *ce)
+{
+ struct conflict_part *conflict;
+ int flags;
+
+ conflict = xmalloc(sizeof(struct conflict_part));
+ flags = CONFLICT_CONFLICTED;
+ flags |= ce_stage(ce) << CONFLICT_STAGESHIFT;
+ conflict->flags = flags;
+ conflict->entry_mode = ce->ce_mode;
+ conflict->next = NULL;
+ hashcpy(conflict->sha1, ce->sha1);
+ return conflict;
+}
+
+static void conflict_to_ondisk(struct conflict_part *cp,
+ struct ondisk_conflict_part *ondisk)
+{
+ ondisk->flags = htons(cp->flags);
+ ondisk->entry_mode = htons(cp->entry_mode);
+ hashcpy(ondisk->sha1, cp->sha1);
+}
+
+void add_conflict_to_directory_entry(struct directory_entry *de,
+ struct conflict_entry *conflict_entry)
+{
+ de->de_ncr++;
+ de->conflict_size += conflict_entry->namelen + 1 + 8 - conflict_entry->pathlen;
+ conflict_entry_push(&de->conflict, &de->conflict_last, conflict_entry);
+}
+
+void insert_directory_entry(struct directory_entry *de,
+ struct hash_table *table,
+ int *total_dir_len,
+ unsigned int *ndir,
+ uint32_t crc)
+{
+ struct directory_entry *insert;
+
+ insert = (struct directory_entry *)insert_hash(crc, de, table);
+ if (insert) {
+ de->next_hash = insert->next_hash;
+ insert->next_hash = de;
+ }
+ (*ndir)++;
+ if (de->de_pathlen == 0)
+ (*total_dir_len)++;
+ else
+ *total_dir_len += de->de_pathlen + 2;
+}
+
+static struct conflict_entry *create_conflict_entry_from_ce(struct cache_entry *ce,
+ int pathlen)
+{
+ return create_new_conflict(ce->name, ce_namelen(ce), pathlen);
+}
+
+static struct directory_entry *compile_directory_data(struct index_state *istate,
+ int nfile,
+ unsigned int *ndir,
+ int *non_conflicted,
+ int *total_dir_len,
+ int *total_file_len)
+{
+ int i, dir_len = -1;
+ char *dir;
+ struct directory_entry *de, *current, *search, *found, *new, *previous_entry;
+ struct cache_entry **cache = istate->cache;
+ struct conflict_entry *conflict_entry;
+ struct hash_table table;
+ uint32_t crc;
+
+ init_hash(&table);
+ de = init_directory_entry("", 0);
+ current = de;
+ *ndir = 1;
+ *total_dir_len = 1;
+ crc = crc32(0, (Bytef*)de->pathname, de->de_pathlen);
+ insert_hash(crc, de, &table);
+ conflict_entry = NULL;
+ for (i = 0; i < nfile; i++) {
+ int new_entry;
+ if (cache[i]->ce_flags & CE_REMOVE)
+ continue;
+
+ new_entry = !ce_stage(cache[i]) || !conflict_entry
+ || cache_name_compare(conflict_entry->name, conflict_entry->namelen,
+ cache[i]->name, ce_namelen(cache[i]));
+ if (new_entry)
+ (*non_conflicted)++;
+ if (dir_len < 0 || strncmp(cache[i]->name, dir, dir_len)
+ || cache[i]->name[dir_len] != '/'
+ || strchr(cache[i]->name + dir_len + 1, '/')) {
+ dir = super_directory(cache[i]->name);
+ if (!dir)
+ dir_len = 0;
+ else
+ dir_len = strlen(dir);
+ crc = crc32(0, (Bytef*)dir, dir_len);
+ found = lookup_hash(crc, &table);
+ search = found;
+ while (search && dir_len != 0 && strcmp(dir, search->pathname) != 0)
+ search = search->next_hash;
+ }
+ previous_entry = current;
+ if (!search || !found) {
+ new = init_directory_entry(dir, dir_len);
+ current->next = new;
+ current = current->next;
+ insert_directory_entry(new, &table, total_dir_len, ndir, crc);
+ search = current;
+ }
+ if (new_entry) {
+ search->de_nfiles++;
+ *total_file_len += ce_namelen(cache[i]) + 1;
+ if (search->de_pathlen)
+ *total_file_len -= search->de_pathlen + 1;
+ ce_queue_push(&(search->ce), &(search->ce_last), cache[i]);
+ }
+ if (ce_stage(cache[i]) > 0) {
+ struct conflict_part *conflict_part;
+ if (new_entry) {
+ conflict_entry = create_conflict_entry_from_ce(cache[i], search->de_pathlen);
+ add_conflict_to_directory_entry(search, conflict_entry);
+ }
+ conflict_part = conflict_part_from_inmemory(cache[i]);
+ add_part_to_conflict_entry(search, conflict_entry, conflict_part);
+ }
+ if (dir && !found) {
+ struct directory_entry *no_subtrees;
+
+ no_subtrees = current;
+ dir = super_directory(dir);
+ if (dir)
+ dir_len = strlen(dir);
+ else
+ dir_len = 0;
+ crc = crc32(0, (Bytef*)dir, dir_len);
+ found = lookup_hash(crc, &table);
+ while (!found) {
+ new = init_directory_entry(dir, dir_len);
+ new->de_nsubtrees = 1;
+ new->next = no_subtrees;
+ no_subtrees = new;
+ insert_directory_entry(new, &table, total_dir_len, ndir, crc);
+ dir = super_directory(dir);
+ if (!dir)
+ dir_len = 0;
+ else
+ dir_len = strlen(dir);
+ crc = crc32(0, (Bytef*)dir, dir_len);
+ found = lookup_hash(crc, &table);
+ }
+ search = found;
+ while (search->next_hash && strcmp(dir, search->pathname) != 0)
+ search = search->next_hash;
+ if (search)
+ found = search;
+ found->de_nsubtrees++;
+ previous_entry->next = no_subtrees;
+ }
+ }
+ return de;
+}
+
+static void ondisk_from_cache_entry(struct cache_entry *ce,
+ struct ondisk_cache_entry *ondisk)
+{
+ unsigned int flags;
+
+ flags = ce->ce_flags & CE_STAGEMASK;
+ flags |= ce->ce_flags & CE_VALID;
+ flags |= ce->ce_flags & CE_SMUDGED;
+ if (ce->ce_flags & CE_INTENT_TO_ADD)
+ flags |= CE_INTENT_TO_ADD_V5;
+ if (ce->ce_flags & CE_SKIP_WORKTREE)
+ flags |= CE_SKIP_WORKTREE_V5;
+ ondisk->flags = htons(flags);
+ ondisk->mode = htons(ce->ce_mode);
+ ondisk->mtime.sec = htonl(ce->ce_mtime.sec);
+#ifdef USE_NSEC
+ ondisk->mtime.nsec = htonl(ce->ce_mtime.nsec);
+#else
+ ondisk->mtime.nsec = 0;
+#endif
+ ondisk->size = htonl(ce->ce_size);
+ if (!ce->ce_stat_crc)
+ ce->ce_stat_crc = calculate_stat_crc(ce);
+ ondisk->stat_crc = htonl(ce->ce_stat_crc);
+ hashcpy(ondisk->sha1, ce->sha1);
+}
+
+static int write_directories(struct directory_entry *de, int fd, int conflict_offset)
+{
+ struct directory_entry *current;
+ struct ondisk_directory_entry ondisk;
+ int current_offset, offset_write, ondisk_size, foffset;
+ uint32_t crc;
+
+ /*
+ * This is needed because the compiler aligns structs to sizes multipe
+ * of 4
+ */
+ ondisk_size = sizeof(ondisk.flags)
+ + sizeof(ondisk.foffset)
+ + sizeof(ondisk.cr)
+ + sizeof(ondisk.ncr)
+ + sizeof(ondisk.nsubtrees)
+ + sizeof(ondisk.nfiles)
+ + sizeof(ondisk.nentries)
+ + sizeof(ondisk.sha1);
+ current = de;
+ current_offset = 0;
+ foffset = 0;
+ while (current) {
+ int pathlen;
+
+ offset_write = htonl(current_offset);
+ if (ce_write(NULL, fd, &offset_write, 4) < 0)
+ return -1;
+ if (current->de_pathlen == 0)
+ pathlen = 0;
+ else
+ pathlen = current->de_pathlen + 1;
+ current_offset += pathlen + 1 + ondisk_size + 4;
+ current = current->next;
+ }
+ /*
+ * Write one more offset, which points to the end of the entries,
+ * because we use it for calculating the dir length, instead of
+ * using strlen.
+ */
+ offset_write = htonl(current_offset);
+ if (ce_write(NULL, fd, &offset_write, 4) < 0)
+ return -1;
+ current = de;
+ while (current) {
+ crc = 0;
+ if (current->de_pathlen == 0) {
+ if (ce_write(&crc, fd, current->pathname, 1) < 0)
+ return -1;
+ } else {
+ char *path;
+ path = xmalloc(sizeof(char) * (current->de_pathlen + 2));
+ memcpy(path, current->pathname, current->de_pathlen);
+ memcpy(path + current->de_pathlen, "/\0", 2);
+ if (ce_write(&crc, fd, path, current->de_pathlen + 2) < 0)
+ return -1;
+ }
+ current->de_foffset = foffset;
+ current->de_cr = conflict_offset;
+ ondisk_from_directory_entry(current, &ondisk);
+ if (ce_write(&crc, fd, &ondisk, ondisk_size) < 0)
+ return -1;
+ crc = htonl(crc);
+ if (ce_write(NULL, fd, &crc, 4) < 0)
+ return -1;
+ conflict_offset += current->conflict_size;
+ foffset += current->de_nfiles * 4;
+ current = current->next;
+ }
+ return 0;
+}
+
+static int write_entries(struct index_state *istate,
+ struct directory_entry *de,
+ int entries,
+ int fd,
+ int offset_to_offset)
+{
+ int offset, offset_write, ondisk_size;
+ struct directory_entry *current;
+
+ offset = 0;
+ ondisk_size = sizeof(struct ondisk_cache_entry);
+ current = de;
+ while (current) {
+ int pathlen;
+ struct cache_entry *ce = current->ce;
+
+ if (current->de_pathlen == 0)
+ pathlen = 0;
+ else
+ pathlen = current->de_pathlen + 1;
+ while (ce) {
+ if (ce->ce_flags & CE_REMOVE)
+ continue;
+ if (!ce_uptodate(ce) && is_racy_timestamp(istate, ce))
+ ce_smudge_racily_clean_entry(ce);
+
+ offset_write = htonl(offset);
+ if (ce_write(NULL, fd, &offset_write, 4) < 0)
+ return -1;
+ offset += ce_namelen(ce) - pathlen + 1 + ondisk_size + 4;
+ ce = ce->next;
+ }
+ current = current->next;
+ }
+ /*
+ * Write one more offset, which points to the end of the entries,
+ * because we use it for calculating the file length, instead of
+ * using strlen.
+ */
+ offset_write = htonl(offset);
+ if (ce_write(NULL, fd, &offset_write, 4) < 0)
+ return -1;
+
+ offset = offset_to_offset;
+ current = de;
+ while (current) {
+ int pathlen;
+ struct cache_entry *ce = current->ce;
+
+ if (current->de_pathlen == 0)
+ pathlen = 0;
+ else
+ pathlen = current->de_pathlen + 1;
+ while (ce) {
+ struct ondisk_cache_entry ondisk;
+ uint32_t crc, calc_crc;
+
+ if (ce->ce_flags & CE_REMOVE)
+ continue;
+ calc_crc = htonl(offset);
+ crc = crc32(0, (Bytef*)&calc_crc, 4);
+ if (ce_write(&crc, fd, ce->name + pathlen,
+ ce_namelen(ce) - pathlen + 1) < 0)
+ return -1;
+ ondisk_from_cache_entry(ce, &ondisk);
+ if (ce_write(&crc, fd, &ondisk, ondisk_size) < 0)
+ return -1;
+ crc = htonl(crc);
+ if (ce_write(NULL, fd, &crc, 4) < 0)
+ return -1;
+ offset += 4;
+ ce = ce->next;
+ }
+ current = current->next;
+ }
+ return 0;
+}
+
+static int write_conflict(struct conflict_entry *conflict, int fd)
+{
+ struct conflict_entry *current;
+ struct conflict_part *current_part;
+ uint32_t crc;
+
+ current = conflict;
+ while (current) {
+ unsigned int to_write;
+
+ crc = 0;
+ if (ce_write(&crc, fd,
+ (Bytef*)(current->name + current->pathlen),
+ current->namelen - current->pathlen) < 0)
+ return -1;
+ if (ce_write(&crc, fd, (Bytef*)"\0", 1) < 0)
+ return -1;
+ to_write = htonl(current->nfileconflicts);
+ if (ce_write(&crc, fd, (Bytef*)&to_write, 4) < 0)
+ return -1;
+ current_part = current->entries;
+ while (current_part) {
+ struct ondisk_conflict_part ondisk;
+
+ conflict_to_ondisk(current_part, &ondisk);
+ if (ce_write(&crc, fd, (Bytef*)&ondisk, sizeof(struct ondisk_conflict_part)) < 0)
+ return 0;
+ current_part = current_part->next;
+ }
+ to_write = htonl(crc);
+ if (ce_write(NULL, fd, (Bytef*)&to_write, 4) < 0)
+ return -1;
+ current = current->next;
+ }
+ return 0;
+}
+
+static int write_conflicts(struct index_state *istate,
+ struct directory_entry *de,
+ int fd)
+{
+ struct directory_entry *current;
+
+ current = de;
+ while (current) {
+ if (current->de_ncr != 0) {
+ if (write_conflict(current->conflict, fd) < 0)
+ return -1;
+ }
+ current = current->next;
+ }
+ return 0;
+}
+
+static int write_index_v5(struct index_state *istate, int newfd)
+{
+ struct cache_version_header hdr;
+ struct cache_header hdr_v5;
+ struct cache_entry **cache = istate->cache;
+ struct directory_entry *de;
+ struct ondisk_directory_entry *ondisk;
+ int entries = istate->cache_nr;
+ int i, removed, non_conflicted, total_dir_len, ondisk_directory_size;
+ int total_file_len, conflict_offset, offset_to_offset;
+ unsigned int ndir;
+ uint32_t crc;
+
+ for (i = removed = 0; i < entries; i++) {
+ if (cache[i]->ce_flags & CE_REMOVE)
+ removed++;
+ }
+ hdr.hdr_signature = htonl(CACHE_SIGNATURE);
+ hdr.hdr_version = htonl(istate->version);
+ hdr_v5.hdr_nfile = htonl(entries - removed);
+ hdr_v5.hdr_nextension = htonl(0); /* Currently no extensions are supported */
+
+ non_conflicted = 0;
+ total_dir_len = 0;
+ total_file_len = 0;
+ de = compile_directory_data(istate, entries, &ndir, &non_conflicted,
+ &total_dir_len, &total_file_len);
+ hdr_v5.hdr_ndir = htonl(ndir);
+
+ /*
+ * This is needed because the compiler aligns structs to sizes multipe
+ * of 4
+ */
+ ondisk_directory_size = sizeof(ondisk->flags)
+ + sizeof(ondisk->foffset)
+ + sizeof(ondisk->cr)
+ + sizeof(ondisk->ncr)
+ + sizeof(ondisk->nsubtrees)
+ + sizeof(ondisk->nfiles)
+ + sizeof(ondisk->nentries)
+ + sizeof(ondisk->sha1);
+ hdr_v5.hdr_fblockoffset = htonl(sizeof(hdr) + sizeof(hdr_v5) + 4
+ + (ndir + 1) * 4
+ + total_dir_len
+ + ndir * (ondisk_directory_size + 4)
+ + (non_conflicted + 1) * 4);
+
+ crc = 0;
+ if (ce_write(&crc, newfd, &hdr, sizeof(hdr)) < 0)
+ return -1;
+ if (ce_write(&crc, newfd, &hdr_v5, sizeof(hdr_v5)) < 0)
+ return -1;
+ crc = htonl(crc);
+ if (ce_write(NULL, newfd, &crc, 4) < 0)
+ return -1;
+
+ conflict_offset = sizeof(hdr) + sizeof(hdr_v5) + 4
+ + (ndir + 1) * 4
+ + total_dir_len
+ + ndir * (ondisk_directory_size + 4)
+ + (non_conflicted + 1) * 4
+ + total_file_len
+ + non_conflicted * (sizeof(struct ondisk_cache_entry) + 4);
+ if (write_directories(de, newfd, conflict_offset) < 0)
+ return -1;
+ offset_to_offset = sizeof(hdr) + sizeof(hdr_v5) + 4
+ + (ndir + 1) * 4
+ + total_dir_len
+ + ndir * (ondisk_directory_size + 4);
+ if (write_entries(istate, de, entries, newfd, offset_to_offset) < 0)
+ return -1;
+ if (write_conflicts(istate, de, newfd) < 0)
+ return -1;
+ return ce_flush(newfd);
+}
+
struct index_ops v5_ops = {
match_stat_basic,
verify_hdr,
read_index_v5,
- NULL
+ write_index_v5
};
diff --git a/read-cache.c b/read-cache.c
index 9d2bd62..7f360f9 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -51,7 +51,7 @@ void rename_index_entry_at(struct index_state *istate, int nr, const char *new_n
add_index_entry(istate, new, ADD_CACHE_OK_TO_ADD|ADD_CACHE_OK_TO_REPLACE);
}
-static uint32_t calculate_stat_crc(struct cache_entry *ce)
+uint32_t calculate_stat_crc(struct cache_entry *ce)
{
unsigned int ctimens = 0;
uint32_t stat, stat_crc;
@@ -172,13 +172,15 @@ int ce_modified_check_fs(struct cache_entry *ce, struct stat *st)
return 0;
}
-static void check_set_istate_ops(struct index_state *istate)
+void set_istate_ops(struct index_state *istate)
{
if (!istate->version)
istate->version = INDEX_FORMAT_DEFAULT;
- if (!istate->ops)
- if (istate->version >= 2 && istate->version <=4)
- istate->ops = &v2_ops;
+
+ if (istate->version >= 2 && istate->version <=4)
+ istate->ops = &v2_ops;
+ if (istate->version == 5)
+ istate->ops = &v5_ops;
}
int ce_match_stat_basic(struct index_state *istate,
@@ -215,7 +217,7 @@ int ce_match_stat_basic(struct index_state *istate,
die("internal error: ce_mode is %o", ce->ce_mode);
}
- check_set_istate_ops(istate);
+ set_istate_ops(istate);
changed = istate->ops->match_stat_basic(ce, st, changed);
return changed;
}
@@ -1185,6 +1187,8 @@ static int verify_hdr_version(struct index_state *istate,
hdr_version = ntohl(hdr->hdr_version);
if (hdr_version >= 2 && hdr_version <= 4)
istate->ops = &v2_ops;
+ else if (hdr_version == 5)
+ istate->ops = &v5_ops;
else
return error("bad index version %d", hdr_version);
return 0;
@@ -1339,10 +1343,9 @@ void update_index_if_able(struct index_state *istate, struct lock_file *lockfile
else
rollback_lock_file(lockfile);
}
-
int write_index(struct index_state *istate, int newfd)
{
- check_set_istate_ops(istate);
+ set_istate_ops(istate);
return istate->ops->write_index(istate, newfd);
}
diff --git a/read-cache.h b/read-cache.h
index f41ba75..09ed55b 100644
--- a/read-cache.h
+++ b/read-cache.h
@@ -29,6 +29,7 @@ struct index_ops {
};
extern struct index_ops v2_ops;
+extern struct index_ops v5_ops;
#ifndef NEEDS_ALIGNED_ACCESS
#define ntoh_s(var) ntohs(var)
@@ -55,3 +56,5 @@ extern int ce_match_stat_basic(struct index_state *istate,
struct cache_entry *ce, struct stat *st);
extern int is_racy_timestamp(const struct index_state *istate, struct cache_entry *ce);
extern void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce);
+extern uint32_t calculate_stat_crc(struct cache_entry *ce);
+extern void set_istate_ops(struct index_state *istate);
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 10/13] Write index-v5 cache-tree data
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
` (8 preceding siblings ...)
2012-08-16 9:58 ` [PATCH/RFC v4 09/13] Write index-v5 Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 11/13] Write resolve-undo data for index-v5 Thomas Gummerer
` (2 subsequent siblings)
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Write the cache-tree data for the index version 5 file format. The
in-memory cache-tree data is converted to the ondisk format, by adding
it to the directory entries, that were compiled from the cache-entries
in the step before.
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
read-cache-v5.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
diff --git a/read-cache-v5.c b/read-cache-v5.c
index a2afc94..d740d0b 100644
--- a/read-cache-v5.c
+++ b/read-cache-v5.c
@@ -856,6 +856,57 @@ static struct conflict_entry *create_conflict_entry_from_ce(struct cache_entry *
return create_new_conflict(ce->name, ce_namelen(ce), pathlen);
}
+static void convert_one_to_ondisk_v5(struct hash_table *table, struct cache_tree *it,
+ const char *path, int pathlen, uint32_t crc)
+{
+ int i;
+ struct directory_entry *found, *search;
+
+ crc = crc32(crc, (Bytef*)path, pathlen);
+ found = lookup_hash(crc, table);
+ search = found;
+ while (search && strcmp(path, search->pathname + search->de_pathlen - strlen(path)) != 0)
+ search = search->next_hash;
+ if (!search)
+ return;
+ /*
+ * The number of subtrees is already calculated by
+ * compile_directory_data, therefore we only need to
+ * add the entry_count
+ */
+ search->de_nentries = it->entry_count;
+ if (0 <= it->entry_count)
+ hashcpy(search->sha1, it->sha1);
+ if (strcmp(path, "") != 0)
+ crc = crc32(crc, (Bytef*)"/", 1);
+
+#if DEBUG
+ if (0 <= it->entry_count)
+ fprintf(stderr, "cache-tree <%.*s> (%d ent, %d subtree) %s\n",
+ pathlen, path, it->entry_count, it->subtree_nr,
+ sha1_to_hex(it->sha1));
+ else
+ fprintf(stderr, "cache-tree <%.*s> (%d subtree) invalid\n",
+ pathlen, path, it->subtree_nr);
+#endif
+
+ for (i = 0; i < it->subtree_nr; i++) {
+ struct cache_tree_sub *down = it->down[i];
+ if (i) {
+ struct cache_tree_sub *prev = it->down[i-1];
+ if (subtree_name_cmp(down->name, down->namelen,
+ prev->name, prev->namelen) <= 0)
+ die("fatal - unsorted cache subtree");
+ }
+ convert_one_to_ondisk_v5(table, down->cache_tree, down->name, down->namelen, crc);
+ }
+}
+
+static void cache_tree_to_ondisk_v5(struct hash_table *table, struct cache_tree *root)
+{
+ convert_one_to_ondisk_v5(table, root, "", 0, 0);
+}
+
static struct directory_entry *compile_directory_data(struct index_state *istate,
int nfile,
unsigned int *ndir,
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 11/13] Write resolve-undo data for index-v5
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
` (9 preceding siblings ...)
2012-08-16 9:58 ` [PATCH/RFC v4 10/13] Write index-v5 cache-tree data Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 12/13] update-index.c: rewrite index when index-version is given Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 13/13] p0002-index.sh: add perf test for the index formats Thomas Gummerer
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Make git read the resolve-undo data from the index.
Since the resolve-undo data is joined with the conflicts in
the ondisk format of the index file version 5, conflicts and
resolved data is read at the same time, and the resolve-undo
data is then converted to the in-memory format.
Helped-by: Thomas Rast <trast@student.ethz.ch>
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
read-cache-v5.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 96 insertions(+)
diff --git a/read-cache-v5.c b/read-cache-v5.c
index d740d0b..ce2375a 100644
--- a/read-cache-v5.c
+++ b/read-cache-v5.c
@@ -907,6 +907,99 @@ static void cache_tree_to_ondisk_v5(struct hash_table *table, struct cache_tree
convert_one_to_ondisk_v5(table, root, "", 0, 0);
}
+static void resolve_undo_to_ondisk_v5(struct hash_table *table,
+ struct string_list *resolve_undo,
+ unsigned int *ndir, int *total_dir_len,
+ struct directory_entry *de)
+{
+ struct string_list_item *item;
+ struct directory_entry *search;
+
+ if (!resolve_undo)
+ return;
+ for_each_string_list_item(item, resolve_undo) {
+ struct conflict_entry *conflict_entry;
+ struct resolve_undo_info *ui = item->util;
+ char *super;
+ int i, dir_len, len;
+ uint32_t crc;
+ struct directory_entry *found, *current, *new_tree;
+
+ if (!ui)
+ continue;
+
+ super = super_directory(item->string);
+ if (!super)
+ dir_len = 0;
+ else
+ dir_len = strlen(super);
+ crc = crc32(0, (Bytef*)super, dir_len);
+ found = lookup_hash(crc, table);
+ current = NULL;
+ new_tree = NULL;
+
+ while (!found) {
+ struct directory_entry *new;
+
+ new = init_directory_entry(super, dir_len);
+ if (!current)
+ current = new;
+ insert_directory_entry(new, table, total_dir_len, ndir, crc);
+ if (new_tree != NULL)
+ new->de_nsubtrees = 1;
+ new->next = new_tree;
+ new_tree = new;
+ super = super_directory(super);
+ if (!super)
+ dir_len = 0;
+ else
+ dir_len = strlen(super);
+ crc = crc32(0, (Bytef*)super, dir_len);
+ found = lookup_hash(crc, table);
+ }
+ search = found;
+ while (search->next_hash && strcmp(super, search->pathname) != 0)
+ search = search->next_hash;
+ if (search && !current)
+ current = search;
+ if (!search && !current)
+ current = new_tree;
+ if (!super && new_tree) {
+ new_tree->next = de->next;
+ de->next = new_tree;
+ de->de_nsubtrees++;
+ } else if (new_tree) {
+ struct directory_entry *temp;
+
+ search = de->next;
+ while (strcmp(super, search->pathname))
+ search = search->next;
+ temp = new_tree;
+ while (temp->next)
+ temp = temp->next;
+ search->de_nsubtrees++;
+ temp->next = search->next;
+ search->next = new_tree;
+ }
+
+ len = strlen(item->string);
+ conflict_entry = create_new_conflict(item->string, len, current->de_pathlen);
+ add_conflict_to_directory_entry(current, conflict_entry);
+ for (i = 0; i < 3; i++) {
+ if (ui->mode[i]) {
+ struct conflict_part *cp;
+
+ cp = xmalloc(sizeof(struct conflict_part));
+ cp->flags = (i + 1) << CONFLICT_STAGESHIFT;
+ cp->entry_mode = ui->mode[i];
+ cp->next = NULL;
+ hashcpy(cp->sha1, ui->sha1[i]);
+ add_part_to_conflict_entry(current, conflict_entry, cp);
+ }
+ }
+ }
+}
+
static struct directory_entry *compile_directory_data(struct index_state *istate,
int nfile,
unsigned int *ndir,
@@ -1012,6 +1105,9 @@ static struct directory_entry *compile_directory_data(struct index_state *istate
previous_entry->next = no_subtrees;
}
}
+ if (istate->cache_tree)
+ cache_tree_to_ondisk_v5(&table, istate->cache_tree);
+ resolve_undo_to_ondisk_v5(&table, istate->resolve_undo, ndir, total_dir_len, de);
return de;
}
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 12/13] update-index.c: rewrite index when index-version is given
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
` (10 preceding siblings ...)
2012-08-16 9:58 ` [PATCH/RFC v4 11/13] Write resolve-undo data for index-v5 Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
2012-08-16 9:58 ` [PATCH/RFC v4 13/13] p0002-index.sh: add perf test for the index formats Thomas Gummerer
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
Make update-index always rewrite the index when a index-version
is given, even if the index already has the right version.
This option is used for performance testing the writer and
reader.
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
builtin/update-index.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/builtin/update-index.c b/builtin/update-index.c
index 4ce341c..c31d176 100644
--- a/builtin/update-index.c
+++ b/builtin/update-index.c
@@ -6,6 +6,7 @@
#include "cache.h"
#include "quote.h"
#include "cache-tree.h"
+#include "read-cache.h"
#include "tree-walk.h"
#include "builtin.h"
#include "refs.h"
@@ -861,6 +862,7 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
if (the_index.version != preferred_index_format)
active_cache_changed = 1;
the_index.version = preferred_index_format;
+ set_istate_ops(&the_index);
}
if (read_from_stdin) {
@@ -886,7 +888,7 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
strbuf_release(&buf);
}
- if (active_cache_changed) {
+ if (active_cache_changed || preferred_index_format) {
if (newfd < 0) {
if (refresh_args.flags & REFRESH_QUIET)
exit(128);
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH/RFC v4 13/13] p0002-index.sh: add perf test for the index formats
2012-08-16 9:58 [PATCH/RFC v4 0/13] Introduce index file format version 5 Thomas Gummerer
` (11 preceding siblings ...)
2012-08-16 9:58 ` [PATCH/RFC v4 12/13] update-index.c: rewrite index when index-version is given Thomas Gummerer
@ 2012-08-16 9:58 ` Thomas Gummerer
12 siblings, 0 replies; 14+ messages in thread
From: Thomas Gummerer @ 2012-08-16 9:58 UTC (permalink / raw)
To: git; +Cc: trast, pclouds, mhagger, gitster, robin.rosenberg, t.gummerer
From: Thomas Rast <trast@student.ethz.ch>
Add a performance test for index version [23]/4/5 by using
git update-index --index-version=x, thus testing both the reader
and the writer speed of all index formats.
Signed-off-by: Thomas Rast <trast@student.ethz.ch>
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
t/perf/p0002-index.sh | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
create mode 100755 t/perf/p0002-index.sh
diff --git a/t/perf/p0002-index.sh b/t/perf/p0002-index.sh
new file mode 100755
index 0000000..140c7a0
--- /dev/null
+++ b/t/perf/p0002-index.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+test_description="Tests index versions [23]/4/5"
+
+. ./perf-lib.sh
+
+test_perf_large_repo
+
+test_expect_success 'convert to v3' '
+ git update-index --index-version=3
+'
+
+test_perf 'v[23]: update-index' '
+ git update-index --index-version=3 >/dev/null
+'
+
+test_expect_success 'convert to v4' '
+ git update-index --index-version=4
+'
+
+test_perf 'v4: update-index' '
+ git update-index --index-version=4 >/dev/null
+'
+
+test_expect_success 'convert to v5' '
+ git update-index --index-version=5
+'
+
+test_perf 'v5: update-index' '
+ git update-index --index-version=5 >/dev/null
+'
+
+test_done
--
1.7.11.2
^ permalink raw reply related [flat|nested] 14+ messages in thread