* [PATCH 2/6 (v4)] basic revision cache system, no integration or features
From: Nick Edelen @ 2009-08-17 12:31 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
Second in the revision cache series, this particular patch provides:
- minimal API: caching only commit topo data
- minimal porcelain: add and walk cache slices
- appropriate tests
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
Makefile | 2 +
builtin-rev-cache.c | 206 ++++++++
builtin.h | 1 +
commit.c | 2 +
git.c | 1 +
rev-cache.c | 1169 +++++++++++++++++++++++++++++++++++++++++++++
rev-cache.h | 107 ++++
revision.c | 2 +-
revision.h | 26 +-
t/t6015-rev-cache-list.sh | 104 ++++
10 files changed, 1618 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
index e6df8ec..1ceb514 100644
--- a/Makefile
+++ b/Makefile
@@ -537,6 +537,7 @@ LIB_OBJS += reflog-walk.o
LIB_OBJS += refs.o
LIB_OBJS += remote.o
LIB_OBJS += rerere.o
+LIB_OBJS += rev-cache.o
LIB_OBJS += revision.o
LIB_OBJS += run-command.o
LIB_OBJS += server-info.o
@@ -627,6 +628,7 @@ BUILTIN_OBJS += builtin-reflog.o
BUILTIN_OBJS += builtin-remote.o
BUILTIN_OBJS += builtin-rerere.o
BUILTIN_OBJS += builtin-reset.o
+BUILTIN_OBJS += builtin-rev-cache.o
BUILTIN_OBJS += builtin-rev-list.o
BUILTIN_OBJS += builtin-rev-parse.o
BUILTIN_OBJS += builtin-revert.o
diff --git a/builtin-rev-cache.c b/builtin-rev-cache.c
new file mode 100644
index 0000000..c27e0f3
--- /dev/null
+++ b/builtin-rev-cache.c
@@ -0,0 +1,206 @@
+#include "cache.h"
+#include "object.h"
+#include "commit.h"
+#include "diff.h"
+#include "revision.h"
+#include "rev-cache.h"
+
+/* porcelain for rev-cache.c */
+static int handle_add(int argc, const char *argv[]) /* args beyond this command */
+{
+ struct rev_info revs;
+ struct rev_cache_info rci;
+ char dostdin = 0;
+ unsigned int flags = 0;
+ int i, retval;
+ unsigned char cache_sha1[20];
+ struct commit_list *starts = 0, *ends = 0;
+ struct commit *commit;
+
+ init_revisions(&revs, 0);
+ init_rev_cache_info(&rci);
+
+ for (i = 0; i < argc; i++) {
+ if (!strcmp(argv[i], "--stdin"))
+ dostdin = 1;
+ else if (!strcmp(argv[i], "--fresh"))
+ starts_from_slices(&revs, UNINTERESTING);
+ else if (!strcmp(argv[i], "--not"))
+ flags ^= UNINTERESTING;
+ else if (!strcmp(argv[i], "--legs"))
+ rci.legs = 1;
+ else if (!strcmp(argv[i], "--no-objects"))
+ rci.objects = 0;
+ else if (!strcmp(argv[i], "--all")) {
+ const char *args[2];
+ int argn = 0;
+
+ args[argn++] = "rev-list";
+ args[argn++] = "--all";
+ setup_revisions(argn, args, &revs, 0);
+ } else
+ handle_revision_arg(argv[i], &revs, flags, 1);
+ }
+
+ if (dostdin) {
+ char line[1000];
+
+ flags = 0;
+ while (fgets(line, sizeof(line), stdin)) {
+ int len = strlen(line);
+ while (len && (line[len - 1] == '\n' || line[len - 1] == '\r'))
+ line[--len] = 0;
+
+ if (!len)
+ break;
+
+ if (!strcmp(line, "--not"))
+ flags ^= UNINTERESTING;
+ else
+ handle_revision_arg(line, &revs, flags, 1);
+ }
+ }
+
+ retval = make_cache_slice(&rci, &revs, &starts, &ends, cache_sha1);
+ if (retval < 0)
+ return retval;
+
+ printf("%s\n", sha1_to_hex(cache_sha1));
+
+ fprintf(stderr, "endpoints:\n");
+ while ((commit = pop_commit(&starts)))
+ fprintf(stderr, "S %s\n", sha1_to_hex(commit->object.sha1));
+ while ((commit = pop_commit(&ends)))
+ fprintf(stderr, "E %s\n", sha1_to_hex(commit->object.sha1));
+
+ return 0;
+}
+
+static int handle_walk(int argc, const char *argv[])
+{
+ struct commit *commit;
+ struct rev_info revs;
+ struct commit_list *queue, *work, **qp;
+ unsigned char *sha1p, *sha1pt;
+ unsigned long date = 0;
+ unsigned int flags = 0;
+ int retval, slop = 5, i;
+
+ init_revisions(&revs, 0);
+
+ for (i = 0; i < argc; i++) {
+ if (!strcmp(argv[i], "--not"))
+ flags ^= UNINTERESTING;
+ else if (!strcmp(argv[i], "--objects"))
+ revs.tree_objects = revs.blob_objects = 1;
+ else
+ handle_revision_arg(argv[i], &revs, flags, 1);
+ }
+
+ work = 0;
+ sha1p = 0;
+ for (i = 0; i < revs.pending.nr; i++) {
+ commit = lookup_commit(revs.pending.objects[i].item->sha1);
+
+ sha1pt = get_cache_slice(commit);
+ if (!sha1pt)
+ die("%s: not in a cache slice", sha1_to_hex(commit->object.sha1));
+
+ if (!i)
+ sha1p = sha1pt;
+ else if (sha1p != sha1pt)
+ die("walking porcelain is /per/ cache slice; commits cannot be spread out amoung several");
+
+ insert_by_date(commit, &work);
+ }
+
+ if (!sha1p)
+ die("nothing to traverse!");
+
+ queue = 0;
+ qp = &queue;
+ commit = pop_commit(&work);
+ retval = traverse_cache_slice(&revs, sha1p, commit, &date, &slop, &qp, &work);
+ if (retval < 0)
+ return retval;
+
+ fprintf(stderr, "queue:\n");
+ while ((commit = pop_commit(&queue)) != 0) {
+ printf("%s\n", sha1_to_hex(commit->object.sha1));
+ }
+
+ fprintf(stderr, "work:\n");
+ while ((commit = pop_commit(&work)) != 0) {
+ printf("%s\n", sha1_to_hex(commit->object.sha1));
+ }
+
+ fprintf(stderr, "pending:\n");
+ for (i = 0; i < revs.pending.nr; i++) {
+ struct object *obj = revs.pending.objects[i].item;
+
+ /* unfortunately, despite our careful generation, object duplication *is* a possibility...
+ * (eg. same object introduced into two different branches) */
+ if (obj->flags & SEEN)
+ continue;
+
+ printf("%s\n", sha1_to_hex(revs.pending.objects[i].item->sha1));
+ obj->flags |= SEEN;
+ }
+
+ return 0;
+}
+
+static int handle_help(void)
+{
+ char *usage = "\
+usage:\n\
+git-rev-cache COMMAND [options] [<commit-id>...]\n\
+commands:\n\
+ add - add revisions to the cache. reads commit ids from stdin, \n\
+ formatted as: START START ... --not END END ...\n\
+ options:\n\
+ --all use all branch heads as starts\n\
+ --fresh exclude everything already in a cache slice\n\
+ --stdin also read commit ids from stdin (same form as cmd)\n\
+ --legs ensure branch is entirely self-contained\n\
+ --no-objects don't add non-commit objects to slice\n\
+ walk - walk a cache slice based on set of commits; formatted as add\n\
+ options:\n\
+ --objects include non-commit objects in traversals\n\
+ fuse - coalesce cache slices into a single cache.\n\
+ options:\n\
+ --all include all objects in repository\n\
+ --no-objects don't add non-commit objects to slice\n\
+ --ignore-size[=N] ignore slices of size >= N; defaults to ~5MB\n\
+ index - regnerate the cache index.";
+
+ puts(usage);
+
+ return 0;
+}
+
+int cmd_rev_cache(int argc, const char *argv[], const char *prefix)
+{
+ const char *arg;
+ int r;
+
+ git_config(git_default_config, NULL);
+
+ if (argc > 1)
+ arg = argv[1];
+ else
+ arg = "";
+
+ argc -= 2;
+ argv += 2;
+ if (!strcmp(arg, "add"))
+ r = handle_add(argc, argv);
+ else if (!strcmp(arg, "walk"))
+ r = handle_walk(argc, argv);
+ else
+ return handle_help();
+
+ fprintf(stderr, "final return value: %d\n", r);
+
+ return 0;
+}
diff --git a/builtin.h b/builtin.h
index 20427d2..00ecc9c 100644
--- a/builtin.h
+++ b/builtin.h
@@ -87,6 +87,7 @@ extern int cmd_remote(int argc, const char **argv, const char *prefix);
extern int cmd_config(int argc, const char **argv, const char *prefix);
extern int cmd_rerere(int argc, const char **argv, const char *prefix);
extern int cmd_reset(int argc, const char **argv, const char *prefix);
+extern int cmd_rev_cache(int argc, const char **argv, const char *prefix);
extern int cmd_rev_list(int argc, const char **argv, const char *prefix);
extern int cmd_rev_parse(int argc, const char **argv, const char *prefix);
extern int cmd_revert(int argc, const char **argv, const char *prefix);
diff --git a/commit.c b/commit.c
index e2bcbe8..682e7a7 100644
--- a/commit.c
+++ b/commit.c
@@ -252,6 +252,8 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
item->tree = lookup_tree(parent);
bufptr += 46; /* "tree " + "hex sha1" + "\n" */
pptr = &item->parents;
+ while (pop_commit(pptr))
+ ; /* clear anything from cache */
graft = lookup_commit_graft(item->object.sha1);
while (bufptr + 48 < tail && !memcmp(bufptr, "parent ", 7)) {
diff --git a/git.c b/git.c
index 4588a8b..7105c74 100644
--- a/git.c
+++ b/git.c
@@ -342,6 +342,7 @@ static void handle_internal_command(int argc, const char **argv)
{ "repo-config", cmd_config },
{ "rerere", cmd_rerere, RUN_SETUP },
{ "reset", cmd_reset, RUN_SETUP },
+ { "rev-cache", cmd_rev_cache, RUN_SETUP },
{ "rev-list", cmd_rev_list, RUN_SETUP },
{ "rev-parse", cmd_rev_parse },
{ "revert", cmd_revert, RUN_SETUP | NEED_WORK_TREE },
diff --git a/rev-cache.c b/rev-cache.c
new file mode 100644
index 0000000..a01db87
--- /dev/null
+++ b/rev-cache.c
@@ -0,0 +1,1169 @@
+#include "cache.h"
+#include "object.h"
+#include "commit.h"
+#include "tree.h"
+#include "tree-walk.h"
+#include "blob.h"
+#include "tag.h"
+#include "diff.h"
+#include "revision.h"
+#include "rev-cache.h"
+#include "run-command.h"
+
+/* list resembles pack index format */
+static uint32_t fanout[0xff + 2];
+
+static unsigned char *idx_map;
+static int idx_size;
+static struct rc_index_header idx_head;
+static unsigned char *idx_caches;
+static char no_idx;
+
+static struct strbuf *acc_buffer;
+
+#define SLOP 5
+
+/* initialization */
+
+struct rc_index_entry *from_disked_rc_index_entry(struct rc_index_entry_ondisk *src, struct rc_index_entry *dst)
+{
+ static struct rc_index_entry entry[4];
+ static int cur;
+
+ if (!dst)
+ dst = &entry[cur++ & 0x3];
+
+ dst->sha1 = src->sha1;
+ dst->is_start = !!(src->flags & 0x80);
+ dst->cache_index = src->flags & 0x7f;
+ dst->pos = ntohl(src->pos);
+
+ return dst;
+}
+
+struct rc_index_entry_ondisk *to_disked_rc_index_entry(struct rc_index_entry *src, struct rc_index_entry_ondisk *dst)
+{
+ static struct rc_index_entry_ondisk entry[4];
+ static int cur;
+
+ if (!dst)
+ dst = &entry[cur++ & 0x3];
+
+ hashcpy(dst->sha1, src->sha1);
+ dst->flags = (unsigned char)src->is_start << 7 | (unsigned char)src->cache_index;
+ dst->pos = htonl(src->pos);
+
+ return dst;
+}
+
+struct rc_object_entry *from_disked_rc_object_entry(struct rc_object_entry_ondisk *src, struct rc_object_entry *dst)
+{
+ static struct rc_object_entry entry[4];
+ static int cur;
+
+ if (!dst)
+ dst = &entry[cur++ & 0x3];
+
+ dst->type = src->flags >> 5;
+ dst->is_end = !!(src->flags & 0x10);
+ dst->is_start = !!(src->flags & 0x08);
+ dst->uninteresting = !!(src->flags & 0x04);
+ dst->include = !!(src->flags & 0x02);
+ dst->flag = !!(src->flags & 0x01);
+
+ dst->sha1 = src->sha1;
+ dst->merge_nr = src->merge_nr;
+ dst->split_nr = src->split_nr;
+
+ dst->size_size = src->sizes >> 5;
+ dst->padding = src->sizes & 0x1f;
+
+ dst->date = ntohl(src->date);
+ dst->path = ntohs(src->path);
+
+ return dst;
+}
+
+struct rc_object_entry_ondisk *to_disked_rc_object_entry(struct rc_object_entry *src, struct rc_object_entry_ondisk *dst)
+{
+ static struct rc_object_entry_ondisk entry[4];
+ static int cur;
+
+ if (!dst)
+ dst = &entry[cur++ & 0x3];
+
+ dst->flags = (unsigned char)src->type << 5;
+ dst->flags |= (unsigned char)src->is_end << 4;
+ dst->flags |= (unsigned char)src->is_start << 3;
+ dst->flags |= (unsigned char)src->uninteresting << 2;
+ dst->flags |= (unsigned char)src->include << 1;
+ dst->flags |= (unsigned char)src->flag;
+
+ hashcpy(dst->sha1, src->sha1);
+ dst->merge_nr = src->merge_nr;
+ dst->split_nr = src->split_nr;
+
+ dst->sizes = (unsigned char)src->size_size << 5;
+ dst->sizes |= (unsigned char)src->padding;
+
+ dst->date = htonl(src->date);
+ dst->path = htons(src->path);
+
+ return dst;
+}
+
+static int get_index_head(unsigned char *map, int len, struct rc_index_header *head, uint32_t *fanout, unsigned char **caches)
+{
+ struct rc_index_header whead;
+ int i, index = sizeof(struct rc_index_header);
+
+ memcpy(&whead, map, sizeof(struct rc_index_header));
+ if (memcmp(whead.signature, "REVINDEX", 8) || whead.version != SUPPORTED_REVINDEX_VERSION)
+ return -1;
+
+ memcpy(head->signature, "REVINDEX", 8);
+ head->version = whead.version;
+ head->ofs_objects = ntohl(whead.ofs_objects);
+ head->object_nr = ntohl(whead.object_nr);
+ head->cache_nr = whead.cache_nr;
+ head->max_date = ntohl(whead.max_date);
+
+ if (len < index + head->cache_nr * 20 + 0x100 * sizeof(uint32_t))
+ return -2;
+
+ *caches = xmalloc(head->cache_nr * 20);
+ memcpy(*caches, map + index, head->cache_nr * 20);
+ index += head->cache_nr * 20;
+
+ memcpy(fanout, map + index, 0x100 * sizeof(uint32_t));
+ for (i = 0; i <= 0xff; i++)
+ fanout[i] = ntohl(fanout[i]);
+ fanout[0x100] = len;
+
+ return 0;
+}
+
+/* added in init_index */
+static void cleanup_cache_slices(void)
+{
+ if (idx_map) {
+ free(idx_caches);
+ munmap(idx_map, idx_size);
+ idx_map = 0;
+ }
+
+}
+
+static int init_index(void)
+{
+ int fd;
+ struct stat fi;
+
+ fd = open(git_path("rev-cache/index"), O_RDONLY);
+ if (fd == -1 || fstat(fd, &fi))
+ goto end;
+ if (fi.st_size < sizeof(struct rc_index_header))
+ goto end;
+
+ idx_size = fi.st_size;
+ idx_map = xmmap(0, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ close(fd);
+ if (idx_map == MAP_FAILED)
+ goto end;
+ if (get_index_head(idx_map, fi.st_size, &idx_head, fanout, &idx_caches))
+ goto end;
+
+ atexit(cleanup_cache_slices);
+
+ return 0;
+
+end:
+ idx_map = 0;
+ no_idx = 1;
+ return -1;
+}
+
+/* this assumes index is already loaded */
+static struct rc_index_entry_ondisk *search_index_1(unsigned char *sha1)
+{
+ int start, end, starti, endi, i, len, r;
+ struct rc_index_entry_ondisk *ie;
+
+ if (!idx_map)
+ return 0;
+
+ /* binary search */
+ start = fanout[(int)sha1[0]];
+ end = fanout[(int)sha1[0] + 1];
+ len = (end - start) / sizeof(struct rc_index_entry_ondisk);
+ if (!len || len * sizeof(struct rc_index_entry_ondisk) != end - start)
+ return 0;
+
+ starti = 0;
+ endi = len - 1;
+ for (;;) {
+ i = (endi + starti) / 2;
+ ie = (struct rc_index_entry_ondisk *)(idx_map + start + i * sizeof(struct rc_index_entry_ondisk));
+ r = hashcmp(sha1, ie->sha1);
+
+ if (r) {
+ if (starti + 1 == endi) {
+ starti++;
+ continue;
+ } else if (starti == endi)
+ break;
+
+ if (r > 0)
+ starti = i;
+ else /* r < 0 */
+ endi = i;
+ } else
+ return ie;
+ }
+
+ return 0;
+}
+
+static struct rc_index_entry *search_index(unsigned char *sha1)
+{
+ struct rc_index_entry_ondisk *ied = search_index_1(sha1);
+
+ if (ied)
+ return from_disked_rc_index_entry(ied, 0);
+
+ return 0;
+}
+
+unsigned char *get_cache_slice(struct commit *commit)
+{
+ struct rc_index_entry *ie;
+
+ if (!idx_map) {
+ if (no_idx)
+ return 0;
+ init_index();
+ }
+
+ if (commit->date > idx_head.max_date)
+ return 0;
+
+ ie = search_index(commit->object.sha1);
+ if (ie && ie->cache_index < idx_head.cache_nr)
+ return idx_caches + ie->cache_index * 20;
+
+ return 0;
+}
+
+
+/* traversal */
+
+static int setup_traversal(struct rc_slice_header *head, unsigned char *map, struct commit *commit, struct commit_list **work)
+{
+ struct rc_index_entry *iep;
+ struct rc_object_entry *oep;
+ struct commit_list *prev, *wp, **wpp;
+ int retval;
+
+ iep = search_index(commit->object.sha1), 0;
+ oep = RC_OBTAIN_OBJECT_ENTRY(map + iep->pos);
+
+ /* the .uniniteresting bit isn't strictly necessary, as we check the object during traversal as well,
+ * but we might as well initialize it while we're at it */
+ oep->include = 1;
+ oep->uninteresting = !!(commit->object.flags & UNINTERESTING);
+ to_disked_rc_object_entry(oep, (struct rc_object_entry_ondisk *)(map + iep->pos));
+ retval = iep->pos;
+
+ /* include any others in the work array */
+ prev = 0;
+ wpp = work;
+ wp = *work;
+ while (wp) {
+ struct object *obj = &wp->item->object;
+ struct commit *co;
+
+ /* is this in our cache slice? */
+ iep = search_index(obj->sha1);
+ if (!iep || hashcmp(idx_caches + iep->cache_index * 20, head->sha1)) {
+ prev = wp;
+ wp = wp->next;
+ wpp = ℘
+ continue;
+ }
+
+ if (iep->pos < retval)
+ retval = iep->pos;
+
+ oep = RC_OBTAIN_OBJECT_ENTRY(map + iep->pos);
+
+ /* mark this for later */
+ oep->include = 1;
+ oep->uninteresting = !!(obj->flags & UNINTERESTING);
+ to_disked_rc_object_entry(oep, (struct rc_object_entry_ondisk *)(map + iep->pos));
+
+ /* remove from work list */
+ co = pop_commit(wpp);
+ wp = *wpp;
+ if (prev)
+ prev->next = wp;
+ }
+
+ return retval;
+}
+
+#define IPATH 0x40
+#define UPATH 0x80
+
+#define GET_COUNT(x) ((x) & 0x3f)
+#define SET_COUNT(x, s) ((x) = ((x) & ~0x3f) | ((s) & 0x3f))
+
+static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *map,
+ struct rev_info *revs, struct commit *commit,
+ unsigned long *date_so_far, int *slop_so_far,
+ struct commit_list ***queue, struct commit_list **work)
+{
+ struct commit_list *insert_cache = 0;
+ struct commit **last_objects, *co;
+ int i, total_path_nr = head->path_nr, retval = -1;
+ char consume_children = 0;
+ unsigned char *paths;
+
+ i = setup_traversal(head, map, commit, work);
+ if (i < 0)
+ return -1;
+
+ paths = xcalloc(total_path_nr, sizeof(uint16_t));
+ last_objects = xcalloc(total_path_nr, sizeof(struct commit *));
+
+ /* i already set */
+ while (i < head->size) {
+ struct rc_object_entry *entry = RC_OBTAIN_OBJECT_ENTRY(map + i);
+ int path = entry->path;
+ struct object *obj;
+ int index = i;
+
+ i += RC_ACTUAL_OBJECT_ENTRY_SIZE(entry);
+
+ /* add extra objects if necessary */
+ if (entry->type != OBJ_COMMIT)
+ continue;
+ else
+ consume_children = 0;
+
+ if (path >= total_path_nr)
+ goto end;
+
+ /* in one of our branches?
+ * uninteresting trumps interesting */
+ if (entry->include)
+ paths[path] |= entry->uninteresting ? UPATH : IPATH;
+ else if (!paths[path])
+ continue;
+
+ /* date stuff */
+ if (revs->max_age != -1 && entry->date < revs->max_age)
+ paths[path] |= UPATH;
+
+ /* lookup object */
+ co = lookup_commit(entry->sha1);
+ obj = &co->object;
+
+ if (obj->flags & UNINTERESTING)
+ paths[path] |= UPATH;
+
+ if ((paths[path] & IPATH) && (paths[path] & UPATH)) {
+ paths[path] = UPATH;
+
+ /* mark edge */
+ if (last_objects[path]) {
+ parse_commit(last_objects[path]);
+
+ last_objects[path]->object.flags &= ~FACE_VALUE;
+ last_objects[path] = 0;
+ }
+ }
+
+ /* now we gotta re-assess the whole interesting thing... */
+ entry->uninteresting = !!(paths[path] & UPATH);
+
+ /* first close paths */
+ if (entry->split_nr) {
+ int j, off = index + sizeof(struct rc_object_entry_ondisk) + RC_PATH_SIZE(entry->merge_nr);
+
+ for (j = 0; j < entry->split_nr; j++) {
+ unsigned short p = ntohs(*(uint16_t *)(map + off + RC_PATH_SIZE(j)));
+
+ if (p >= total_path_nr)
+ goto end;
+
+ /* boundary commit? */
+ if ((paths[p] & IPATH) && entry->uninteresting) {
+ if (last_objects[p]) {
+ parse_commit(last_objects[p]);
+
+ last_objects[p]->object.flags &= ~FACE_VALUE;
+ last_objects[p] = 0;
+ }
+ } else if (last_objects[p] && !last_objects[p]->object.parsed)
+ commit_list_insert(co, &last_objects[p]->parents);
+
+ /* can't close a merge path until all are parents have been encountered */
+ if (GET_COUNT(paths[p])) {
+ SET_COUNT(paths[p], GET_COUNT(paths[p]) - 1);
+
+ if (GET_COUNT(paths[p]))
+ continue;
+ }
+
+ paths[p] = 0;
+ last_objects[p] = 0;
+ }
+ }
+
+ /* make topo relations */
+ if (last_objects[path] && !last_objects[path]->object.parsed)
+ commit_list_insert(co, &last_objects[path]->parents);
+
+ /* initialize commit */
+ if (!entry->is_end) {
+ co->date = entry->date;
+ obj->flags |= ADDED | FACE_VALUE;
+ } else
+ parse_commit(co);
+
+ obj->flags |= SEEN;
+
+ if (entry->uninteresting)
+ obj->flags |= UNINTERESTING;
+
+ /* we need to know what the edges are */
+ last_objects[path] = co;
+
+ /* add to list */
+ if (!(obj->flags & UNINTERESTING) || revs->show_all) {
+ if (entry->is_end)
+ insert_by_date_cached(co, work, insert_cache, &insert_cache);
+ else
+ *queue = &commit_list_insert(co, *queue)->next;
+
+ /* add children to list as well */
+ if (obj->flags & UNINTERESTING)
+ consume_children = 0;
+ else
+ consume_children = 1;
+ }
+
+ /* open parents */
+ if (entry->merge_nr) {
+ int j, off = index + sizeof(struct rc_object_entry_ondisk);
+ char flag = entry->uninteresting ? UPATH : IPATH;
+
+ for (j = 0; j < entry->merge_nr; j++) {
+ unsigned short p = ntohs(*(uint16_t *)(map + off + RC_PATH_SIZE(j)));
+
+ if (p >= total_path_nr)
+ goto end;
+
+ if (paths[p] & flag)
+ continue;
+
+ paths[p] |= flag;
+ }
+
+ /* make sure we don't use this path before all our parents have had their say */
+ SET_COUNT(paths[path], entry->merge_nr);
+ }
+
+ }
+
+ retval = 0;
+
+end:
+ free(paths);
+ free(last_objects);
+
+ return retval;
+}
+
+static int get_cache_slice_header(unsigned char *cache_sha1, unsigned char *map, int len, struct rc_slice_header *head)
+{
+ int t;
+
+ memcpy(head, map, sizeof(struct rc_slice_header));
+ head->ofs_objects = ntohl(head->ofs_objects);
+ head->object_nr = ntohl(head->object_nr);
+ head->size = ntohl(head->size);
+ head->path_nr = ntohs(head->path_nr);
+
+ if (memcmp(head->signature, "REVCACHE", 8))
+ return -1;
+ if (head->version != SUPPORTED_REVCACHE_VERSION)
+ return -2;
+ if (hashcmp(head->sha1, cache_sha1))
+ return -3;
+ t = sizeof(struct rc_slice_header);
+ if (t != head->ofs_objects || t >= len)
+ return -4;
+
+ head->size = len;
+
+ return 0;
+}
+
+int traverse_cache_slice(struct rev_info *revs,
+ unsigned char *cache_sha1, struct commit *commit,
+ unsigned long *date_so_far, int *slop_so_far,
+ struct commit_list ***queue, struct commit_list **work)
+{
+ int fd = -1, retval = -3;
+ struct stat fi;
+ struct rc_slice_header head;
+ struct rev_cache_info *rci;
+ unsigned char *map = MAP_FAILED;
+
+ /* the index should've been loaded already to find cache_sha1, but it's good
+ * to be absolutely sure... */
+ if (!idx_map)
+ init_index();
+ if (!idx_map)
+ return -1;
+
+ /* load options */
+ rci = &revs->rev_cache_info;
+
+ memset(&head, 0, sizeof(struct rc_slice_header));
+
+ fd = open(git_path("rev-cache/%s", sha1_to_hex(cache_sha1)), O_RDONLY);
+ if (fd == -1)
+ goto end;
+ if (fstat(fd, &fi) || fi.st_size < sizeof(struct rc_slice_header))
+ goto end;
+
+ map = xmmap(0, fi.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ if (map == MAP_FAILED)
+ goto end;
+ if (get_cache_slice_header(cache_sha1, map, fi.st_size, &head))
+ goto end;
+
+ retval = traverse_cache_slice_1(&head, map, revs, commit, date_so_far, slop_so_far, queue, work);
+
+end:
+ if (map != MAP_FAILED)
+ munmap(map, fi.st_size);
+ if (fd != -1)
+ close(fd);
+
+ return retval;
+}
+
+
+
+/* generation */
+
+struct path_track {
+ struct commit *commit;
+ int path; /* for keeping track of children */
+
+ struct path_track *next, *prev;
+};
+
+static unsigned char *paths;
+static int path_nr = 1, path_sz;
+
+static struct path_track *path_track;
+static struct path_track *path_track_alloc;
+
+#define PATH_IN_USE 0x80 /* biggest bit we can get as a char */
+
+static int get_new_path(void)
+{
+ int i;
+
+ for (i = 1; i < path_nr; i++)
+ if (!paths[i])
+ break;
+
+ if (i == path_nr) {
+ if (path_nr >= path_sz) {
+ path_sz += 50;
+ paths = xrealloc(paths, path_sz);
+ memset(paths + path_sz - 50, 0, 50);
+ }
+ path_nr++;
+ }
+
+ paths[i] = PATH_IN_USE;
+ return i;
+}
+
+static void remove_path_track(struct path_track **ppt, char total_free)
+{
+ struct path_track *t = *ppt;
+
+ if (t->next)
+ t->next->prev = t->prev;
+ if (t->prev)
+ t->prev->next = t->next;
+
+ t = t->next;
+
+ if (total_free)
+ free(*ppt);
+ else {
+ (*ppt)->next = path_track_alloc;
+ path_track_alloc = *ppt;
+ }
+
+ *ppt = t;
+}
+
+static struct path_track *make_path_track(struct path_track **head, struct commit *commit)
+{
+ struct path_track *pt;
+
+ if (path_track_alloc) {
+ pt = path_track_alloc;
+ path_track_alloc = pt->next;
+ } else
+ pt = xmalloc(sizeof(struct path_track));
+
+ memset(pt, 0, sizeof(struct path_track));
+ pt->commit = commit;
+
+ pt->next = *head;
+ if (*head)
+ (*head)->prev = pt;
+ *head = pt;
+
+ return pt;
+}
+
+static void add_path_to_track(struct commit *commit, int path)
+{
+ make_path_track(&path_track, commit);
+ path_track->path = path;
+}
+
+static void handle_paths(struct commit *commit, struct rc_object_entry *object, struct strbuf *merge_str, struct strbuf *split_str)
+{
+ int child_nr, parent_nr, open_parent_nr, this_path;
+ struct commit_list *list;
+ struct commit *first_parent;
+ struct path_track **ppt, *pt;
+
+ /* we can only re-use a closed path once all it's children have been encountered,
+ * as we need to keep track of commit boundaries */
+ ppt = &path_track;
+ pt = *ppt;
+ child_nr = 0;
+ while (pt) {
+ if (pt->commit == commit) {
+ uint16_t write_path;
+
+ if (paths[pt->path] != PATH_IN_USE)
+ paths[pt->path]--;
+
+ /* make sure we can handle this */
+ child_nr++;
+ if (child_nr > 0x7f)
+ die("%s: too many branches! rev-cache can only handle %d parents/children per commit",
+ sha1_to_hex(object->sha1), 0x7f);
+
+ /* add to split list */
+ object->split_nr++;
+ write_path = htons((uint16_t)pt->path);
+ strbuf_add(split_str, &write_path, sizeof(uint16_t));
+
+ remove_path_track(ppt, 0);
+ pt = *ppt;
+ } else {
+ pt = pt->next;
+ ppt = &pt;
+ }
+ }
+
+ /* initialize our self! */
+ if (!commit->indegree) {
+ commit->indegree = get_new_path();
+ object->is_start = 1;
+ }
+
+ this_path = commit->indegree;
+ paths[this_path] = PATH_IN_USE;
+ object->path = this_path;
+
+ /* count interesting parents */
+ parent_nr = open_parent_nr = 0;
+ first_parent = 0;
+ for (list = commit->parents; list; list = list->next) {
+ if (list->item->object.flags & UNINTERESTING) {
+ object->is_end = 1;
+ continue;
+ }
+
+ parent_nr++;
+ if (!list->item->indegree)
+ open_parent_nr++;
+ if (!first_parent)
+ first_parent = list->item;
+ }
+
+ if (!parent_nr)
+ return;
+
+ if (parent_nr == 1 && open_parent_nr == 1) {
+ first_parent->indegree = this_path;
+ return;
+ }
+
+ /* bail out on obscene parent/child #s */
+ if (parent_nr > 0x7f)
+ die("%s: too many parents in merge! rev-cache can only handle %d parents/children per commit",
+ sha1_to_hex(object->sha1), 0x7f);
+
+ /* make merge list */
+ object->merge_nr = parent_nr;
+ paths[this_path] = parent_nr;
+
+ for (list = commit->parents; list; list = list->next) {
+ struct commit *p = list->item;
+ uint16_t write_path;
+
+ if (p->object.flags & UNINTERESTING)
+ continue;
+
+ /* unfortunately due to boundary tracking we can't re-use merge paths
+ * (unable to guarantee last parent path = this -> last won't always be able to
+ * set this as a boundary object */
+ if (!p->indegree)
+ p->indegree = get_new_path();
+
+ write_path = htons((uint16_t)p->indegree);
+ strbuf_add(merge_str, &write_path, sizeof(uint16_t));
+
+ /* make sure path is properly ended */
+ add_path_to_track(p, this_path);
+ }
+
+}
+
+
+static void add_object_entry(const unsigned char *sha1, int type, struct rc_object_entry *nothisone,
+ struct strbuf *merge_str, struct strbuf *split_str)
+{
+ struct rc_object_entry object;
+
+ if (!nothisone) {
+ memset(&object, 0, sizeof(object));
+ object.sha1 = (unsigned char *)sha1;
+ object.type = type;
+
+ if (merge_str)
+ object.merge_nr = merge_str->len / sizeof(uint16_t);
+ if (split_str)
+ object.split_nr = split_str->len / sizeof(uint16_t);
+
+ nothisone = &object;
+ }
+
+ strbuf_add(acc_buffer, to_disked_rc_object_entry(nothisone, 0), sizeof(struct rc_object_entry_ondisk));
+
+ if (merge_str && merge_str->len)
+ strbuf_add(acc_buffer, merge_str->buf, merge_str->len);
+ if (split_str && split_str->len)
+ strbuf_add(acc_buffer, split_str->buf, split_str->len);
+
+}
+
+static void init_revcache_directory(void)
+{
+ struct stat fi;
+
+ if (stat(git_path("rev-cache"), &fi) || !S_ISDIR(fi.st_mode))
+ if (mkdir(git_path("rev-cache"), 0666))
+ die("can't make rev-cache directory");
+
+}
+
+void init_rev_cache_info(struct rev_cache_info *rci)
+{
+ rci->objects = 1;
+ rci->legs = 0;
+ rci->make_index = 1;
+
+ rci->add_to_pending = 1;
+
+ rci->ignore_size = 0;
+}
+
+void maybe_fill_with_defaults(struct rev_cache_info *rci)
+{
+ static struct rev_cache_info def_rci;
+
+ if (rci)
+ return;
+
+ init_rev_cache_info(&def_rci);
+ rci = &def_rci;
+}
+
+int make_cache_slice(struct rev_cache_info *rci,
+ struct rev_info *revs, struct commit_list **starts, struct commit_list **ends,
+ unsigned char *cache_sha1)
+{
+ struct rev_info therevs;
+ struct strbuf buffer, startlist, endlist;
+ struct rc_slice_header head;
+ struct commit *commit;
+ unsigned char sha1[20];
+ struct strbuf merge_paths, split_paths;
+ int object_nr, total_sz, fd;
+ char file[PATH_MAX], *newfile;
+ struct rev_cache_info *trci;
+ git_SHA_CTX ctx;
+
+ maybe_fill_with_defaults(rci);
+
+ init_revcache_directory();
+ strcpy(file, git_path("rev-cache/XXXXXX"));
+ fd = xmkstemp(file);
+
+ strbuf_init(&buffer, 0);
+ strbuf_init(&startlist, 0);
+ strbuf_init(&endlist, 0);
+ strbuf_init(&merge_paths, 0);
+ strbuf_init(&split_paths, 0);
+ acc_buffer = &buffer;
+
+ if (!revs) {
+ revs = &therevs;
+ init_revisions(revs, 0);
+
+ /* we're gonna assume no one else has already traversed this... */
+ while ((commit = pop_commit(starts)))
+ add_pending_object(revs, &commit->object, 0);
+
+ while ((commit = pop_commit(ends))) {
+ commit->object.flags |= UNINTERESTING;
+ add_pending_object(revs, &commit->object, 0);
+ }
+ }
+
+ /* write head placeholder */
+ memset(&head, 0, sizeof(head));
+ head.ofs_objects = htonl(sizeof(head));
+ xwrite(fd, &head, sizeof(head));
+
+ /* init revisions! */
+ revs->tree_objects = 1;
+ revs->blob_objects = 1;
+ revs->topo_order = 1;
+ revs->lifo = 1;
+
+ /* re-use info from other caches if possible */
+ trci = &revs->rev_cache_info;
+ init_rev_cache_info(trci);
+ trci->add_to_pending = 0;
+
+ setup_revisions(0, 0, revs, 0);
+ if (prepare_revision_walk(revs))
+ die("died preparing revision walk");
+
+ object_nr = total_sz = 0;
+ while ((commit = get_revision(revs)) != 0) {
+ struct rc_object_entry object;
+
+ strbuf_setlen(&merge_paths, 0);
+ strbuf_setlen(&split_paths, 0);
+
+ memset(&object, 0, sizeof(object));
+ object.type = OBJ_COMMIT;
+ object.date = commit->date;
+ object.sha1 = commit->object.sha1;
+
+ handle_paths(commit, &object, &merge_paths, &split_paths);
+
+ if (object.is_end) {
+ strbuf_add(&endlist, object.sha1, 20);
+ if (ends)
+ commit_list_insert(commit, ends);
+ }
+ /* the two *aren't* mutually exclusive */
+ if (object.is_start) {
+ strbuf_add(&startlist, object.sha1, 20);
+ if (starts)
+ commit_list_insert(commit, starts);
+ }
+
+ commit->indegree = 0;
+
+ add_object_entry(0, 0, &object, &merge_paths, &split_paths);
+ object_nr++;
+
+ /* print every ~1MB or so */
+ if (buffer.len > 1000000) {
+ write_in_full(fd, buffer.buf, buffer.len);
+ total_sz += buffer.len;
+
+ strbuf_setlen(&buffer, 0);
+ }
+ }
+
+ if (buffer.len) {
+ write_in_full(fd, buffer.buf, buffer.len);
+ total_sz += buffer.len;
+ }
+
+ /* go ahead a free some stuff... */
+ strbuf_release(&buffer);
+ strbuf_release(&merge_paths);
+ strbuf_release(&split_paths);
+ if (path_sz)
+ free(paths);
+ while (path_track_alloc)
+ remove_path_track(&path_track_alloc, 1);
+
+ /* the meaning of the hash name is more or less irrelevant, it's the uniqueness that matters */
+ strbuf_add(&endlist, startlist.buf, startlist.len);
+ git_SHA1_Init(&ctx);
+ git_SHA1_Update(&ctx, endlist.buf, endlist.len);
+ git_SHA1_Final(sha1, &ctx);
+
+ /* now actually initialize header */
+ strcpy(head.signature, "REVCACHE");
+ head.version = SUPPORTED_REVCACHE_VERSION;
+
+ head.object_nr = htonl(object_nr);
+ head.size = htonl(ntohl(head.ofs_objects) + total_sz);
+ head.path_nr = htons(path_nr);
+ hashcpy(head.sha1, sha1);
+
+ /* some info! */
+ fprintf(stderr, "objects: %d\n", object_nr);
+ fprintf(stderr, "paths: %d\n", path_nr);
+
+ lseek(fd, 0, SEEK_SET);
+ xwrite(fd, &head, sizeof(head));
+
+ if (rci->make_index && make_cache_index(rci, sha1, fd, ntohl(head.size)) < 0)
+ die("can't update index");
+
+ close(fd);
+
+ newfile = git_path("rev-cache/%s", sha1_to_hex(sha1));
+ if (rename(file, newfile))
+ die("can't move temp file");
+
+ /* let our caller know what we've just made */
+ if (cache_sha1)
+ hashcpy(cache_sha1, sha1);
+
+ strbuf_release(&endlist);
+ strbuf_release(&startlist);
+
+ return 0;
+}
+
+
+static int index_sort_hash(const void *a, const void *b)
+{
+ return hashcmp(((struct rc_index_entry_ondisk *)a)->sha1, ((struct rc_index_entry_ondisk *)b)->sha1);
+}
+
+static int write_cache_index(struct strbuf *body)
+{
+ struct rc_index_header whead;
+ struct lock_file *lk;
+ int fd, i;
+
+ /* clear index map if loaded */
+ if (idx_map) {
+ munmap(idx_map, idx_size);
+ idx_map = 0;
+ }
+
+ lk = xcalloc(sizeof(struct lock_file), 1);
+ fd = hold_lock_file_for_update(lk, git_path("rev-cache/index"), 0);
+ if (fd < 0) {
+ free(lk);
+ return -1;
+ }
+
+ /* endianness yay! */
+ memset(&whead, 0, sizeof(whead));
+ memcpy(whead.signature, "REVINDEX", 8);
+ whead.version = idx_head.version;
+ whead.ofs_objects = htonl(idx_head.ofs_objects);
+ whead.object_nr = htonl(idx_head.object_nr);
+ whead.cache_nr = idx_head.cache_nr;
+ whead.max_date = htonl(idx_head.max_date);
+
+ write(fd, &whead, sizeof(struct rc_index_header));
+ write_in_full(fd, idx_caches, idx_head.cache_nr * 20);
+
+ for (i = 0; i <= 0xff; i++)
+ fanout[i] = htonl(fanout[i]);
+ write_in_full(fd, fanout, 0x100 * sizeof(uint32_t));
+
+ write_in_full(fd, body->buf, body->len);
+
+ if (commit_lock_file(lk) < 0)
+ return -2;
+
+ /* lk freed by lockfile.c */
+
+ return 0;
+}
+
+int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
+ int fd, unsigned int size)
+{
+ struct strbuf buffer;
+ int i, cache_index, cur;
+ unsigned char *map;
+ unsigned long max_date;
+
+ if (!idx_map)
+ init_index();
+
+ lseek(fd, 0, SEEK_SET);
+ map = xmmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
+ if (map == MAP_FAILED)
+ return -1;
+
+ strbuf_init(&buffer, 0);
+ if (idx_map) {
+ strbuf_add(&buffer, idx_map + fanout[0], fanout[0x100] - fanout[0]);
+ } else {
+ /* not an update */
+ memset(&idx_head, 0, sizeof(struct rc_index_header));
+ idx_caches = 0;
+
+ strcpy(idx_head.signature, "REVINDEX");
+ idx_head.version = SUPPORTED_REVINDEX_VERSION;
+ idx_head.ofs_objects = sizeof(struct rc_index_header) + 0x100 * sizeof(uint32_t);
+ }
+
+ /* are we remaking a slice? */
+ for (i = 0; i < idx_head.cache_nr; i++)
+ if (!hashcmp(idx_caches + i * 20, cache_sha1))
+ break;
+
+ if (i == idx_head.cache_nr) {
+ cache_index = idx_head.cache_nr++;
+ idx_head.ofs_objects += 20;
+
+ idx_caches = xrealloc(idx_caches, idx_head.cache_nr * 20);
+ hashcpy(idx_caches + cache_index * 20, cache_sha1);
+ } else
+ cache_index = i;
+
+ i = sizeof(struct rc_slice_header); /* offset */
+ max_date = idx_head.max_date;
+ while (i < size) {
+ struct rc_index_entry index_entry, *entry;
+ struct rc_index_entry_ondisk *disked_entry;
+ struct rc_object_entry *object_entry = RC_OBTAIN_OBJECT_ENTRY(map + i);
+ unsigned long date;
+ int off, pos = i;
+
+ i += RC_ACTUAL_OBJECT_ENTRY_SIZE(object_entry);
+
+ if (object_entry->type != OBJ_COMMIT)
+ continue;
+
+ /* don't include ends; otherwise we'll find ourselves in loops */
+ if (object_entry->is_end)
+ continue;
+
+ /* handle index duplication
+ * -> keep old copy unless new one is a start -- based on expected usage, older ones will be more
+ * likely to lead to greater slice traversals than new ones */
+ date = object_entry->date;
+ if (date > idx_head.max_date) {
+ disked_entry = 0;
+ if (date > max_date)
+ max_date = date;
+ } else
+ disked_entry = search_index_1(object_entry->sha1);
+
+ if (disked_entry && !object_entry->is_start)
+ continue;
+ else if (disked_entry) {
+ /* mmm, pointer arithmetic... tasty */ /* (entry - idx_map = offset, so cast is valid) */
+ off = (unsigned int)((unsigned char *)disked_entry - idx_map) - fanout[0];
+ disked_entry = (struct rc_index_entry_ondisk *)(buffer.buf + off);
+ entry = from_disked_rc_index_entry(disked_entry, 0);
+ } else
+ entry = &index_entry;
+
+ memset(entry, 0, sizeof(index_entry));
+ entry->sha1 = object_entry->sha1;
+ entry->is_start = object_entry->is_start;
+ entry->cache_index = cache_index;
+ entry->pos = pos;
+
+ if (entry == &index_entry) {
+ strbuf_add(&buffer, to_disked_rc_index_entry(entry, 0), sizeof(struct rc_index_entry_ondisk));
+ idx_head.object_nr++;
+ } else
+ to_disked_rc_index_entry(entry, disked_entry);
+
+ }
+
+ idx_head.max_date = max_date;
+ qsort(buffer.buf, buffer.len / sizeof(struct rc_index_entry_ondisk), sizeof(struct rc_index_entry_ondisk), index_sort_hash);
+
+ /* generate fanout */
+ cur = 0x00;
+ for (i = 0; i < buffer.len; i += sizeof(struct rc_index_entry_ondisk)) {
+ struct rc_index_entry_ondisk *entry = (struct rc_index_entry_ondisk *)(buffer.buf + i);
+
+ while (cur <= entry->sha1[0])
+ fanout[cur++] = i + idx_head.ofs_objects;
+ }
+
+ while (cur <= 0xff)
+ fanout[cur++] = idx_head.ofs_objects + buffer.len;
+
+ /* BOOM! */
+ if (write_cache_index(&buffer))
+ return -1;
+
+ munmap(map, size);
+ strbuf_release(&buffer);
+
+ /* idx_map is unloaded without cleanup_cache_slices(), so regardless of previous index existence
+ * we can still free this up */
+ free(idx_caches);
+
+ return 0;
+}
+
+
+/* add start-commits from each cache slice (uninterestingness will be propogated) */
+void starts_from_slices(struct rev_info *revs, unsigned int flags)
+{
+ struct commit *commit;
+ int i;
+
+ if (!idx_map)
+ init_index();
+ if (!idx_map)
+ return;
+
+ for (i = idx_head.ofs_objects; i < idx_size; i += sizeof(struct rc_index_entry_ondisk)) {
+ struct rc_index_entry *entry = RC_OBTAIN_INDEX_ENTRY(idx_map + i);
+
+ if (!entry->is_start)
+ continue;
+
+ commit = lookup_commit(entry->sha1);
+ if (!commit)
+ continue;
+
+ commit->object.flags |= flags;
+ add_pending_object(revs, &commit->object, 0);
+ }
+
+}
diff --git a/rev-cache.h b/rev-cache.h
new file mode 100644
index 0000000..a471fbf
--- /dev/null
+++ b/rev-cache.h
@@ -0,0 +1,107 @@
+#ifndef REV_CACHE_H
+#define REV_CACHE_H
+
+#define SUPPORTED_REVCACHE_VERSION 1
+#define SUPPORTED_REVINDEX_VERSION 1
+
+#define RC_PATH_SIZE(x) (sizeof(uint16_t) * (x))
+
+#define RC_OBTAIN_OBJECT_ENTRY(p) from_disked_rc_object_entry((struct rc_object_entry_ondisk *)(p), 0)
+#define RC_OBTAIN_INDEX_ENTRY(p) from_disked_rc_index_entry((struct rc_index_entry_ondisk *)(p), 0)
+
+#define RC_ACTUAL_OBJECT_ENTRY_SIZE(e) (sizeof(struct rc_object_entry_ondisk) + RC_PATH_SIZE((e)->merge_nr + (e)->split_nr) + (e)->size_size)
+
+/* single index maps objects to cache files */
+struct rc_index_header {
+ char signature[8]; /* REVINDEX */
+ unsigned char version;
+ uint32_t ofs_objects;
+
+ uint32_t object_nr;
+ unsigned char cache_nr;
+
+ uint32_t max_date;
+};
+
+struct rc_index_entry_ondisk {
+ unsigned char sha1[20];
+ unsigned char flags;
+ uint32_t pos;
+};
+
+struct rc_index_entry {
+ unsigned char *sha1;
+ unsigned is_start : 1;
+ unsigned cache_index : 7;
+ uint32_t pos;
+};
+
+
+/* structure for actual cache file */
+struct rc_slice_header {
+ char signature[8]; /* REVCACHE */
+ unsigned char version;
+ uint32_t ofs_objects;
+
+ uint32_t object_nr;
+ uint16_t path_nr;
+ uint32_t size;
+
+ unsigned char sha1[20];
+};
+
+struct rc_object_entry_ondisk {
+ unsigned char flags;
+ unsigned char sha1[20];
+
+ unsigned char merge_nr;
+ unsigned char split_nr;
+ unsigned char sizes;
+
+ uint32_t date;
+ uint16_t path;
+};
+
+struct rc_object_entry {
+ unsigned type : 3;
+ unsigned is_end : 1;
+ unsigned is_start : 1;
+ unsigned uninteresting : 1;
+ unsigned include : 1;
+ unsigned flag : 1; /* unused */
+ unsigned char *sha1; /* 20 byte */
+
+ unsigned char merge_nr; /* : 7 */
+ unsigned char split_nr; /* : 7 */
+ unsigned size_size : 3;
+ unsigned padding : 5;
+
+ uint32_t date;
+ uint16_t path;
+
+ /* merge paths */
+ /* split paths */
+ /* size */
+};
+
+struct rc_index_entry *from_disked_rc_index_entry(struct rc_index_entry_ondisk *src, struct rc_index_entry *dst);
+struct rc_index_entry_ondisk *to_disked_rc_index_entry(struct rc_index_entry *src, struct rc_index_entry_ondisk *dst);
+struct rc_object_entry *from_disked_rc_object_entry(struct rc_object_entry_ondisk *src, struct rc_object_entry *dst);
+struct rc_object_entry_ondisk *to_disked_rc_object_entry(struct rc_object_entry *src, struct rc_object_entry_ondisk *dst);
+
+extern unsigned char *get_cache_slice(struct commit *commit);
+extern int traverse_cache_slice(struct rev_info *revs,
+ unsigned char *cache_sha1, struct commit *commit,
+ unsigned long *date_so_far, int *slop_so_far,
+ struct commit_list ***queue, struct commit_list **work);
+
+extern void init_rev_cache_info(struct rev_cache_info *rci);
+extern int make_cache_slice(struct rev_cache_info *rci,
+ struct rev_info *revs, struct commit_list **starts, struct commit_list **ends,
+ unsigned char *cache_sha1);
+extern int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
+ int fd, unsigned int size);
+
+extern void starts_from_slices(struct rev_info *revs, unsigned int flags);
+
+#endif
diff --git a/revision.c b/revision.c
index 9f5dac5..485bf72 100644
--- a/revision.c
+++ b/revision.c
@@ -432,7 +432,7 @@ static void try_to_simplify_commit(struct rev_info *revs, struct commit *commit)
commit->object.flags |= TREESAME;
}
-static void insert_by_date_cached(struct commit *p, struct commit_list **head,
+void insert_by_date_cached(struct commit *p, struct commit_list **head,
struct commit_list *cached_base, struct commit_list **cache)
{
struct commit_list *new_entry;
diff --git a/revision.h b/revision.h
index fb74492..e767a43 100644
--- a/revision.h
+++ b/revision.h
@@ -13,11 +13,25 @@
#define CHILD_SHOWN (1u<<6)
#define ADDED (1u<<7) /* Parents already parsed and added? */
#define SYMMETRIC_LEFT (1u<<8)
-#define ALL_REV_FLAGS ((1u<<9)-1)
+#define FACE_VALUE (1u<<9)
+#define ALL_REV_FLAGS ((1u<<10)-1)
struct rev_info;
struct log_info;
+struct rev_cache_info {
+ /* generation flags */
+ unsigned objects : 1,
+ legs : 1,
+ make_index : 1;
+
+ /* traversal flags */
+ unsigned add_to_pending : 1;
+
+ /* fuse options */
+ unsigned int ignore_size;
+};
+
struct rev_info {
/* Starting list */
struct commit_list *commits;
@@ -73,6 +87,10 @@ struct rev_info {
dense_combined_merges:1,
always_show_header:1;
+ /* rev-cache flags */
+ unsigned int for_pack:1,
+ dont_cache_me:1;
+
/* Format info */
unsigned int shown_one:1,
show_merge:1,
@@ -116,6 +134,9 @@ struct rev_info {
struct reflog_walk_info *reflog_info;
struct decoration children;
struct decoration merge_simplification;
+
+ /* caching info, used ONLY by traverse_cache_slice */
+ struct rev_cache_info rev_cache_info;
};
#define REV_TREE_SAME 0
@@ -167,4 +188,7 @@ enum commit_action {
extern enum commit_action simplify_commit(struct rev_info *revs, struct commit *commit);
+extern void insert_by_date_cached(struct commit *p, struct commit_list **head,
+ struct commit_list *cached_base, struct commit_list **cache);
+
#endif
diff --git a/t/t6015-rev-cache-list.sh b/t/t6015-rev-cache-list.sh
new file mode 100755
index 0000000..e7474fd
--- /dev/null
+++ b/t/t6015-rev-cache-list.sh
@@ -0,0 +1,104 @@
+#!/bin/sh
+
+test_description='git rev-cache tests'
+. ./test-lib.sh
+
+test_cmp_sorted() {
+ grep -io "[a-f0-9]*" $1 | sort >.tmpfile1 &&
+ grep -io "[a-f0-9]*" $2 | sort >.tmpfile2 &&
+ test_cmp .tmpfile1 .tmpfile2
+}
+
+# we want a totally wacked out branch structure...
+# we need branching and merging of sizes up through 3, tree
+# addition/deletion, and enough branching to exercise path
+# reuse
+test_expect_success 'init repo' '
+ echo bla >file &&
+ git add . &&
+ git commit -m "bla" &&
+
+ git branch b1 &&
+ git checkout b1 &&
+ echo blu >file2 &&
+ mkdir d1 &&
+ echo bang >d1/filed1 &&
+ git add . &&
+ git commit -m "blu" &&
+
+ git checkout master &&
+ git branch b2 &&
+ git checkout b2 &&
+ echo kaplaa >>file &&
+ git commit -a -m "kaplaa" &&
+
+ git checkout master &&
+ mkdir smoke &&
+ echo omg >smoke/bong &&
+ git add . &&
+ git commit -m "omg" &&
+
+ git branch b4 &&
+ git checkout b4 &&
+ echo shazam >file8 &&
+ git add . &&
+ git commit -m "shazam" &&
+ git merge -m "merge b2" b2 &&
+
+ echo bam >smoke/pipe &&
+ git add .
+ git commit -m "bam" &&
+
+ git checkout master &&
+ echo pow >file7 &&
+ git add . &&
+ git commit -m "pow" &&
+ git merge -m "merge b4" b4 &&
+
+ git checkout b1 &&
+ echo stuff >d1/filed1 &&
+ git commit -a -m "stuff" &&
+
+ git branch b11 &&
+ git checkout b11 &&
+ echo wazzup >file3 &&
+ git add file3 &&
+ git commit -m "wazzup" &&
+
+ git checkout b1 &&
+ mkdir d1/d2 &&
+ echo lol >d1/d2/filed2 &&
+ git add . &&
+ git commit -m "lol" &&
+
+ git checkout master &&
+ git merge -m "triple merge" b1 b11 &&
+ git rm -r d1 &&
+ git commit -a -m "oh noes"
+'
+
+git-rev-list HEAD --not HEAD~3 >proper_commit_list_limited
+git-rev-list HEAD >proper_commit_list
+
+test_expect_success 'make cache slice' '
+ git-rev-cache add HEAD 2>output.err &&
+ grep "final return value: 0" output.err
+'
+
+test_expect_success 'remake cache slice' '
+ git-rev-cache add HEAD 2>output.err &&
+ grep "final return value: 0" output.err
+'
+
+#check core mechanics and rev-list hook for commits
+test_expect_success 'test rev-caches walker directly (limited)' '
+ git-rev-cache walk HEAD --not HEAD~3 >list &&
+ test_cmp_sorted list proper_commit_list_limited
+'
+
+test_expect_success 'test rev-caches walker directly (unlimited)' '
+ git-rev-cache walk HEAD >list &&
+ test_cmp_sorted list proper_commit_list
+'
+
+test_done
--
tg: (64d5fe0..) t/revcache/basic (depends on: master)
^ permalink raw reply related
* [PATCH 3/6 (v4)] support for non-commit object caching in rev-cache
From: Nick Edelen @ 2009-08-17 12:31 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
Summarized, this third patch contains:
- support for non-commit object caching
- expansion of porcelain to accomodate non-commit objects
- appropriate tests
Objects are stored relative to the commit in which they were introduced --
commits are 'diffed' against their parents. This will eliminate the need for
tree recursion in cached commits (significantly reducing I/O), and potentially
be useful to external applications.
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
rev-cache.c | 204 ++++++++++++++++++++++++++++++++++++++++++++-
t/t6015-rev-cache-list.sh | 8 ++
2 files changed, 209 insertions(+), 3 deletions(-)
diff --git a/rev-cache.c b/rev-cache.c
index a01db87..ddcc596 100644
--- a/rev-cache.c
+++ b/rev-cache.c
@@ -257,6 +257,32 @@ unsigned char *get_cache_slice(struct commit *commit)
/* traversal */
+static void handle_noncommit(struct rev_info *revs, unsigned char *ptr, struct rc_object_entry *entry)
+{
+ struct object *obj = 0;
+
+ switch (entry->type) {
+ case OBJ_TREE :
+ if (revs->tree_objects)
+ obj = (struct object *)lookup_tree(entry->sha1);
+ break;
+ case OBJ_BLOB :
+ if (revs->blob_objects)
+ obj = (struct object *)lookup_blob(entry->sha1);
+ break;
+ case OBJ_TAG :
+ if (revs->tag_objects)
+ obj = (struct object *)lookup_tag(entry->sha1);
+ break;
+ }
+
+ if (!obj)
+ return;
+
+ obj->flags |= FACE_VALUE;
+ add_pending_object(revs, obj, "");
+}
+
static int setup_traversal(struct rc_slice_header *head, unsigned char *map, struct commit *commit, struct commit_list **work)
{
struct rc_index_entry *iep;
@@ -293,7 +319,7 @@ static int setup_traversal(struct rc_slice_header *head, unsigned char *map, str
if (iep->pos < retval)
retval = iep->pos;
-
+
oep = RC_OBTAIN_OBJECT_ENTRY(map + iep->pos);
/* mark this for later */
@@ -345,9 +371,12 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
i += RC_ACTUAL_OBJECT_ENTRY_SIZE(entry);
/* add extra objects if necessary */
- if (entry->type != OBJ_COMMIT)
+ if (entry->type != OBJ_COMMIT) {
+ if (consume_children)
+ handle_noncommit(revs, map + index, entry);
+
continue;
- else
+ } else
consume_children = 0;
if (path >= total_path_nr)
@@ -775,6 +804,171 @@ static void add_object_entry(const unsigned char *sha1, int type, struct rc_obje
}
+/* returns non-zero to continue parsing, 0 to skip */
+typedef int (*dump_tree_fn)(const unsigned char *, const char *, unsigned int); /* sha1, path, mode */
+
+/* we need to walk the trees by hash, so unfortunately we can't use traverse_trees in tree-walk.c */
+static int dump_tree(struct tree *tree, dump_tree_fn fn)
+{
+ struct tree_desc desc;
+ struct name_entry entry;
+ struct tree *subtree;
+ int r;
+
+ if (parse_tree(tree))
+ return -1;
+
+ init_tree_desc(&desc, tree->buffer, tree->size);
+ while (tree_entry(&desc, &entry)) {
+ switch (fn(entry.sha1, entry.path, entry.mode)) {
+ case 0 :
+ goto continue_loop;
+ default :
+ break;
+ }
+
+ if (S_ISDIR(entry.mode)) {
+ subtree = lookup_tree(entry.sha1);
+ if (!subtree)
+ return -2;
+
+ if ((r = dump_tree(subtree, fn)) < 0)
+ return r;
+ }
+
+continue_loop:
+ continue;
+ }
+
+ return 0;
+}
+
+static int dump_tree_callback(const unsigned char *sha1, const char *path, unsigned int mode)
+{
+ unsigned char data[21];
+
+ hashcpy(data, sha1);
+ data[20] = !!S_ISDIR(mode);
+
+ strbuf_add(acc_buffer, data, 21);
+
+ return 1;
+}
+
+static void tree_addremove(struct diff_options *options,
+ int whatnow, unsigned mode,
+ const unsigned char *sha1,
+ const char *concatpath)
+{
+ unsigned char data[21];
+
+ if (whatnow != '+')
+ return;
+
+ hashcpy(data, sha1);
+ data[20] = !!S_ISDIR(mode);
+
+ strbuf_add(acc_buffer, data, 21);
+}
+
+static void tree_change(struct diff_options *options,
+ unsigned old_mode, unsigned new_mode,
+ const unsigned char *old_sha1,
+ const unsigned char *new_sha1,
+ const char *concatpath)
+{
+ unsigned char data[21];
+
+ if (!hashcmp(old_sha1, new_sha1))
+ return;
+
+ hashcpy(data, new_sha1);
+ data[20] = !!S_ISDIR(new_mode);
+
+ strbuf_add(acc_buffer, data, 21);
+}
+
+static int sort_type_hash(const void *a, const void *b)
+{
+ const unsigned char *sa = (const unsigned char *)a,
+ *sb = (const unsigned char *)b;
+
+ if (sa[20] == sb[20])
+ return hashcmp(sa, sb);
+
+ return sa[20] > sb[20] ? -1 : 1;
+}
+
+static int add_unique_objects(struct commit *commit)
+{
+ struct commit_list *list;
+ struct strbuf os, ost, *orig_buf;
+ struct diff_options opts;
+ int i, j, next;
+ char is_first = 1;
+
+ strbuf_init(&os, 0);
+ strbuf_init(&ost, 0);
+ orig_buf = acc_buffer;
+
+ diff_setup(&opts);
+ DIFF_OPT_SET(&opts, RECURSIVE);
+ DIFF_OPT_SET(&opts, TREE_IN_RECURSIVE);
+ opts.change = tree_change;
+ opts.add_remove = tree_addremove;
+
+ /* this is only called for non-ends (ie. all parents interesting) */
+ for (list = commit->parents; list; list = list->next) {
+ if (is_first)
+ acc_buffer = &os;
+ else
+ acc_buffer = &ost;
+
+ strbuf_setlen(acc_buffer, 0);
+ diff_tree_sha1(list->item->tree->object.sha1, commit->tree->object.sha1, "", &opts);
+ qsort(acc_buffer->buf, acc_buffer->len / 21, 21, (int (*)(const void *, const void *))hashcmp);
+
+ /* take intersection */
+ if (!is_first) {
+ for (next = i = j = 0; i < os.len; i += 21) {
+ while (j < ost.len && hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)) < 0)
+ j += 21;
+
+ if (j >= ost.len || hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)))
+ continue;
+
+ if (next != i)
+ memcpy(os.buf + next, os.buf + i, 21);
+ next += 21;
+ }
+
+ if (next != i)
+ strbuf_setlen(&os, next);
+ } else
+ is_first = 0;
+ }
+
+ if (is_first) {
+ acc_buffer = &os;
+ dump_tree(commit->tree, dump_tree_callback);
+ }
+
+ if (os.len)
+ qsort(os.buf, os.len / 21, 21, sort_type_hash);
+
+ acc_buffer = orig_buf;
+ for (i = 0; i < os.len; i += 21)
+ add_object_entry((unsigned char *)(os.buf + i), os.buf[i + 20] ? OBJ_TREE : OBJ_BLOB, 0, 0, 0);
+
+ /* last but not least, the main tree */
+ add_object_entry(commit->tree->object.sha1, OBJ_TREE, 0, 0, 0);
+
+ strbuf_release(&ost);
+ strbuf_release(&os);
+
+ return i / 21 + 1;
+}
+
static void init_revcache_directory(void)
{
struct stat fi;
@@ -900,6 +1094,10 @@ int make_cache_slice(struct rev_cache_info *rci,
add_object_entry(0, 0, &object, &merge_paths, &split_paths);
object_nr++;
+ /* add all unique children for this commit */
+ if (rci->objects && !object.is_end)
+ object_nr += add_unique_objects(commit);
+
/* print every ~1MB or so */
if (buffer.len > 1000000) {
write_in_full(fd, buffer.buf, buffer.len);
diff --git a/t/t6015-rev-cache-list.sh b/t/t6015-rev-cache-list.sh
index e7474fd..afa0303 100755
--- a/t/t6015-rev-cache-list.sh
+++ b/t/t6015-rev-cache-list.sh
@@ -79,6 +79,7 @@ test_expect_success 'init repo' '
git-rev-list HEAD --not HEAD~3 >proper_commit_list_limited
git-rev-list HEAD >proper_commit_list
+git-rev-list HEAD --objects >proper_object_list
test_expect_success 'make cache slice' '
git-rev-cache add HEAD 2>output.err &&
@@ -101,4 +102,11 @@ test_expect_success 'test rev-caches walker directly (unlimited)' '
test_cmp_sorted list proper_commit_list
'
+#do the same for objects
+test_expect_success 'test rev-caches walker with objects' '
+ git-rev-cache walk --objects HEAD >list &&
+ test_cmp_sorted list proper_object_list
+'
+
test_done
+
--
tg: (212c9b6..) t/revcache/objects (depends on: t/revcache/basic)
^ permalink raw reply related
* [PATCH 1/6 (v4)] man page and technical discussion for rev-cache
From: Nick Edelen @ 2009-08-17 12:31 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
Before any code is introduced the full documentation is put forth. This
provides a man page for the porcelain, and a technical doc in technical/. The
latter describes the API, and discusses rev-cache's design, file format and
mechanics.
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
Documentation/git-rev-cache.txt | 144 ++++++++
Documentation/technical/rev-cache.txt | 614 +++++++++++++++++++++++++++++++++
2 files changed, 758 insertions(+), 0 deletions(-)
diff --git a/Documentation/git-rev-cache.txt b/Documentation/git-rev-cache.txt
new file mode 100644
index 0000000..3479499
--- /dev/null
+++ b/Documentation/git-rev-cache.txt
@@ -0,0 +1,144 @@
+git-rev-cache(1)
+================
+
+NAME
+----
+git-rev-cache - Add, walk and maintain revision cache slices
+
+SYNOPSIS
+--------
+'git-rev-cache' COMMAND [options] [<commit>...]
+
+DESCRIPTION
+-----------
+The revision cache ('rev-cache') provides a mechanism for significantly
+speeding up revision traversals. It does this by creating an efficient
+database (cache) of commits, their related objects and topological relations.
+Independant of packs and the object store, this database is composed of
+rev-cache "slices" -- each a different file storing a given segment of commit
+history. To map commits to their respective slices, a single index file is
+kept for the rev-cache.
+
+'git-rev-cache' provides a front-end for the rev-cache mechanism, intended for
+updating and maintaining rev-cache slices in the current repository. New cache
+slice files can be 'add'ed, to keep the cache up-to-date; individual slices can
+be traversed; smaller slices can be 'fuse'd into a larger slice; and the
+rev-cache index can be regenerated.
+
+COMMANDS
+--------
+
+add
+~~~
+Add revisions to the cache by creating a new cache slice. Reads a revision
+list from the command line, formatted as: `START START ... \--not END END ...`
+
+Options:
+
+\--all::
+ Include all refs in the new cache slice, like the \--all option in
+ 'rev-list'.
+
+\--fresh::
+ Exclude everything already in the revision cache, analogous to
+ \--incremental in 'pack-objects'.
+
+\--stdin::
+ Read newline-seperated revisions from the standard input. Use \--not
+ to exclude commits, as on the command line.
+
+\--legs::
+ Ensure newly-generated cache slice has no partial ends. This means that
+ no commit has partially cached parents, in that all its parents are
+ cached or none of them are.
++
+\--legs will cause 'rev-cache' to expand potential slice end-points (creating
+"legs") until this condition is met, simplifying the cache slice structure.
+'rev-cache' itself does not care if a slice has legs or not, but the condition
+may reduce the required complexity of other applications that might use the
+revision cache.
+
+\--no-objects::
+ Non-commit objects are normally included along with the commit with
+ which they were introduced. This is obviously very benificial, but can
+ take longer in cache slice generation. Using this option will disable
+ non-commit object caching.
++
+\--no-objects is mainly intended for debugging or development purposes, but may
+find use in special situations (e.g. common traversal of only commits).
+
+walk
+~~~~
+Analogous to a slice-oriented 'rev-list', 'walk' will traverse a region in a
+particular cache slice. Interesting and uninteresting (delimited, as with
+'rev-list', with \--not) are specified on the command line, and output is the
+same as vanilla 'rev-list'.
+
+Options:
+
+\--objects::
+ Like 'rev-list', 'walk' will normally only list commits. Use this
+ option to list non-commit objects as well, if they are present in the
+ cache slice.
+
+fuse
+~~~~
+Merge several cache slices into a single large slice, like 'repack' for
+'rev-cache'. On each invocation of 'add' a new file ("slice") is added to the
+revision cache directory, and after several additions the directory may become
+populated with many, relatively small slices. Numerous smaller slices will
+yield poorer performance than a one or two large ones, because of the overhead
+of loading new slices into memory.
+
+Running 'fuse' every once in a while will solve this problem by coalescing all
+the cache slices into one larger slice. For very large projects, using
+\--ignore-size is advisable to prevent overly large cache slices. Setting git
+'config' option 'gc.revcache' to 1 will enable cache slice fusion upon garbage
+collection.
+
+Note that 'fuse' uses the internal revision walker, so the options used in
+fusion override those of the cache slices upon which it operates. For example,
+if some slices were generated with \--no-objects, yet 'fuse' was performed with
+non-commit objects, the resulting slice would still contain objects but would
+take longer to generate.
+
+Options:
+
+\--all::
+ Normally fuse will only include everything that's already in the
+ revision cache. \--add tells it to start walking from the branch
+ heads, effectively an `add --all --fresh; fuse` (pseudo-command).
+
+\--no-objects::
+ As in 'add', this option disables inclusion of non-commit objects. If
+ some cache slices do contain such objects, the information will be lost.
+
+\--ignore-size[=N]::
+ Do not merge cache slices of size >=N (be aware that slices must be
+ mapped to memory). N can have a suffix of "k" or "m", denoting N as
+ kilobytes and megabytes, respectively. If N is not provided 'fuse'
+ will default to a size of ~25MB.
+
+index
+~~~~~
+Regenerate the revision cache index. If the rev-cache index file associating
+objects with cache slices gets corrupted, lost, or otherwise becomes unusable,
+'index' will quickly regenerate the file. It's most likely that this won't be
+needed in every day use, as it is targeted towards debugging and development.
+
+alt
+~~~
+Create a cache slice pointer to another slice, identified by its full path:
+`fuse path/to/other/slice`
+
+This command is useful if you have several repositories sharing a common
+history. Although space requirements for rev-cache are slim anyway, you can in
+this situation reduce it further by using slice pointers, pointing to relavant
+slices in other repositories. Note that only one level of redirection is
+allowed, and the slice pointer will break if the original slice is removed.
+'fuse' will not touch slice pointers.
+
+DISCUSSION
+----------
+For an explanation of the API and its inner workings, see
+link:technical/rev-cache.txt[technical info on rev-cache].
diff --git a/Documentation/technical/rev-cache.txt b/Documentation/technical/rev-cache.txt
new file mode 100644
index 0000000..6e7c7f6
--- /dev/null
+++ b/Documentation/technical/rev-cache.txt
@@ -0,0 +1,614 @@
+rev-cache
+=========
+
+The revision cache API ('rev-cache') provides a method for efficiently storing
+and accessing commit branch sections. Such branch slices are defined by a
+series of start/top (interesting) and end/bottom (uninteresting) commits. Each
+slice contains information on commits in topological order. Recorded with each
+commit is:
+
+* All intra-slice topological relations, encoded into path "channels" (see
+ 'Mechanics' for full explanation).
+* Object meta-data: type, SHA-1, size, date (for commits).
+* Objects introduced by that commit, not present in the its cached parents.
+
+In addition to the API, basic structures are exported for the possibility of
+direct access.
+
+The API
+-------
+You can find the function prototypes in `revision.h`.
+
+Data Structures
+~~~~~~~~~~~~~~~
+The `rev_cache_info` struct holds all the options and flags for the API.
+
+----
+struct rev_cache_info {
+ /* generation flags */
+ unsigned objects : 1,
+ legs : 1,
+ make_index : 1,
+ fuse_me : 1;
+
+ /* index inclusion */
+ unsigned overwrite_all : 1;
+
+ /* traversal flags */
+ unsigned add_to_pending : 1;
+
+ /* fuse options */
+ unsigned int ignore_size;
+
+ /* reserved */
+ struct rev_cache_slice_map *maps,
+ *last_map;
+};
+----
+
+The fields:
+
+`objects`::
+ Add non-commit objects to slice.
+
+`legs`::
+ Ensure end/bottom commits have no children.
+
+`make_index`::
+ Integrate newly-made slice into index.
+
+`fuse_me`::
+ This is specified if a fuse is occuring, and slices are to be reused.
+ This option requires `maps` and `last_maps` to be initialized.
+
+`overwrite_all`::
+ When a cache slice is added to the index, sometimes overlap occures
+ between it and other slices. Normally, original index entries are kept
+ unless the new entry represents a start commit (older entries are more
+ likely to lead to greater in-slice traversals). This options overrides
+ that, and updates all entries of the new slice.
+
+`add_to_pending`::
+ Append unique non-commit objects to the `pending` object list in the
+ passed `rev_info` instance.
+
+`add_names`::
+ Include non-commit object names in the pending object entries if
+ `add_to_pending` is set.
+
+`ignore_size`::
+ If non-zero, ignore slices with size greater or equal to this during
+fusion.
+
+`maps`/`last_map`::
+ An array of slice mappings, indexed by their id in the slice index
+ header, to be re-used with `fuse_me`. `last_map` points to the last
+ mapping used, and should be initialized to 0.
+
+Functions
+~~~~~~~~~
+
+init_rev_cache
+^^^^^^^^^^^^^^
+----
+void init_rev_cache_info(
+ struct rev_cache_info *rci OUT
+)
+----
+
+Initialize `rci` to default options.
+
+make_cache_slice
+^^^^^^^^^^^^^^^^
+----
+int make_cache_slice(
+ struct rev_cache_info *rci IN,
+ struct rev_info *revs IN,
+ struct commit_list **starts IN/OUT,
+ struct commit_list **ends IN/OUT,
+ unsigned char *cache_sha1 OUT
+)
+----
+
+Create a cache slice based on either `revs` (if non-NULL) *or* the `starts` and
+`ends` lists. The actual list of start and end commits of the slice may be
+different from the parameters, based on what defines the branch segment, and
+this actual list is passed back through `starts` and `ends`.
+
+The cache slice is identified via a SHA-1 generated from the actual start/end
+commit lists. `cache_sha1`, if non-NULL, can recieve the cache slice name.
+`rci` is used to specify generation options, but can be NULL if you want
+`make_cache_slice` to fall back on defaults. Returns 0 on success, non-zero on
+failure.
+
+make_cache_index
+^^^^^^^^^^^^^^^^
+----
+int make_cache_index(
+ struct rev_cache_info *rci IN,
+ unsigned char *cache_sha1 IN,
+ int fd IN,
+ unsigned int size IN
+)
+----
+
+Add a slice to the rev-cache index. `cache_sha1` is the identity hash of the
+cache slice; `fd` is a file descriptor of the cache slice opened with
+read/write privileges (the slice is not actually modified); `size` is the size
+of the cache slice. Although there are currently no options for index
+updating, `rci` is a placeholder in case of future options. Note that this
+function is normally called by `make_cache_slice`. Returns 0 on success,
+non-zero on failure.
+
+open_cache_slice
+^^^^^^^^^^^^^^^^
+----
+int open_cache_slice(
+ unsigned char *sha1 IN,
+ int flags IN
+)
+----
+
+Returns a file descriptor to a cache slice described by `sha1` hash, using
+`flags` as the access mode. This will follow cache slice pointers to one level
+of indirection.
+
+get_cache_slice
+^^^^^^^^^^^^^^^
+----
+unsigned char *get_cache_slice(
+ struct commit *commit IN
+)
+----
+
+Given a commit object `get_cache_slice` will search the revision cache index
+and return, if found, the cache slice SHA-1.
+
+traverse_cache_slice
+^^^^^^^^^^^^^^^^^^^^
+----
+int traverse_cache_slice(
+ struct rev_info *revs IN/OUT,
+ unsigned char *cache_sha1 IN,
+ struct commit *commit IN,
+ unsigned long *date_so_far IN/OUT,
+ int *slop_so_far IN/OUT,
+ struct commit_list ***queue OUT,
+ struct commit_list **work IN/OUT
+)
+----
+
+Traverse a specified cache slice. An explanation of the each field:
+
+`revs`::
+ The revision walk instance. `traverse_cache_slice` uses this for
+ general options (e.g. which objects are included) and slice traversal
+ options (in the `rev_cache_info` field). If the `add_to_pending`
+ option is specified, non-commit objects are appended to the `pending`
+ object list field.
+
+`cache_sha1`::
+ SHA-1 identifying the cache slice to use. This can be taken directly
+ from `get_cache_slice`.
+
+`commit`::
+ The current commit object in the revision walk, i.e. the commit which
+ inspired this slice traversal. Although theoretically redundant in
+ view of the `work` list, this simplifies interaction with normal
+ revision walks, which pop commits from `work` before analyzing them.
+
+`date_so_far`::
+ The date of the oldest encountered interesting commit. Passing NULL
+ will let `traverse_cache_slice` use defaults.
+
+`slop_so_far`::
+ The `slop` value, a la revision.c. This is a counter used to determine
+ when to stop traversing, based on how many extra uninteresting commits
+ should be encountered. NULL will enable defaults, as above.
+
+`queue`::
+ Refers to a pointer to the head of a FIFO commit list, recieving the
+ commits we've seen and added.
+
+`work`::
+ A date-ordered list of commits that have yet to be processed (i.e. seen
+ but not added). Commits from here present in the slice are removed
+ (and, obviously, used as starting places for traversal), and any end
+ commits encountered are inserted.
+
+starts_from_slices
+^^^^^^^^^^^^^^^^^^
+----
+void starts_from_slices(
+ struct rev_info *revs OUT,
+ unsigned int flags IN,
+ unsigned char *which IN,
+ int n IN
+)
+----
+
+Will mark start-commits in certain rev-cache slices with `flag`, and added them
+to the pending list of `revs`. If `n` is zero, `starts_from_slices` will use
+all slices. Otherwise `which` will specify an *unseperated* list of cache
+SHA-1s to use (20 bytes each), and `n` will contain the number of slices (i.e.
+20 * `n` = size of `which`).
+
+fuse_cache_slices
+^^^^^^^^^^^^^^^^^
+----
+int fuse_cache_slices(
+ struct rev_cache_info *rci IN,
+ struct rev_info *revs IN
+)
+----
+
+Generate a slice based on `revs`, replacing all encountered slices with one
+(larger) slice. The `ignore_size` field in `rci`, if non-zero, will dictate
+which cache slice sizes to ignore in both traversal and replacement.
+
+regenerate_cache_index
+^^^^^^^^^^^^^^^^^^^^^^
+----
+int regenerate_cache_index(
+ struct rev_cache_info *rci IN
+)
+----
+
+Remake the revision cache index, including all the slices. Currently no
+options in `rci` exist for index (re)generation, but some may develop in the
+future.
+
+to/from_disked_rc_object/index_entry
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+----
+struct rc_object/index_entry *from_disked_rc_object/index_entry(
+ struct rc_object/index_entry_ondisk *src IN,
+ struct rc_object/index_entry *dst OUT
+)
+
+struct rc_object/index_entry_ondisk *to_disked_rc_object/index_entry(
+ struct rc_object/index_entry *src IN,
+ struct rc_object/index_entry_ondisk *dst OUT
+)
+----
+
+Functions to convert between the internal and storage (`_ondisk`) versions of
+object and index entry structures. These are necessary for direct access to
+the cache slices. If NULL is provided for `dst` a statically allocated
+structure is used, and a pointer to the struct is returned. Otherwise the
+functions return `dst`.
+
+Example Usage
+-------------
+
+A few examples to demonstrate usage:
+
+.Creating a slice
+----
+/* pretend you're a porcelain for rev-cache reading from the command line */
+struct rev_info revs;
+struct rev_cache_info rci;
+
+init_revisions(&revs, 0);
+init_rci(&rci);
+
+flags = 0;
+for (i = 1; i < argc; i++) {
+ if (!strcmp(argv[i], "--not"))
+ flags ^= UNINTERESTING;
+ else if(!strcmp(argv[i], "--fresh"))
+ starts_from_slices(&revs, UNINTERESTING, 0, 0);
+ else
+ handle_revision_arg(argv[i], &revs, flags, 1);
+}
+
+/* we want to explicitly set certain options */
+rci.objects = 0;
+
+if (!make_cache_slice(&rci, &revs, 0, 0, cache_sha1))
+ printf("made slice! it's called %s\n", sha1_to_hex(cache_sha1));
+----
+
+.Traversing a slice
+----
+/* let's say you're walking the tree with a 'work' list of current heads and a
+ * FILO output list 'out' */
+out = 0;
+outp = &out;
+
+while (work) {
+ struct commit *commit = pop_commit(&work);
+ struct object *object = &commit->object;
+ unsigned char *cache_sha1;
+
+ if (cache_sha1 = get_cache_slice(object->sha1)) {
+ /* note that this will instatiate any topo-relations
+ * as it goes */
+ if (traverse_cache_slice(&revs, cache_sha1,
+ commit, 0, 0, /* use defaults */
+ &outp, &work) < 0)
+ die("I'm overreacting to a non-fatal cache error");
+ } else {
+ struct commit_list *parents = commit->parents;
+
+ while (parents) {
+ struct commit *p = parents->item;
+ struct object *po = &p->object;
+
+ parents = parents->next;
+ if (po->flags & UNINTERESTING)
+ continue;
+
+ if (object->flags & UNINTERESTING)
+ po->flags |= UNINTERESTING;
+ else if (po->flags & SEEN)
+ continue;
+
+ if (!po->parsed)
+ parse_commit(p);
+ insert_by_date(p, &work);
+ }
+
+ if (object->flags & (SEEN | UNINTERESTING) == 0)
+ outp = &commit_list_insert(commit, outp)->next;
+ object->flags |= SEEN;
+ }
+}
+----
+
+Some Internals
+--------------
+For more advanced usage, the slice and index file(s) may be accessed directly.
+Relavant structures are availabe in `rev-cache.h`.
+
+File Formats
+~~~~~~~~~~~~
+
+Cache Slices
+^^^^^^^^^^^^
+A slice has a basic fixed-size header, followed by a certain number of object
+entries, then a NULL-seperated list of object names. Commits are sorted in
+topo-order, and each commit entry is followed by the objects added in that
+commit.
+
+----
+ -- +--------------------------------+
+header | object number, etc... |
+ -- +--------------------------------+
+commit | commit info |
+entry | path data |
+ +--------------------------------+
+ | tree/blob info |
+ +--------------------------------+
+ | tree/blob info |
+ +--------------------------------+
+ | ... |
+ -- +--------------------------------+
+commit | commit info |
+entry | path data |
+ +--------------------------------+
+ | tree/blob info |
+ +--------------------------------+
+ | ... |
+ -- +--------------------------------+
+... ...
+ -- +--------------------------------+
+name list | \0some_file_name\0 |
+(note +--------------------------------+
+preceeding | another_file\0 |
+null) ... |
+ +--------------------------------+
+----
+
+Here is the header:
+
+----
+struct rc_cache_slice_header {
+ char signature[8]; /* REVCACHE */
+ unsigned char version;
+ uint32_t ofs_objects;
+
+ uint32_t object_nr;
+ uint16_t path_nr;
+ uint32_t size;
+
+ unsigned char sha1[20];
+
+ uint32_t names_size;
+};
+----
+
+Explanations:
+
+`signature`::
+ The identifying signature of cache slice file. Always "REVCACHE".
+`version`::
+ The version number, currently 1.
+`ofs_objects`::
+ The byte offset at which the commit/object listing starts. Always
+ present at the 10th byte, regardless of file version.
+`object_nr`::
+ The total number of objects (commit + non-commit objects) present in
+ the slice.
+`path_nr`::
+ The total number of paths/channels used in encoding the topological
+ data. Note that paths are reused (see 'Mechanics'), so there will
+ never be more than a few hundred paths (if that) used.
+`size`::
+ The size of the slice *excluding* the name list. In other words, the
+ size of the portion mapped to memory.
+`sha1`::
+ The cache slice SHA-1.
+`names_size`::
+ The size of the name list. `size` + `names_size` = size of slice
+
+Revision Cache Index
+^^^^^^^^^^^^^^^^^^^^
+The index is a single file that associates SHA-1s with cache slices and file
+positions. It is somewhat similar to pack-file indexes, containing a fanout
+table and a list of index entries sorted by hash.
+
+----
+ -- +--------------------------------+
+header | object #, cache #, etc. |
+ -- +--------------------------------+
+sha1s of | SHA-1 |
+slices | ... |
+ -- +--------------------------------+
+fanout | fanout[0x00] |
+table ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ | fanout[0xff] |
+ -- +--------------------------------+
+index | SHA-1 of object |
+entries | index of cache slice SHA-1 |
+ | position in cache slice |
+ +--------------------------------+
+ | |
+ ...
+ +--------------------------------+
+----
+
+The header:
+
+----
+struct rc_index_header {
+ char signature[8]; /* REVINDEX */
+ unsigned char version;
+ uint32_t ofs_objects;
+
+ uint32_t object_nr;
+ unsigned char cache_nr;
+
+ uint32_t max_date;
+};
+----
+
+Explanations:
+
+`signature`::
+ Always "REVINDEX".
+`version`::
+ Version number, currently 1.
+`ofs_objects`::
+ Offset at which the entry objects begin. This is more obviously useful
+ in the index because the list of slice SHA-1s is variably-sized.
+`object_nr`::
+ Number of index entry objects present.
+`cache_nr`::
+ Number of cache slices to which the index maps, and hence the number of
+slice SHA-1s listed.
+`max_date`::
+ The oldest commit represented in the index. This is used to help speed
+up lookup times by knowing what range of commits we definitely don't have
+cached. Normal usage of 'rev-cache' would leave no "holes" in its coverage of
+commit history -- once a commit is cached, everything reachable from it should
+be cached as well. Most of the time refs are added to rev-cache simultaneous
+as well. This means that in most situations almost everything <= `max_date`
+will be cached.
+
+Mechanics
+~~~~~~~~~
+
+The most important part of rev-cache is its method of encoding topological
+relations. To ensure fluid traversal and reconstruction, commits are related
+through high-level "streams"/"channels" rather than individual
+interconnections. Intuitively, rev-cache stores history the way gitk shows it:
+commits strung up on lines, which interconnect at merges and branches.
+
+Each commit is associated to a given channel/path via a 'path id', and
+variable-length fields govern which paths (if any) are closed or opened at that
+object. This means that topo-data can be preserved in only a few bytes extra
+per object entry. Other information stored per entry is the sha-1 hash, type,
+date, size, name, and status in cache slice. Here is format of an object
+entry, both on-disk and in-memory:
+
+----
+struct object_entry {
+ unsigned type : 3;
+ unsigned is_end : 1;
+ unsigned is_start : 1;
+ unsigned uninteresting : 1;
+ unsigned include : 1;
+ unsigned flags : 1;
+ unsigned char sha1[20];
+
+ unsigned char merge_nr;
+ unsigned char split_nr;
+ unsigned size_size : 3;
+ unsigned name_size : 3;
+
+ uint32_t date;
+ uint16_t path;
+
+ /* merge paths */
+ /* split paths */
+ /* size */
+ /* name index */
+};
+----
+
+An explanation of each field:
+
+`type`::
+ Object type
+`is_end`::
+ The commit has some parents outside the cache slice (all if slice has
+ legs)
+`is_start`::
+ The commit has no children in cache slice
+`uninteresting`::
+ Run-time flag, used in traversal
+`include`::
+ Run-time flag, used in traversal (initialization)
+`flags`::
+ Currently unused, extra bit
+`sha1`::
+ Object SHA-1 hash
+
+`merge_nr`::
+ The number of paths the current channel diverges into; the current path
+ ends upon any merge.
+`split_nr`::
+ The number of paths this commit ends; used on both merging and
+ branching.
+`size_size`::
+ Number of bytes the object size takes up.
+`name_size`::
+ Number of bytes the name index takes up.
+
+`date`::
+ The date of the commit.
+`path`::
+ The path ID of the channel with which this commit is associated.
+
+merge paths::
+ The path IDs (16-bit) that are to be created. Overflow is not a
+ problem as path IDs are reused, leaving even complicated projects to
+ consume no more than a few hundred IDs.
+split paths::
+ The path IDs (16-bit) that are to be ended.
+size::
+ The size split into the minimum number of bytes. That is, 1-8 bytes
+ representing the size, least-significant byte first.
+name index::
+ An offset for the null-seperated, object name list at the end of the
+ cache slice. Also split into the minimum number of bytes.
+
+Each path ID refers to an index in a 'path array', which stores the current
+status (eg. active, interestingness) of each channel.
+
+Due to topo-relations and boundary tracking, all of a commit's parents must be
+encountered before the path is reallocated. This is achieved by using a
+counter system per merge: starting at the parent number, the counter is
+decremented as each parent is encountered (dictated by 'split paths'); at 0 the
+path is cleared.
+
+Boundary tracking is necessary because non-commits are stored relative to the
+commit in which they were introduced. If a series of commits is not included
+in the output, the last interesting commit must be parsed manually to ensure
+all objects are accounted for.
+
+To prevent list-objects from recursing into trees that we've already taken care
+of, the flag `FACE_VALUE` is introduced. An object with this flag is not
+explored (= "taken at face value"), significantly reducing I/O and processing
+time.
--
tg: (c3b7310..) t/revcache/docs (depends on: t/revcache/integration)
^ permalink raw reply related
* [PATCH 0/6 (v4)] revision caching mechanism to significantly speed up packing/walking (last version)
From: Nick Edelen @ 2009-08-17 12:31 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
SUGGESTED FOR 'PU':
Traversing objects is currently very costly, as every commit and tree must be
loaded and parsed. Much time and energy could be saved by caching metadata and
topological info in an efficient, easily accessible manner. Furthermore, this
could improve git's interfacing potential, by providing a condensed summary of
a repository's commit tree.
This is a series to implement such a revision caching mechanism, aptly named
rev-cache. The series will provide:
- a core API to manipulate and traverse caches
- an integration into the internal revision walker
- a porcelain front-end providing access to users and (shell) applications
- a series of tests to verify/demonstrate correctness
- documentation of the API, porcelain and core concepts
In cold starts rev-cache has sped up packing and walking by a factor of 4, and
over twice that on warm starts. Some times on slax for the linux repository:
rev-list --all --objects >/dev/null
default
cold 1:13
warm 0:43
rev-cache'd
cold 0:19
warm 0:02
pack-objects --revs --all --stdout >/dev/null
default
cold 2:44
warm 1:21
rev-cache'd
cold 0:44
warm 0:10
The mechanism is minimally intrusive: most of the changes take place in
seperate files, and only a handful of git's existing functions are modified.
Hope you find this useful.
- Nick
---
BOMBS AWAY!!!
To clarify my magnificent muddying of this final update I'm flash flooding y'all with a mint patchset! This probably isn't deserving of it's own version, but it's the last one and I've confuddled everyone enough already.
SO, what's changed from the last one?
- fixed whitespace errors
- cleaned up header definitions
- implemented _ondisk versions of structs for ensured portability
- fixed typo
Sorry again for the size of the patches, but like I said earlier they're not so much patches as files -- the changes in existing git files are actually pretty small. Since most everything in the patches is intertwined I couldn't split it up a bunch more without having non-functioning patches.
I promise this is the last one :-P
Documentation/git-rev-cache.txt | 144 ++
Documentation/technical/rev-cache.txt | 614 +++++++++
Makefile | 2 +
builtin-gc.c | 9 +
builtin-rev-cache.c | 322 +++++
builtin.h | 1 +
commit.c | 2 +
git.c | 1 +
list-objects.c | 49 +-
rev-cache.c | 2314 +++++++++++++++++++++++++++++++++
rev-cache.h | 124 ++
revision.c | 90 +-
revision.h | 44 +-
t/t6015-rev-cache-list.sh | 251 ++++
14 files changed, 3942 insertions(+), 25 deletions(-)
create mode 100644 Documentation/git-rev-cache.txt
create mode 100644 Documentation/technical/rev-cache.txt
create mode 100644 builtin-rev-cache.c
create mode 100644 rev-cache.c
create mode 100644 rev-cache.h
create mode 100755 t/t6015-rev-cache-list.sh
^ permalink raw reply
* Re: sparse support in pu
From: Johannes Schindelin @ 2009-08-17 12:29 UTC (permalink / raw)
To: Nguyen Thai Ngoc Duy; +Cc: Johannes Sixt, skillzero, git
In-Reply-To: <fcaeb9bf0908170441o30005085nb0d4e08f333b6146@mail.gmail.com>
Hi,
On Mon, 17 Aug 2009, Nguyen Thai Ngoc Duy wrote:
> On Mon, Aug 17, 2009 at 5:36 PM, Johannes Sixt<j.sixt@viscovery.net> wrote:
>
> > In order to advocate my earlier proposal: Name the file
> > .git/info/phantoms, then it's clear: "The files mentioned here are
> > phantoms" - they exist in the index, but not in the worktree; no
> > phantoms means that everything is checked out.
>
> OK. Phantom checkout, must be unique in VCS world ;-) If no one objects
> my next series will use this name as it's better than "sparse" and
> "assume-unchanged" is just too vague. Would option names to
> enable/disable this be --with[out]-phantoms?
The term 'phantom' is not specified at all. At least interested people on
the mailing list know 'sparse'. But I agree that the naming is a major
problem, hence my earlier (unanswered) call.
However, I would find specifying what you do _not_ want in that file
rather unintuitive, in the same leage as receive.denyNonFastForwards = no.
If I want to have a sparse checkout, I know which files I _want_.
Ciao,
Dscho
^ permalink raw reply
* Re: sparse support in pu
From: Nguyen Thai Ngoc Duy @ 2009-08-17 12:19 UTC (permalink / raw)
To: Johannes Sixt; +Cc: skillzero, git
In-Reply-To: <4A894676.1090803@viscovery.net>
On Mon, Aug 17, 2009 at 7:00 PM, Johannes Sixt<j.sixt@viscovery.net> wrote:
> Nguyen Thai Ngoc Duy schrieb:
>> On Mon, Aug 17, 2009 at 5:36 PM, Johannes Sixt<j.sixt@viscovery.net> wrote:
>>> In order to advocate my earlier proposal: Name the file
>>> .git/info/phantoms, then it's clear: "The files mentioned here are
>>> phantoms" - they exist in the index, but not in the worktree; no phantoms
>>> means that everything is checked out.
>>
>> OK. Phantom checkout, must be unique in VCS world ;-) If no one
>> objects my next series will use this name as it's better than "sparse"
>> and "assume-unchanged" is just too vague. Would option names to
>> enable/disable this be --with[out]-phantoms?
>
> --phantoms vs. --no-phantoms?
Yeah good too. I have probably dealed with autotools too much lately.
--
Duy
^ permalink raw reply
* Re: sparse support in pu
From: Johannes Sixt @ 2009-08-17 12:00 UTC (permalink / raw)
To: Nguyen Thai Ngoc Duy; +Cc: skillzero, git
In-Reply-To: <fcaeb9bf0908170441o30005085nb0d4e08f333b6146@mail.gmail.com>
Nguyen Thai Ngoc Duy schrieb:
> On Mon, Aug 17, 2009 at 5:36 PM, Johannes Sixt<j.sixt@viscovery.net> wrote:
>> In order to advocate my earlier proposal: Name the file
>> .git/info/phantoms, then it's clear: "The files mentioned here are
>> phantoms" - they exist in the index, but not in the worktree; no phantoms
>> means that everything is checked out.
>
> OK. Phantom checkout, must be unique in VCS world ;-) If no one
> objects my next series will use this name as it's better than "sparse"
> and "assume-unchanged" is just too vague. Would option names to
> enable/disable this be --with[out]-phantoms?
--phantoms vs. --no-phantoms?
-- Hannes
^ permalink raw reply
* Re: sparse support in pu
From: Nguyen Thai Ngoc Duy @ 2009-08-17 11:43 UTC (permalink / raw)
To: Johannes Schindelin; +Cc: skillzero, git
In-Reply-To: <alpine.DEB.1.00.0908171113420.4991@intel-tinevez-2-302>
On Mon, Aug 17, 2009 at 4:15 PM, Johannes
Schindelin<Johannes.Schindelin@gmx.de> wrote:
>> >> 3. One thing that was confusing is that I needed a trailing slash on
>> >> directories in .git/info/sparse to get them excluded. This seems
>> >> different than .gitignore, which works for me without the trailing
>> >> slash.
>> >
>> > Hmm.. probably because Git feeds directories to .gitignore handling
>> > functions. There is not much I can do, index does not have
>> > directories. I don't know if it's worth generating "directories" from
>> > index.
>>
>> Maybe just add a note in the documentation? If there's a default
>> .git/info/sparse file then it might be good place to put a note as well.
>
> I rather think that this should be fixed. Maybe you can come up with a
> patch to the tests which shows this behavior (with test_expect_failure)?
> Then it will be much easier to come up with a fix.
Agreed. Thanks.
--
Duy
^ permalink raw reply
* Continue git clone after interruption
From: Tomasz Kontusz @ 2009-08-17 11:42 UTC (permalink / raw)
To: git
Hi,
is anybody working on making it possible to continue git clone after
interruption? It would be quite useful for people with bad internet
connection (I was downloading a big repo lately, and it was a bit
frustrating to start it over every time git stopped at ~90%).
Tomasz Kontusz
^ permalink raw reply
* Re: sparse support in pu
From: Nguyen Thai Ngoc Duy @ 2009-08-17 11:41 UTC (permalink / raw)
To: Johannes Sixt; +Cc: skillzero, git
In-Reply-To: <4A8932BB.7030002@viscovery.net>
On Mon, Aug 17, 2009 at 5:36 PM, Johannes Sixt<j.sixt@viscovery.net> wrote:
> Nguyen Thai Ngoc Duy schrieb:
>> [I haven't read the rest of the mail, will read it through later]
>>
>> On Mon, Aug 17, 2009 at 3:49 PM, <skillzero@gmail.com> wrote:
>>> It would be nice if .git/info/sparse is there by default (like
>>> .git/info/exclude) with some commented out instructions (also like
>>> .git/info/exclude).
>>
>> No it can't be there by default. An empty .git/info/sparse means clear
>> assume-unchanged bit out of all files in index. It's not the same as
>> lacking .git/info/sparse (which does disable sparse checkout feature).
>
> Huh? Shouldn't the meaning of .git/info/sparse be: "the files mentioned in
> this file are not checked out." That is, if the file is empty, then no
> file is not checked out, IOW, all files are checked out.
That's correct. Empty .git/info/sparse -> no assume-unchanged files ->
all files are checked out. That's what I meant. It's different from
empty .git/info/sparse as "don't touch index and worktree, leave them
as they are no matther if they have assume-unchanged files".
> In order to advocate my earlier proposal: Name the file
> .git/info/phantoms, then it's clear: "The files mentioned here are
> phantoms" - they exist in the index, but not in the worktree; no phantoms
> means that everything is checked out.
OK. Phantom checkout, must be unique in VCS world ;-) If no one
objects my next series will use this name as it's better than "sparse"
and "assume-unchanged" is just too vague. Would option names to
enable/disable this be --with[out]-phantoms?
--
Duy
^ permalink raw reply
* Re: Linus' sha1 is much faster!
From: Giuseppe Scrivano @ 2009-08-17 10:51 UTC (permalink / raw)
To: Pádraig Brady; +Cc: Bug-coreutils, Linus Torvalds, Git Mailing List
In-Reply-To: <4A88B80D.40804@draigBrady.com>
Pádraig Brady <P@draigBrady.com> writes:
> -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions
> -fstack-protector --param=ssp-buffer-size=4 -m32 -march=i586
> -mtune=generic -fasynchronous-unwind-tables -D_GNU_SOURCE=1
thanks. I did again all tests on my machine using these same options.
I repeated each test 6 times and I took the median without consider the
first result. Except the first run that it is not considered, I didn't
report a big variance on results of the same test.
gcc 4.3.3
gnulib sha1: real 0m2.543s
gnulib sha1 lookup: real 0m1.906s (-25%)
linus's sha1: real 0m2.468s (-3%)
linus's sha1 no asm: real 0m2.289s (-9%)
gcc 4.4.1
gnulib sha1: real 0m3.386s
gnulib sha1 lookup: real 0m3.110s (-8%)
linus's sha1: real 0m1.701s (-49%)
linus's sha1 no asm: real 0m1.284s (-62%)
I don't see such big differences in asm generated by gcc 4.4.1 and gcc
4.3.3 to explain this performance difference, what I noticed immediately
is that in the gcc-4.4 generated asm there are more "lea" instructions
(+30%), but I doubt this is the reason of these poor results. Anyway, I
haven't yet looked much in details.
Cheers,
Giuseppe
^ permalink raw reply
* Re: Git User's Survey 2009 partial summary, part 2 - from first 10
From: Johan Herland @ 2009-08-17 10:44 UTC (permalink / raw)
To: git; +Cc: Jakub Narebski, msysgit
In-Reply-To: <200908171224.44686.jnareb@gmail.com>
On Monday 17 August 2009, Jakub Narebski wrote:
> 9) On which operating system(s) do you use Git?
> (Choice - Multiple answers)
>
> On Unix-based operating system you can get the name of operation
> system by running 'uname'.
I find it interesting to compare the answers to this question against
previous years' surveys:
=============================================================
Operating system | 2009 [%] | 2008 [%] | 2007 [%]
-------------------------------------------------------------
Linux | 88% | 86% | 90%
MacOS X (Darwin) | 44% | 47% | 15%
MS Windows/Cygwin | 9% | 10% | >3%?
MS Windows/msysGit (MINGW) | 21% | 16% | >0%?
............................|..........|..........|..........
MS Windows (any) | 27% | <26%?| 9%
Although several of these numbers are probably inaccurate, and
assuming that the responses on this question is representative,
we see a stabilizing trend in Git's user base, at least when
looking at Linux vs. MacOS. Windows seems to be growing steadily,
and it's good to see that the msysGit portion is outgrowing the
Cygwin portion.
Thanks to Dscho and all the others that make msysGit better
every day!
Have fun! :)
...Johan
--
Johan Herland, <johan@herland.net>
www.herland.net
^ permalink raw reply
* Re: sparse support in pu
From: Johannes Sixt @ 2009-08-17 10:36 UTC (permalink / raw)
To: Nguyen Thai Ngoc Duy; +Cc: skillzero, git
In-Reply-To: <fcaeb9bf0908170321o43fa4a6bv95dd78ff7889686f@mail.gmail.com>
Nguyen Thai Ngoc Duy schrieb:
> [I haven't read the rest of the mail, will read it through later]
>
> On Mon, Aug 17, 2009 at 3:49 PM, <skillzero@gmail.com> wrote:
>> It would be nice if .git/info/sparse is there by default (like
>> .git/info/exclude) with some commented out instructions (also like
>> .git/info/exclude).
>
> No it can't be there by default. An empty .git/info/sparse means clear
> assume-unchanged bit out of all files in index. It's not the same as
> lacking .git/info/sparse (which does disable sparse checkout feature).
Huh? Shouldn't the meaning of .git/info/sparse be: "the files mentioned in
this file are not checked out." That is, if the file is empty, then no
file is not checked out, IOW, all files are checked out.
In order to advocate my earlier proposal: Name the file
.git/info/phantoms, then it's clear: "The files mentioned here are
phantoms" - they exist in the index, but not in the worktree; no phantoms
means that everything is checked out.
-- Hannes
^ permalink raw reply
* Re: sparse support in pu
From: Nguyen Thai Ngoc Duy @ 2009-08-17 10:21 UTC (permalink / raw)
To: skillzero; +Cc: git
In-Reply-To: <2729632a0908170149o425544dcw52aeb6ac6ee1437d@mail.gmail.com>
[I haven't read the rest of the mail, will read it through later]
On Mon, Aug 17, 2009 at 3:49 PM, <skillzero@gmail.com> wrote:
> It would be nice if .git/info/sparse is there by default (like
> .git/info/exclude) with some commented out instructions (also like
> .git/info/exclude).
No it can't be there by default. An empty .git/info/sparse means clear
assume-unchanged bit out of all files in index. It's not the same as
lacking .git/info/sparse (which does disable sparse checkout feature).
--
Duy
^ permalink raw reply
* Git User's Survey 2009 partial summary, part 2 - from first 10
From: Jakub Narebski @ 2009-08-17 10:24 UTC (permalink / raw)
To: git
Git User's Survey 2009 partial summary, part 2 - git difficulty,
proficiency, uses, install, OS, editors.
It more than a month since Git User's Suvey 2009 was started (it was
started on 15 July 2009), which is a half of planned duration time of
the survey.
So I think this is time for second part partial summary of Git User's
Survey 2009. The previous part can be found at
http://thread.gmane.org/gmane.comp.version-control.git/124599
Subject: Git User's Survey 2009 partial summary, part 1 -
- announcing survey, participation
Message-Id: <200907312246.12134.jnareb@gmail.com>
You can see summary of Git User's Survey 2009 responses (and make your
own analysis) at the following URL:
http://www.survs.com/shareResults?survey=2PIMZGU0&rndm=678J66QRA2
http://tinyurl.com/GitSurvey2009Analyze
After the survey ends (or earlier, if it is requested) the raw data
would be published on GitSurvey2009 page on Git Wiki in CSV and XLS
formats (like for GitSurvey2008).
We have currently (at the time I have checked this) around 3089
responses, as compared to 3236 individual responses (including 21
responses in 'test' channel) for survey in 2008, 683 individual
responses in 2007, and around 117 responses in 2006.
Below there is analysis of few questions from this survey; note that
results might do not add to 100% because of replies during using live
analysis tool, and the rounding.
3) Have you found Git easy to learn?
4) Have you found Git easy to use?
(Choice - Single answer)
================================================
Answer | to learn [%] | to use [%]
------------------------------------------------
Very easy | 4% | 9%
Easy | 20% | 36%
Reasonably easy | 55% | 45%
Hard | 19% | 8%
Very hard | 2% | 1%
------------------------------------------------
Total respondents | 2942 | 2959
================================================
As you can see the myth that Git is hard to learn was busted ;-) Not.
When analyzing this data you should take into account that somebody
considering or finding Git to hard to learn wouldn't be probably Git
user, and thus wouldn't fill this survey. So because it is
*Git User's* Survey we should consider that results can be skewed
towards lower value (easier).
What's interesting is comparing (percentage) results for questions
3. and 4.; how hard is git to learn versus how hard is to use. It
seems like Git is reasonably easy to learn, and reasonably easy to
easy to use. So it looks like Git just have somewhat steep learning
curve, and the difficulty to learn pays in being more powerful to
use.
6) Rate your own proficiency with Git:
(Choice - Single answer)
You can think of it as 1-5 numerical grade of your proficiency in Git.
================================================
Proficiency | resp [%] | resp [n]
------------------------------------------------
1. novice | 4% | 114
2. casual, needs advice | 17% | 520
3. everyday use | 38% | 1138
4. can offer advice | 34% | 1020
5. know it very well | 6% | 192
--------------------------+---------------------
Total respondents | 2984
Skipped this question | 105
================================================
As you can see most responders know Git enough for everyday use, or
can even offer some Git advice. If we treat possible answers as a
proficiency grade, the average proficiency is around 3.2.
Either Git users don't stay novices long, or survey announcement(s)
didn't reach novice users.
This data could be useful later to be able to check how it differs
git usage for novices and for gurus (what commands they run, what
features they use at different levels of proficiency).
7) I use Git for (check all that apply):
(Choice - Multiple answers)
Note that above choices are neither orthogonal nor exclusive. One
might want to check multiple answers even for a single repository.
============================================
Answer | resp [%] | resp [n]
--------------------------------------------
work projects | 79% | 2366
unpaid projects | 80% | 2391
............................................
work and unpaid | 66% | 1960
only work | 14% | 407
only unpaid | 15% | 432
not answered | 6% | 190
--------------------------------------------
proprietary projects | 37% | 1108
OSS development | 69% | 2054
private code | 77% | 2315
--------------------------------------------
code (programming) | 87% | 2591
web app | 43% | 1274
configuration files | 34% | 1010
personal data | 33% | 972
documents | 32% | 956
static website | 29% | 881
--------------------------------------------
frontend to other SCM | 32% | 971
sharing data or sync | 22% | 645
backup | 24% | 722
backend for web app | 9% | 263
............................................
--------------------------------------------
other | 3% | 84
============================================
Total respondents | 2989
Skipped this question | 104
============================================
The answers in the table above are in slightly different order than in
original, i.e. in the survey. Analysis of correlations was done here
only for work / unpaid part of answer (see table above).
This question joins together (to reduce number of questions in the
survey) few separate issues:
* Whether one use Git for work, or for unpaid projects: this survey
results shows that the amount of paid and unpaid projects are about
the same, and that most of people (most of responders) us Git for
both work (paid) and free (unpaid) projects.
* The question of openess / license: whether one uses Git for
proprietary projects or for FLOSS (Free/Libre/Open Source
Software), or perhaps for private code (not covered by license).
Here OSS development and private code/data dominates over using Git
for proprietary code, with around twice as much answers. This may
be caused by the fact that Git is / feels more suited to opensource
(like) development (proprietary code can use proprietary SCM), but
it might be that announcement didn't make it to people using Git
for proprietary works.
* Whether one uses Git for code (for something it was created for),
or for other things like documents etc. Here code dominates,
having around twice as many replies as the next in turn, 'web app'
(which is also code, but a special case, and not only code).
* next is UGFWIINI part: using Git for what it was not intended.
Well, except perhaps using Git as frontend or fat-client for other
SCM (e.g. git-svn for Subversion). Using Git for other SCM
dominates with 32%, next are using Git for sharing, and for backup
(both with around 23% of responses).
* The "other (please specify)" is not yet analyzed (nor even began
analysis so far).
8) How do/did you obtain Git (install and/or upgrade)?
(Choice - Multiple answers)
Explanation: binary package covers pre-compiled binary (e.g. from rpm
or deb binary packages); source package covers things like deb-src and
SRPMS/*.src.rpm; source script is meant to cover installation in
source-based distributions, like 'emerge' in Gentoo.
Automatic update (apt, yum, etc.) in most cases means binary package
install; unless one uses source-based distribution like Gentoo, CRUX,
or SourceMage, where automatic update means using source package (or
source script).
The option named "preinstalled / sysadmin job" means that either you
didn't need to install git because it was preinstalled (and you didn't
upgrade); or that you have to ask system administrator to have git
installed or upgraded.
Note that this question is multiple choices question because one can
install Git in different ways on different machines or on different
operating systems.
==================================================
Answer | resp [%] | resp [n]
--------------------------------------------------
binary package | 71% | 2118
source package or script | 24% | 704
source tarball | 20% | 588
pull from repository | 18% | 527
preinstalled / sysadmin job | 6% | 185
other - please specify | 4% | 123
----------------------------+---------------------
Total respondents | 2975
Skipped this question | 118
==================================================
Most people (71%) use ready binary packages, which was kind of
expected; that is the easiest way.
9) On which operating system(s) do you use Git?
(Choice - Multiple answers)
On Unix-based operating system you can get the name of operation
system by running 'uname'.
============================================================
Operating system | resp [%] | resp [n]
------------------------------------------------------------
Linux | 88% | 2623
MacOS X (Darwin) | 44% | 1304
MS Windows/Cygwin | 9% | 279
MS Windows/msysGit (MINGW) | 21% | 625
*BSD (FreeBSD, OpenBSD, NetBSD, etc.) | 7% | 213
OpenSolaris | 3% | 101
other Unix | 2% | 60
Other, please specify | 2% | 46
......................................|..........|..........
MS Windows (any) | 27% | 820
--------------------------------------+---------------------
Total respondents | 2991
Respondents who skipped this question | 102
============================================================
Most common used operating system is Linux, next is MacOS X, and then
MS Windows (on MS Window dominates msysGit version). This is quite
understandable, as Git was created on Linux and for Linux, and it
works best there.
The "other" answer include Solaris, AIX, HP-UX (other Unix), GNU/Linux
(Linux), Debian, NixOS (Linux distributions), none (?); iPod, OpenVMS,
QNX, Hurd and "my toaster" (?). And some commentaries and
clarifications.
10) What do you use to edit contents under version control with Git?
What kind of editor, IDE or RAD you use working with Git?
(Choice - Multiple answers)
* "simple text editor" option includes editors such as pico, nano,
joe, Notepad,
* "programmers editor" option includes editors such as Emacs/XEmacs,
Vim, TextMate, SciTE (syntax highlighting, autoindentation,
integration with other programmers tools, etc.)
* "IDE (Integrated Development Environment) and RAD (Rapid Application
Development)" option includes tools such as Eclipse, NetBeans IDE,
IntelliJ IDE, MS Visual Studio, KDevelop, Anjuta, Xcode,
Code::Blocks but also tools such as Quanta+, BlueFish or Screem (for
editing HTML, CSS, PHP etc.), and Kile or LEd for LaTeX.
* "WYSIWYG tools" option includes word processors such as MS Office or
OpenOffice.org, but also tools such as Adobe Acrobat (for PDF) or
GIMP (for images), or WYSIWYG DTP tools such as QuarkXPress,
PageMaker or Scribus, or WYSIWYG HTML editors such as FrontPage,
Dreamweaver or KompoZer.
============================================================
Answer | resp [%] | resp [n]
------------------------------------------------------------
simple text editor | 25% | 752
programmers editor | 88% | 2641
IDE or RAD | 32% | 960
WYSIWYG tool | 5% | 159
other kind - please specify | 3% | 95
--------------------------------------+---------------------
Total respondents | 2985
Respondents who skipped this question | 108
============================================================
Most popular kind of tool is programmer's editor with very strong 88%
lead. Next in turn are IDE or RAD with 32%, followed by simple text
editor, with 25%. The WYSIWYG tools are last, with very small 5% of
replies; also people who use WYSIWYG tools also use other kind of
tools to edit (presumably to edit commit message at least).
>From browsing through "other tool" responses it seem like there should
be included 'web interface' among possible tools (written as: wiki
frontend, GitHub editor, <textarea>). Other responses include Xournal
(note taking application for graphic tablet, using stylus), Tinderbox
(also note taking application), Git GUI tool: GitX (?), custom
git-aware CMS (probably also 'web interface'), scripts and oneliners,
command line and 'On occasion, Unix text manipulation tools'. Also
there is "audio editors" answer (which probably should fall under
"WYSIWYG tool)...
Many answers in 'other tool' falls in one of ready categories; perhaps
they are clarification?
--
Jakub Narebski
Git User's Survey 2009: http://tinyurl.com/GitSurvey2009
^ permalink raw reply
* Re: [PATCH 0/6] "git commit --dry-run" updates
From: Thomas Rast @ 2009-08-17 10:01 UTC (permalink / raw)
To: Junio C Hamano; +Cc: git, Jeff King
In-Reply-To: <1250330803-22171-1-git-send-email-gitster@pobox.com>
Junio C Hamano wrote:
> The third and the fourth ones tentatively introduce "git stat". The
> former is an incomplete implementation that ignores the pathspec, and the
> latter adds pathspec limiting to it. In the final form before it goes to
> 'next', these two should probably be squashed.
If it's not too much work, it would be nice to somehow detect if -u
can have any influence at all. For example,
$ git status bar
# Not currently on any branch.
nothing to commit (use -u to show untracked files)
is a bit misleading if 'bar' is tracked because adding a -u won't make
any difference.
--
Thomas Rast
trast@{inf,student}.ethz.ch
^ permalink raw reply
* Re: sparse support in pu
From: Johannes Schindelin @ 2009-08-17 9:57 UTC (permalink / raw)
To: Johannes Sixt; +Cc: skillzero, Nguyen Thai Ngoc Duy, git
In-Reply-To: <4A89282A.3020907@viscovery.net>
Hi,
On Mon, 17 Aug 2009, Johannes Sixt wrote:
> Johannes Schindelin schrieb:
> > On Mon, 17 Aug 2009, skillzero@gmail.com wrote:
> >> On Mon, Aug 17, 2009 at 1:17 AM, Nguyen Thai Ngoc Duy<pclouds@gmail.com> wrote:
> >>> On Mon, Aug 17, 2009 at 1:09 PM, <skillzero@gmail.com> wrote:
> >>>> 1. Have people decided whether it should be on by default if you have
> >>>> a .git/info/sparse file? I'd definitely like it to be on by
> >>>> default. When I first tried it, I didn't realize I had to use
> >>>> --sparse to git checkout to get it to use the sparse rules. The
> >>>> same goes for a merge I did that happened to have a file in the
> >>>> excluded area (it included it because I didn't use --sparse to git
> >>>> merge).
> >>> I tend to make it enabled by default too. I have made it stricter to
> >>> trigger reading sparse in unpack_trees() -- only do it when
> >>> unpack_opts.update is TRUE. This should make it safer to be enabled by
> >>> default.
> >> Other than it being new and not-widely-tested code, is there any
> >> additional risk to having it enabled by default if there are no sparse
> >> patterns defined?
> >
> > I think that in and of itself is reason enough to turn off the feature
> > when .git/info/sparse is not present.
>
> I might have missed something: Would there be any observable difference
> between whether .git/info/sparse is absent and whether it is empty?
There _should_ not be an _observable_ difference.
> If not, what do you mean by "turn the feature off"?
There is a global variable which triggers the code path that looks through
the sparse patterns. I'd like this variable to be "off" by default.
> >> It would be nice if .git/info/sparse is there by default (like
> >> .git/info/exclude) with some commented out instructions (also like
> >> .git/info/exclude).
> >
> > I'm not a fan of this idea.
>
> For any particular reason?
First, it is an experimental feature. I'd hate it if people who do not
want to use the sparse feature are affected.
Second, even if the sparse pattern list is empty, an additional call to
the function that matches files against the pattern would take additional
runtime.
If I had time, I could think of a third reason.
Ciao,
Dscho
^ permalink raw reply
* Re: sparse support in pu
From: Johannes Sixt @ 2009-08-17 9:51 UTC (permalink / raw)
To: Johannes Schindelin; +Cc: skillzero, Nguyen Thai Ngoc Duy, git
In-Reply-To: <alpine.DEB.1.00.0908171113420.4991@intel-tinevez-2-302>
Johannes Schindelin schrieb:
> On Mon, 17 Aug 2009, skillzero@gmail.com wrote:
>> On Mon, Aug 17, 2009 at 1:17 AM, Nguyen Thai Ngoc Duy<pclouds@gmail.com> wrote:
>>> On Mon, Aug 17, 2009 at 1:09 PM, <skillzero@gmail.com> wrote:
>>>> 1. Have people decided whether it should be on by default if you have
>>>> a .git/info/sparse file? I'd definitely like it to be on by
>>>> default. When I first tried it, I didn't realize I had to use
>>>> --sparse to git checkout to get it to use the sparse rules. The
>>>> same goes for a merge I did that happened to have a file in the
>>>> excluded area (it included it because I didn't use --sparse to git
>>>> merge).
>>> I tend to make it enabled by default too. I have made it stricter to
>>> trigger reading sparse in unpack_trees() -- only do it when
>>> unpack_opts.update is TRUE. This should make it safer to be enabled by
>>> default.
>> Other than it being new and not-widely-tested code, is there any
>> additional risk to having it enabled by default if there are no sparse
>> patterns defined?
>
> I think that in and of itself is reason enough to turn off the feature
> when .git/info/sparse is not present.
I might have missed something: Would there be any observable difference
between whether .git/info/sparse is absent and whether it is empty? If
not, what do you mean by "turn the feature off"?
>> It would be nice if .git/info/sparse is there by default (like
>> .git/info/exclude) with some commented out instructions (also like
>> .git/info/exclude).
>
> I'm not a fan of this idea.
For any particular reason?
-- Hannes
^ permalink raw reply
* Re: sparse support in pu
From: skillzero @ 2009-08-17 9:41 UTC (permalink / raw)
To: Johannes Schindelin; +Cc: Nguyen Thai Ngoc Duy, git
In-Reply-To: <alpine.DEB.1.00.0908171113420.4991@intel-tinevez-2-302>
On Mon, Aug 17, 2009 at 2:15 AM, Johannes
Schindelin<Johannes.Schindelin@gmx.de> wrote:
> Hi,
>
> On Mon, 17 Aug 2009, skillzero@gmail.com wrote:
>
>> On Mon, Aug 17, 2009 at 1:17 AM, Nguyen Thai Ngoc Duy<pclouds@gmail.com> wrote:
>> > On Mon, Aug 17, 2009 at 1:09 PM, <skillzero@gmail.com> wrote:
>> >> 1. Have people decided whether it should be on by default if you have
>> >> a .git/info/sparse file? I'd definitely like it to be on by
>> >> default. When I first tried it, I didn't realize I had to use
>> >> --sparse to git checkout to get it to use the sparse rules. The
>> >> same goes for a merge I did that happened to have a file in the
>> >> excluded area (it included it because I didn't use --sparse to git
>> >> merge).
>> >
>> > I tend to make it enabled by default too. I have made it stricter to
>> > trigger reading sparse in unpack_trees() -- only do it when
>> > unpack_opts.update is TRUE. This should make it safer to be enabled by
>> > default.
>>
>> Other than it being new and not-widely-tested code, is there any
>> additional risk to having it enabled by default if there are no sparse
>> patterns defined?
>
> I think that in and of itself is reason enough to turn off the feature
> when .git/info/sparse is not present.
>
> It also may have a runtime cost, dunno.
I was thinking that it would effectively do this:
Try to read .git/info/sparse
if valid patterns read
Sparse is enabled
else (e.g. file missing or nothing but empty lines/comments)
Sparse is disabled
I wouldn't think there would be any additional cost or risk (other
than the extra code to read the .git/info/sparse file) because the
result is the same as if sparse had been disabled by being defaulted
to off since even in the default off case, it's still a runtime check.
I would think --sparse would really only be useful as a way to negate
--no-sparse (i.e. act as if .git/info/sparse didn't exist
temporarily).
^ permalink raw reply
* Re: sparse support in pu
From: Johannes Schindelin @ 2009-08-17 9:15 UTC (permalink / raw)
To: skillzero; +Cc: Nguyen Thai Ngoc Duy, git
In-Reply-To: <2729632a0908170149o425544dcw52aeb6ac6ee1437d@mail.gmail.com>
Hi,
On Mon, 17 Aug 2009, skillzero@gmail.com wrote:
> On Mon, Aug 17, 2009 at 1:17 AM, Nguyen Thai Ngoc Duy<pclouds@gmail.com> wrote:
> > On Mon, Aug 17, 2009 at 1:09 PM, <skillzero@gmail.com> wrote:
> >> 1. Have people decided whether it should be on by default if you have
> >> a .git/info/sparse file? I'd definitely like it to be on by
> >> default. When I first tried it, I didn't realize I had to use
> >> --sparse to git checkout to get it to use the sparse rules. The
> >> same goes for a merge I did that happened to have a file in the
> >> excluded area (it included it because I didn't use --sparse to git
> >> merge).
> >
> > I tend to make it enabled by default too. I have made it stricter to
> > trigger reading sparse in unpack_trees() -- only do it when
> > unpack_opts.update is TRUE. This should make it safer to be enabled by
> > default.
>
> Other than it being new and not-widely-tested code, is there any
> additional risk to having it enabled by default if there are no sparse
> patterns defined?
I think that in and of itself is reason enough to turn off the feature
when .git/info/sparse is not present.
It also may have a runtime cost, dunno.
> It would be nice if .git/info/sparse is there by default (like
> .git/info/exclude) with some commented out instructions (also like
> .git/info/exclude).
I'm not a fan of this idea.
> >> 3. One thing that was confusing is that I needed a trailing slash on
> >> directories in .git/info/sparse to get them excluded. This seems
> >> different than .gitignore, which works for me without the trailing
> >> slash.
> >
> > Hmm.. probably because Git feeds directories to .gitignore handling
> > functions. There is not much I can do, index does not have
> > directories. I don't know if it's worth generating "directories" from
> > index.
>
> Maybe just add a note in the documentation? If there's a default
> .git/info/sparse file then it might be good place to put a note as well.
I rather think that this should be fixed. Maybe you can come up with a
patch to the tests which shows this behavior (with test_expect_failure)?
Then it will be much easier to come up with a fix.
Ciao,
Dscho
^ permalink raw reply
* Re: [msysGit] quick question about __stdcall at run-command.c mingw.c
From: Johannes Schindelin @ 2009-08-17 9:12 UTC (permalink / raw)
To: Johannes Sixt; +Cc: Pat Thoyts, Frank Li, git, msysGit
In-Reply-To: <4A891840.4050403@viscovery.net>
Hi,
On Mon, 17 Aug 2009, Johannes Sixt wrote:
> Johannes Schindelin schrieb:
> > On Mon, 17 Aug 2009, Johannes Sixt wrote:
> >> Pat Thoyts schrieb:
> >>> 2009/8/17 Frank Li <lznuaa@gmail.com>:
> >>>> I am tring to clear VC build patch.
> >>>>
> >>>> I found __stdcall position break MSVC build.
> >>>>
> >>>> static __stdcall unsigned run_thread(void *data)
> >>>>
> >>>> MSVC require __stdcall should be between return type and function
> >>>> name. like static unsigned __stdcall run_thread(void *data)
> >>>>
> >>>> I think msys gcc should support MSVC format.
> >
> > I think that it does.
> >
> > But it is _your_ duty to check.
>
> Cool down. Asking for "please could you check whether this works" *if*
> you don't have the infrastructure to test it yourself is certainly
> dutyful enough.
>
> Do you have an Irix, Solaris, HP box on your desk next to your Linux, so
> that you don't have to ask others to test your patches?
This is Windows. Frank has Windows.
Downloading msysGit does not incur any cost. Actually running the whole
thing from the net installer just takes a little time and a two-digit
megabyte download.
I put a lot of work into making this procedure as painless to use (for
other people, it caused me a lot of pain). So Frank might just as well
not let my effort go to hell.
Of course, Frank could ask one of the few msysGit contributors to try to
compile something he prepared, wait for the results and possibly fix
things for another round. This appears as a rather poor use of
everybody's time, methinks.
Ciao,
Dscho
^ permalink raw reply
* Re: git http-push and MKCOL error (22/409)
From: Tay Ray Chuan @ 2009-08-17 9:09 UTC (permalink / raw)
To: Thomas Schlichter; +Cc: Junio C Hamano, willievu, Sean Davis, git
In-Reply-To: <200908170725.09592.thomas.schlichter@web.de>
Hi,
On Mon, Aug 17, 2009 at 1:25 PM, Thomas Schlichter<thomas.schlichter@web.de> wrote:
> Hmm. I hope you won't take this as an argument for not fixing it, because it
> is a clear regression! I tried git version 1.6.3.4, and it works flawlessly
> with exactly this server! Even during bisecting, some versions worked, some
> didn't (these after the mentioned commit...)
>
> So I think this commit didn't only refactor things, but unintentionally
> changed the behavior. And this must be the problem. As I'm not into this code,
> and the refactoring was not completely trivial, I was not able to quickly find
> the place where the behavior was changed...
Thomas, indeed you're right about this, the commit you noted in your
earlier email (5424bc5) was indeed at fault.
The commit changed how urls was handled, and resulted (incorrectly) in
an extra slash being added to the url. In other words, urls now look
like this:
http://server/repo.git//objects/2a/
This problem occurs only in http-push.c, because repo urls
(http://server/repo.git) are always made to end with a slash there.
Btw, on my Apache 2.2.x server, this didn't crop up when I tested,
while Thomas worked with a 2.0.x.
I've attached a patch below that hopefully fixes it.
-- >8 --
Subject: [PATCH] http.c: don't assume that urls don't end with slash
Make append_remote_object_url() (and by implication,
get_remote_object_url) use end_url_with_slash() to ensure that the url
ends with a slash.
Previously, they assumed that the url did not end with a slash and
as a result appended a slash, sometimes errorneously.
This fixes an issue introduced in 5424bc5 ("http*: add helper methods
for fetching objects (loose)"), where the append_remote_object_url()
implementation in http-push.c, which assumed that urls end with a
slash, was replaced by another one in http.c, which assumed urls did
not end with a slash.
The above issue was raised by Thomas Schlichter:
http://marc.info/?l=git&m=125043105231327
Signed-off-by: Tay Ray Chuan <rctay89@gmail.com>
---
http.c | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/http.c b/http.c
index 14d5357..eb0c669 100644
--- a/http.c
+++ b/http.c
@@ -719,7 +719,9 @@ void append_remote_object_url(struct strbuf *buf, const char *url,
const char *hex,
int only_two_digit_prefix)
{
- strbuf_addf(buf, "%s/objects/%.*s/", url, 2, hex);
+ end_url_with_slash(buf, url);
+
+ strbuf_addf(buf, "objects/%.*s/", 2, hex);
if (!only_two_digit_prefix)
strbuf_addf(buf, "%s", hex+2);
}
--
1.6.4.dirty
^ permalink raw reply related
* Re: [RFC PATCH v3 8/8] --sparse for porcelains
From: Johannes Schindelin @ 2009-08-17 9:08 UTC (permalink / raw)
To: Jakub Narebski; +Cc: Junio C Hamano, Nguyen Thai Ngoc Duy, git
In-Reply-To: <alpine.DEB.1.00.0908161002460.8306@pacific.mpi-cbg.de>
Hi,
On Sun, 16 Aug 2009, Johannes Schindelin wrote:
> [...] if you want to say that assume-unchanged and sparse are two
> fundamentally different things, I would be interested in some equally
> convincing argument as for the shallow/graft issue.
>
> There is a fundamental difference, I grant you that: the working
> directory does not contain the "sparse'd away" files while the same is
> not true for assume-unchanged files.
>
> But does that matter? The corresponding files are still in the index
> and the repository.
>
> IOW under what circumstances would you want to be able to discern
> between assume-unchanged and "sparse'd away" files in the working
> directory?
>
> I could _imagine_ that you'd want a tool that allows you to change the
> focus of the sparse checkout together with the working directory.
> Example: you have a sparse checkout of Documentation/ and now you want
> to have t/, too. Just changing .git/info/sparse will not be enough.
>
> The question is if the tool to change the "sparseness" [*1*] should not
> change .git/info/sparse itself; if it does not, it would be good to be
> able to discern between the "assume-unchanged" and "sparse'd away"
> files.
>
> Although it might be enough to traverse the index and check the presence
> of the assume-unchanged files in the working directory to determine
> which files are sparse, and which ones are merely assume-unchanged.
>
> Ciao,
> Dscho
>
> Footnote [*1*]: I think we need some nice and clear nomenclature here.
> Any English wizards with a good taste of naming things?
Turns out that somebody on IRC had a problem that requires to have
sparse'd out files which _do_ have working directory copies.
So just having the assume-changed bit may not be enough.
The scenario is this: the repository contains a file that users are
supposed to change, but not commit to (only the super-intelligent inventor
of this scenario is allowed to). As this repository is originally a
subversion one, there is no problem: people just do not switch branches.
But this guy uses git-svn, so he does switch branches, and to avoid
committing the file by mistake, he marked it assume-unchanged. Only that
a branch switch overwrites the local changes.
I suggested the use of the sparse feature, and mark this file (and this
file alone) as sparse'd-out.
Is this an intended usage scenario? Then we cannot reuse the
assume-changed bit [*1*].
Ciao,
Dscho
Footnote [*1*]: in this particular scenario, we could still discern
between sparse'd-out and regular assume-unchanged file, because
.git/info/sparse knows about the file. But the design is now brittle, and
it is not hard at all to come up with a situation where it breaks.
^ permalink raw reply
* Re: [PATCH 8/9] git status: not "commit --dry-run" anymore
From: Thomas Rast @ 2009-08-17 8:57 UTC (permalink / raw)
To: Junio C Hamano; +Cc: git
In-Reply-To: <1250375997-10657-9-git-send-email-gitster@pobox.com>
Junio C Hamano wrote:
> This removes tentative "git stat" and make it take over "git status".
>
> Signed-off-by: Junio C Hamano <gitster@pobox.com>
> ---
>
> * This alone fails some tests; 9/9 will be squashed in in the final round.
>
> Documentation/git-status.txt | 79 ++++++++++++++++++++++++++++++++++++-----
> builtin-commit.c | 29 ++-------------
> builtin.h | 1 -
> git.c | 1 -
> 4 files changed, 73 insertions(+), 37 deletions(-)
[...]
> - { "stat", cmd_stat, RUN_SETUP | NEED_WORK_TREE },
> { "status", cmd_status, RUN_SETUP | NEED_WORK_TREE },
This lacks a corresponding update to the Makefile:
diff --git i/Makefile w/Makefile
index f384a52..8a7509c 100644
--- i/Makefile
+++ w/Makefile
@@ -383,7 +383,6 @@ BUILT_INS += git-init$X
BUILT_INS += git-merge-subtree$X
BUILT_INS += git-peek-remote$X
BUILT_INS += git-repo-config$X
-BUILT_INS += git-stat$X
BUILT_INS += git-show$X
BUILT_INS += git-stage$X
BUILT_INS += git-status$X
--
Thomas Rast
trast@{inf,student}.ethz.ch
^ permalink raw reply related
* Re: sparse support in pu
From: skillzero @ 2009-08-17 8:49 UTC (permalink / raw)
To: Nguyen Thai Ngoc Duy; +Cc: git
In-Reply-To: <fcaeb9bf0908170117v67e9f8b1ga56edcda14821e91@mail.gmail.com>
On Mon, Aug 17, 2009 at 1:17 AM, Nguyen Thai Ngoc Duy<pclouds@gmail.com> wrote:
> On Mon, Aug 17, 2009 at 1:09 PM, <skillzero@gmail.com> wrote:
>> 1. Have people decided whether it should be on by default if you have
>> a .git/info/sparse file? I'd definitely like it to be on by default.
>> When I first tried it, I didn't realize I had to use --sparse to git
>> checkout to get it to use the sparse rules. The same goes for a merge
>> I did that happened to have a file in the excluded area (it included
>> it because I didn't use --sparse to git merge).
>
> I tend to make it enabled by default too. I have made it stricter to
> trigger reading sparse in unpack_trees() -- only do it when
> unpack_opts.update is TRUE. This should make it safer to be enabled by
> default.
Other than it being new and not-widely-tested code, is there any
additional risk to having it enabled by default if there are no sparse
patterns defined?
It would be nice if .git/info/sparse is there by default (like
.git/info/exclude) with some commented out instructions (also like
.git/info/exclude).
>> 2. Is it not hooked up to git reset yet? I did a git checkout --sparse
>> and things look liked I expected then I did a git reset --hard
>> origin/master and it started checking out all the stuff previously
>> excluded via .git/info/sparse. I tried --sparse, but it didn't know
>> about that option.
>
> Because sparse was disabled by default, and "git reset" did not enable
> it. It'd be interesting to see what "git reset --hard" should do in
> this case: will it apply .git/info/sparse or not, which brings us back
> to the "default or not" question, hmm..
It seems like if it's going to be off by default (which it hopefully
won't be) then git reset would need to support --sparse since
otherwise, you'd never be able to git reset without undoing your
sparse options (unless you did a subsequent git checkout --sparse).
I also noticed that after I did the git reset, the sparse stuff seemed
to get into a weird state such that sparse patterns weren't working
reliably. For example, I couldn't get it to exclude a directory.
>> 3. One thing that was confusing is that I needed a trailing slash on
>> directories in .git/info/sparse to get them excluded. This seems
>> different than .gitignore, which works for me without the trailing
>> slash.
>
> Hmm.. probably because Git feeds directories to .gitignore handling
> functions. There is not much I can do, index does not have
> directories. I don't know if it's worth generating "directories" from
> index.
Maybe just add a note in the documentation? If there's a default
.git/info/sparse file then it might be good place to put a note as
well.
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox