* Re: [PATCH 5/6 (v4)] full integration of rev-cache into git, completed test suite
From: Nick Edelen @ 2009-09-07 14:11 UTC (permalink / raw)
To: Nick Edelen, Junio C Hamano, Nicolas Pitre, Johannes Schindelin,
Sam Vilain
In-Reply-To: <op.uyzwycxotdk399@sirnot>
This last patch provides a working integration of rev-cache into the revision
walker, along with some touch-ups:
- integration into revision walker and list-objects
- tweak of object generation
- more fluid handling of damaged cache slices
- numerous tests for both features from the previous patch, and the
integration's integrity
'Integration' is rather broad -- a more detailed description follows for each
aspect:
- rev-cache
the traversal mechanism is updated to handle many of the non-prune options
rev-list does (date limiting, slop-handling, etc.), and is adjusted to allow
for non-fatal cache-traversal failures.
- revision walker
both limited and unlimited traversal attempt to use the cache when possible,
smoothly falling back if it's not.
- list-objects
object listing does not recurse into cached trees, and has been adjusted to
guarantee commit-tag-tree-blob ordering.
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
Fixed issue with upload-pack test. It was a simple matter of event ordering
(printing trees before/after attempting to parse them).
builtin-rev-cache.c | 40 ++++++++
list-objects.c | 46 ++++++++-
rev-cache.c | 230 +++++++++++++++++++++++++++++++++++++++-----
revision.c | 88 ++++++++++++++---
t/t6017-rev-cache-list.sh | 151 ++++++++++++++++++++++++++++-
5 files changed, 501 insertions(+), 54 deletions(-)
diff --git a/builtin-rev-cache.c b/builtin-rev-cache.c
index b894c54..8f41123 100644
--- a/builtin-rev-cache.c
+++ b/builtin-rev-cache.c
@@ -4,6 +4,7 @@
#include "diff.h"
#include "revision.h"
#include "rev-cache.h"
+#include "list-objects.h"
unsigned long default_ignore_size = 50 * 1024 * 1024; /* 50mb */
@@ -78,6 +79,43 @@ static int handle_add(int argc, const char *argv[]) /* args beyond this command
return 0;
}
+static void show_commit(struct commit *commit, void *data)
+{
+ printf("%s\n", sha1_to_hex(commit->object.sha1));
+}
+
+static void show_object(struct object *obj, const struct name_path *path, const char *last)
+{
+ printf("%s\n", sha1_to_hex(obj->sha1));
+}
+
+static int test_rev_list(int argc, const char *argv[])
+{
+ struct rev_info revs;
+ unsigned int flags = 0;
+ int i;
+
+ init_revisions(&revs, 0);
+
+ for (i = 0; i < argc; i++) {
+ if (!strcmp(argv[i], "--not"))
+ flags ^= UNINTERESTING;
+ else if (!strcmp(argv[i], "--objects"))
+ revs.tree_objects = revs.blob_objects = 1;
+ else
+ handle_revision_arg(argv[i], &revs, flags, 1);
+ }
+
+ setup_revisions(0, 0, &revs, 0);
+ revs.topo_order = 1;
+ revs.lifo = 1;
+ prepare_revision_walk(&revs);
+
+ traverse_commit_list(&revs, show_commit, show_object, 0);
+
+ return 0;
+}
+
static int handle_walk(int argc, const char *argv[])
{
struct commit *commit;
@@ -271,6 +309,8 @@ int cmd_rev_cache(int argc, const char *argv[], const char *prefix)
r = handle_walk(argc, argv);
else if (!strcmp(arg, "index"))
r = handle_index(argc, argv);
+ else if (!strcmp(arg, "test"))
+ r = test_rev_list(argc, argv);
else if (!strcmp(arg, "alt"))
r = handle_alt(argc, argv);
else
diff --git a/list-objects.c b/list-objects.c
index 8953548..59df8c7 100644
--- a/list-objects.c
+++ b/list-objects.c
@@ -74,22 +74,34 @@ static void process_tree(struct rev_info *revs,
die("bad tree object");
if (obj->flags & (UNINTERESTING | SEEN))
return;
+ if (obj->flags & FACE_VALUE) {
+ obj->flags |= SEEN;
+ show(obj, path, name);
+ /* not parsing the tree saves a lot of time! */
+ return;
+ }
+
if (parse_tree(tree) < 0)
die("bad tree object %s", sha1_to_hex(obj->sha1));
obj->flags |= SEEN;
show(obj, path, name);
+
me.up = path;
me.elem = name;
me.elem_len = strlen(name);
-
init_tree_desc(&desc, tree->buffer, tree->size);
while (tree_entry(&desc, &entry)) {
- if (S_ISDIR(entry.mode))
+ if (S_ISDIR(entry.mode)) {
+ struct tree *subtree = lookup_tree(entry.sha1);
+ if (!subtree)
+ continue;
+
+ subtree->object.flags &= ~FACE_VALUE;
process_tree(revs,
- lookup_tree(entry.sha1),
+ subtree,
show, &me, entry.path);
- else if (S_ISGITLINK(entry.mode))
+ } else if (S_ISGITLINK(entry.mode))
process_gitlink(revs, entry.sha1,
show, &me, entry.path);
else
@@ -136,6 +148,7 @@ void mark_edges_uninteresting(struct commit_list *list,
static void add_pending_tree(struct rev_info *revs, struct tree *tree)
{
+ tree->object.flags &= ~FACE_VALUE;
add_pending_object(revs, &tree->object, "");
}
@@ -146,17 +159,27 @@ void traverse_commit_list(struct rev_info *revs,
{
int i;
struct commit *commit;
+ enum object_type what = OBJ_TAG;
+ char face_value = 0;
while ((commit = get_revision(revs)) != NULL) {
- add_pending_tree(revs, commit->tree);
+ if (!(commit->object.flags & FACE_VALUE))
+ add_pending_tree(revs, commit->tree);
+ else
+ face_value = 1;
show_commit(commit, data);
}
+
+loop_objects:
for (i = 0; i < revs->pending.nr; i++) {
struct object_array_entry *pending = revs->pending.objects + i;
struct object *obj = pending->item;
const char *name = pending->name;
if (obj->flags & (UNINTERESTING | SEEN))
continue;
+ if (obj->type != what && face_value)
+ continue;
+
if (obj->type == OBJ_TAG) {
obj->flags |= SEEN;
show_object(obj, NULL, name);
@@ -175,6 +198,19 @@ void traverse_commit_list(struct rev_info *revs,
die("unknown pending object %s (%s)",
sha1_to_hex(obj->sha1), name);
}
+ if (face_value) {
+ switch (what) {
+ case OBJ_TAG :
+ what = OBJ_TREE;
+ goto loop_objects;
+ case OBJ_TREE :
+ what = OBJ_BLOB;
+ goto loop_objects;
+ default :
+ break;
+ }
+ }
+
if (revs->pending.nr) {
free(revs->pending.objects);
revs->pending.nr = 0;
diff --git a/rev-cache.c b/rev-cache.c
index 8ca97d3..6becd4b 100644
--- a/rev-cache.c
+++ b/rev-cache.c
@@ -11,6 +11,12 @@
#include "run-command.h"
#include "string-list.h"
+
+struct bad_slice {
+ unsigned char sha1[20];
+ struct bad_slice *next;
+};
+
struct cache_slice_pointer {
char signature[8]; /* REVCOPTR */
char version;
@@ -23,8 +29,9 @@ static uint32_t fanout[0xff + 2];
static unsigned char *idx_map;
static int idx_size;
static struct rc_index_header idx_head;
+static char no_idx, add_to_pending;
+static struct bad_slice *bad_slices;
static unsigned char *idx_caches;
-static char no_idx;
static struct strbuf *acc_buffer;
@@ -121,6 +128,30 @@ struct rc_object_entry_ondisk *to_disked_rc_object_entry(struct rc_object_entry
return dst;
}
+static void mark_bad_slice(unsigned char *sha1)
+{
+ struct bad_slice *bad;
+
+ bad = xcalloc(sizeof(struct bad_slice), 1);
+ hashcpy(bad->sha1, sha1);
+
+ bad->next = bad_slices;
+ bad_slices = bad;
+}
+
+static int is_bad_slice(unsigned char *sha1)
+{
+ struct bad_slice *bad = bad_slices;
+
+ while (bad) {
+ if (!hashcmp(bad->sha1, sha1))
+ return 1;
+ bad = bad->next;
+ }
+
+ return 0;
+}
+
static int get_index_head(unsigned char *map, int len, struct rc_index_header *head, uint32_t *fanout, unsigned char **caches)
{
struct rc_index_header whead;
@@ -246,6 +277,7 @@ static struct rc_index_entry *search_index(unsigned char *sha1)
unsigned char *get_cache_slice(struct commit *commit)
{
struct rc_index_entry *ie;
+ unsigned char *sha1;
if (!idx_map) {
if (no_idx)
@@ -257,8 +289,13 @@ unsigned char *get_cache_slice(struct commit *commit)
return 0;
ie = search_index(commit->object.sha1);
- if (ie && ie->cache_index < idx_head.cache_nr)
- return idx_caches + ie->cache_index * 20;
+ if (ie && ie->cache_index < idx_head.cache_nr) {
+ sha1 = idx_caches + ie->cache_index * 20;
+
+ if (is_bad_slice(sha1))
+ return 0;
+ return sha1;
+ }
return 0;
}
@@ -268,6 +305,20 @@ unsigned char *get_cache_slice(struct commit *commit)
static unsigned long decode_size(unsigned char *str, int len);
+/* on failure */
+static void restore_commit(struct commit *commit)
+{
+ commit->object.flags &= ~(ADDED | SEEN | FACE_VALUE);
+
+ if (!commit->object.parsed) {
+ while (pop_commit(&commit->parents))
+ ;
+
+ parse_commit(commit);
+ }
+
+}
+
static void handle_noncommit(struct rev_info *revs, struct commit *commit, unsigned char *ptr, struct rc_object_entry *entry)
{
struct blob *blob;
@@ -307,23 +358,27 @@ static void handle_noncommit(struct rev_info *revs, struct commit *commit, unsig
}
obj->flags |= FACE_VALUE;
- add_pending_object(revs, obj, "");
+ if (add_to_pending)
+ add_pending_object(revs, obj, "");
}
-static int setup_traversal(struct rc_slice_header *head, unsigned char *map, struct commit *commit, struct commit_list **work)
+static int setup_traversal(struct rc_slice_header *head, unsigned char *map, struct commit *commit, struct commit_list **work,
+ struct commit_list **unwork, int *ipath_nr, int *upath_nr, char *ioutside)
{
struct rc_index_entry *iep;
struct rc_object_entry *oep;
struct commit_list *prev, *wp, **wpp;
int retval;
- iep = search_index(commit->object.sha1), 0;
+ iep = search_index(commit->object.sha1);
oep = RC_OBTAIN_OBJECT_ENTRY(map + iep->pos);
+ if (commit->object.flags & UNINTERESTING) {
+ ++*upath_nr;
+ oep->uninteresting = 1;
+ } else
+ ++*ipath_nr;
- /* the .uniniteresting bit isn't strictly necessary, as we check the object during traversal as well,
- * but we might as well initialize it while we're at it */
oep->include = 1;
- oep->uninteresting = !!(commit->object.flags & UNINTERESTING);
to_disked_rc_object_entry(oep, (struct rc_object_entry_ondisk *)(map + iep->pos));
retval = iep->pos;
@@ -338,6 +393,10 @@ static int setup_traversal(struct rc_slice_header *head, unsigned char *map, str
/* is this in our cache slice? */
iep = search_index(obj->sha1);
if (!iep || hashcmp(idx_caches + iep->cache_index * 20, head->sha1)) {
+ /* there are interesing objects outside the slice */
+ if (!(obj->flags & UNINTERESTING))
+ *ioutside = 1;
+
prev = wp;
wp = wp->next;
wpp = ℘
@@ -354,11 +413,20 @@ static int setup_traversal(struct rc_slice_header *head, unsigned char *map, str
oep->uninteresting = !!(obj->flags & UNINTERESTING);
to_disked_rc_object_entry(oep, (struct rc_object_entry_ondisk *)(map + iep->pos));
+ /* count even if not in slice so we can stop enumerating if possible */
+ if (obj->flags & UNINTERESTING)
+ ++*upath_nr;
+ else
+ ++*ipath_nr;
+
/* remove from work list */
co = pop_commit(wpp);
wp = *wpp;
if (prev)
prev->next = wp;
+
+ /* ...and store in temp list so we can restore work on failure */
+ commit_list_insert(co, unwork);
}
return retval;
@@ -375,13 +443,18 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
unsigned long *date_so_far, int *slop_so_far,
struct commit_list ***queue, struct commit_list **work)
{
- struct commit_list *insert_cache = 0;
+ struct commit_list *insert_cache = 0, *myq = 0, **myqp = &myq, *mywork = 0, **myworkp = &mywork, *unwork = 0;
struct commit **last_objects, *co;
- int i, total_path_nr = head->path_nr, retval = -1;
- char consume_children = 0;
+ unsigned long date = date_so_far ? *date_so_far : ~0ul;
+ int i, ipath_nr = 0, upath_nr = 0, orig_obj_nr = 0,
+ total_path_nr = head->path_nr, retval = -1, slop = slop_so_far ? *slop_so_far : SLOP;
+ char consume_children = 0, ioutside = 0;
unsigned char *paths;
- i = setup_traversal(head, map, commit, work);
+ /* take note in case we need to regress */
+ orig_obj_nr = revs->pending.nr;
+
+ i = setup_traversal(head, map, commit, work, &unwork, &ipath_nr, &upath_nr, &ioutside);
if (i < 0)
return -1;
@@ -429,6 +502,7 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
if ((paths[path] & IPATH) && (paths[path] & UPATH)) {
paths[path] = UPATH;
+ ipath_nr--;
/* mark edge */
if (last_objects[path]) {
@@ -439,6 +513,7 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
last_objects[path]->object.flags &= ~FACE_VALUE;
last_objects[path] = 0;
}
+ obj->flags |= BOUNDARY;
}
/* now we gotta re-assess the whole interesting thing... */
@@ -462,8 +537,10 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
last_objects[p]->object.flags &= ~FACE_VALUE;
last_objects[p] = 0;
}
- } else if (last_objects[p] && !last_objects[p]->object.parsed)
+ obj->flags |= BOUNDARY;
+ } else if (last_objects[p] && !last_objects[p]->object.parsed) {
commit_list_insert(co, &last_objects[p]->parents);
+ }
/* can't close a merge path until all are parents have been encountered */
if (GET_COUNT(paths[p])) {
@@ -473,14 +550,33 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
continue;
}
+ if (paths[p] & IPATH)
+ ipath_nr--;
+ else
+ upath_nr--;
+
paths[p] = 0;
last_objects[p] = 0;
}
}
/* make topo relations */
- if (last_objects[path] && !last_objects[path]->object.parsed)
+ if (last_objects[path] && !last_objects[path]->object.parsed) {
commit_list_insert(co, &last_objects[path]->parents);
+ }
+
+ /* we've been here already */
+ if (obj->flags & ADDED) {
+ if (entry->uninteresting && !(obj->flags & UNINTERESTING)) {
+ obj->flags |= UNINTERESTING;
+ mark_parents_uninteresting(co);
+ upath_nr--;
+ } else if (!entry->uninteresting)
+ ipath_nr--;
+
+ paths[path] = 0;
+ continue;
+ }
/* initialize commit */
if (!entry->is_end) {
@@ -493,24 +589,51 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
if (entry->uninteresting)
obj->flags |= UNINTERESTING;
+ else if (co->date < date)
+ date = co->date;
/* we need to know what the edges are */
last_objects[path] = co;
/* add to list */
- if (!(obj->flags & UNINTERESTING) || revs->show_all) {
- if (entry->is_end)
- insert_by_date_cached(co, work, insert_cache, &insert_cache);
- else
- *queue = &commit_list_insert(co, *queue)->next;
+ if (slop && !(revs->min_age != -1 && co->date > revs->min_age)) {
+
+ if (!(obj->flags & UNINTERESTING) || revs->show_all) {
+ if (entry->is_end)
+ myworkp = &commit_list_insert(co, myworkp)->next;
+ else
+ myqp = &commit_list_insert(co, myqp)->next;
+
+ /* add children to list as well */
+ if (obj->flags & UNINTERESTING)
+ consume_children = 0;
+ else
+ consume_children = 1;
+ }
- /* add children to list as well */
- if (obj->flags & UNINTERESTING)
- consume_children = 0;
- else
- consume_children = 1;
}
+ /* should we continue? */
+ if (!slop) {
+ if (!upath_nr) {
+ break;
+ } else if (ioutside || revs->show_all) {
+ /* pass it back to rev-list
+ * we purposely ignore everything outside this cache, so we don't needlessly traverse the whole
+ * thing on uninteresting, but that does mean that we may need to bounce back
+ * and forth a few times with rev-list */
+ myworkp = &commit_list_insert(co, myworkp)->next;
+
+ paths[path] = 0;
+ upath_nr--;
+ } else {
+ break;
+ }
+ } else if (!ipath_nr && co->date <= date)
+ slop--;
+ else
+ slop = SLOP;
+
/* open parents */
if (entry->merge_nr) {
int j, off = index + sizeof(struct rc_object_entry_ondisk);
@@ -525,6 +648,11 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
if (paths[p] & flag)
continue;
+ if (flag == IPATH)
+ ipath_nr++;
+ else
+ upath_nr++;
+
paths[p] |= flag;
}
@@ -534,12 +662,55 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
}
+ if (date_so_far)
+ *date_so_far = date;
+ if (slop_so_far)
+ *slop_so_far = slop;
retval = 0;
+ /* success: attach to given lists */
+ if (myqp != &myq) {
+ **queue = myq;
+ *queue = myqp;
+ }
+
+ while ((co = pop_commit(&mywork)) != 0) {
+ insert_by_date_cached(co, work, insert_cache, &insert_cache);
+ }
+
+ /* free backup */
+ while (pop_commit(&unwork))
+ ;
+
end:
free(paths);
free(last_objects);
+ /* failure: restore work to previous condition
+ * (cache corruption should *not* be fatal) */
+ if (retval) {
+ while ((co = pop_commit(&unwork)) != 0) {
+ restore_commit(co);
+ co->object.flags |= SEEN;
+ insert_by_date(co, work);
+ }
+
+ /* free lists */
+ while ((co = pop_commit(&myq)) != 0)
+ restore_commit(co);
+
+ while ((co = pop_commit(&mywork)) != 0)
+ restore_commit(co);
+
+ /* truncate object array */
+ for (i = orig_obj_nr; i < revs->pending.nr; i++) {
+ struct object *obj = revs->pending.objects[i].item;
+
+ obj->flags &= ~FACE_VALUE;
+ }
+ revs->pending.nr = orig_obj_nr;
+ }
+
return retval;
}
@@ -630,6 +801,7 @@ int traverse_cache_slice(struct rev_info *revs,
/* load options */
rci = &revs->rev_cache_info;
+ add_to_pending = rci->add_to_pending;
memset(&head, 0, sizeof(struct rc_slice_header));
@@ -653,6 +825,10 @@ end:
if (fd != -1)
close(fd);
+ /* remember this! */
+ if (retval)
+ mark_bad_slice(cache_sha1);
+
return retval;
}
@@ -1128,7 +1304,7 @@ static int add_objects_verbatim_1(struct rev_cache_slice_map *mapping, int *inde
while (i < mapping->size) {
int pos = i;
- entry = RC_OBTAIN_OBJECT_ENTRY(map + i;
+ entry = RC_OBTAIN_OBJECT_ENTRY(map + i);
i += RC_ACTUAL_OBJECT_ENTRY_SIZE(entry);
if (entry->type == OBJ_COMMIT) {
diff --git a/revision.c b/revision.c
index c7fd35f..ed21885 100644
--- a/revision.c
+++ b/revision.c
@@ -12,6 +12,7 @@
#include "patch-ids.h"
#include "decorate.h"
#include "log-tree.h"
+#include "rev-cache.h"
volatile show_early_output_fn_t show_early_output;
@@ -638,6 +639,8 @@ static int limit_list(struct rev_info *revs)
struct commit_list *list = revs->commits;
struct commit_list *newlist = NULL;
struct commit_list **p = &newlist;
+ unsigned char *cache_sha1;
+ char used_cache;
while (list) {
struct commit_list *entry = list;
@@ -650,24 +653,39 @@ static int limit_list(struct rev_info *revs)
if (revs->max_age != -1 && (commit->date < revs->max_age))
obj->flags |= UNINTERESTING;
- if (add_parents_to_list(revs, commit, &list, NULL) < 0)
- return -1;
- if (obj->flags & UNINTERESTING) {
- mark_parents_uninteresting(commit);
- if (revs->show_all)
- p = &commit_list_insert(commit, p)->next;
- slop = still_interesting(list, date, slop);
- if (slop)
+
+ /* rev-cache to the rescue!!! */
+ used_cache = 0;
+ if (!revs->dont_cache_me && !(obj->flags & ADDED)) {
+ cache_sha1 = get_cache_slice(commit);
+ if (cache_sha1) {
+ if (traverse_cache_slice(revs, cache_sha1, commit, &date, &slop, &p, &list) < 0)
+ used_cache = 0;
+ else
+ used_cache = 1;
+ }
+ }
+
+ if (!used_cache) {
+ if (add_parents_to_list(revs, commit, &list, NULL) < 0)
+ return -1;
+ if (obj->flags & UNINTERESTING) {
+ mark_parents_uninteresting(commit); /* ME: why? */
+ if (revs->show_all)
+ p = &commit_list_insert(commit, p)->next;
+ slop = still_interesting(list, date, slop);
+ if (slop > 0)
+ continue;
+ /* If showing all, add the whole pending list to the end */
+ if (revs->show_all)
+ *p = list;
+ break;
+ }
+ if (revs->min_age != -1 && (commit->date > revs->min_age))
continue;
- /* If showing all, add the whole pending list to the end */
- if (revs->show_all)
- *p = list;
- break;
+ date = commit->date;
+ p = &commit_list_insert(commit, p)->next;
}
- if (revs->min_age != -1 && (commit->date > revs->min_age))
- continue;
- date = commit->date;
- p = &commit_list_insert(commit, p)->next;
show = show_early_output;
if (!show)
@@ -813,6 +831,8 @@ void init_revisions(struct rev_info *revs, const char *prefix)
revs->diffopt.prefix = prefix;
revs->diffopt.prefix_length = strlen(prefix);
}
+
+ init_rev_cache_info(&revs->rev_cache_info);
}
static void add_pending_commit_list(struct rev_info *revs,
@@ -1372,6 +1392,11 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, const ch
if (revs->reflog_info && revs->graph)
die("cannot combine --walk-reflogs with --graph");
+ /* limits on caching
+ * todo: implement this functionality */
+ if (revs->prune || revs->diff)
+ revs->dont_cache_me = 1;
+
return left;
}
@@ -1654,6 +1679,8 @@ static int commit_match(struct commit *commit, struct rev_info *opt)
{
if (!opt->grep_filter.pattern_list)
return 1;
+ if (!commit->object.parsed)
+ parse_commit(commit);
return grep_buffer(&opt->grep_filter,
NULL, /* we say nothing, not even filename */
commit->buffer, strlen(commit->buffer));
@@ -1717,6 +1744,7 @@ static struct commit *get_revision_1(struct rev_info *revs)
do {
struct commit_list *entry = revs->commits;
struct commit *commit = entry->item;
+ struct object *obj = &commit->object;
revs->commits = entry->next;
free(entry);
@@ -1733,11 +1761,39 @@ static struct commit *get_revision_1(struct rev_info *revs)
if (revs->max_age != -1 &&
(commit->date < revs->max_age))
continue;
+
+ if (!revs->dont_cache_me) {
+ struct commit_list *queue = 0, **queuep = &queue;
+ unsigned char *cache_sha1;
+
+ if (obj->flags & ADDED)
+ goto skip_parenting;
+
+ cache_sha1 = get_cache_slice(commit);
+ if (cache_sha1) {
+ if (!traverse_cache_slice(revs, cache_sha1, commit, 0, 0, &queuep, &revs->commits)) {
+ struct commit_list *work = revs->commits;
+
+ /* attach queue to end of ->commits */
+ while (work && work->next)
+ work = work->next;
+
+ if (work)
+ work->next = queue;
+ else
+ revs->commits = queue;
+
+ goto skip_parenting;
+ }
+ }
+ }
+
if (add_parents_to_list(revs, commit, &revs->commits, NULL) < 0)
die("Failed to traverse parents of commit %s",
sha1_to_hex(commit->object.sha1));
}
+skip_parenting:
switch (simplify_commit(revs, commit)) {
case commit_ignore:
continue;
diff --git a/t/t6017-rev-cache-list.sh b/t/t6017-rev-cache-list.sh
index dc0fc07..f2e34b1 100755
--- a/t/t6017-rev-cache-list.sh
+++ b/t/t6017-rev-cache-list.sh
@@ -38,6 +38,7 @@ test_expect_success 'init repo' '
git add . &&
git commit -m "omg" &&
+ sleep 2 &&
git branch b4 &&
git checkout b4 &&
echo shazam >file8 &&
@@ -46,7 +47,7 @@ test_expect_success 'init repo' '
git merge -m "merge b2" b2 &&
echo bam >smoke/pipe &&
- git add .
+ git add . &&
git commit -m "bam" &&
git checkout master &&
@@ -71,18 +72,26 @@ test_expect_success 'init repo' '
git add . &&
git commit -m "lol" &&
+ sleep 2 &&
git checkout master &&
git merge -m "triple merge" b1 b11 &&
git rm -r d1 &&
+ sleep 2 &&
git commit -a -m "oh noes"
'
-git-rev-list HEAD --not HEAD~3 >proper_commit_list_limited
-git-rev-list HEAD >proper_commit_list
-git-rev-list HEAD --objects >proper_object_list
+max_date=`git-rev-list --timestamp HEAD~1 --max-count=1 | grep -e "^[0-9]*" -o`
+min_date=`git-rev-list --timestamp b4 --max-count=1 | grep -e "^[0-9]*" -o`
+
+git-rev-list --topo-order HEAD --not HEAD~3 >proper_commit_list_limited
+git-rev-list --topo-order HEAD --not HEAD~2 >proper_commit_list_limited2
+git-rev-list --topo-order HEAD >proper_commit_list
+git-rev-list --objects HEAD >proper_object_list
+git-rev-list HEAD --max-age=$min_date --min-age=$max_date >proper_list_date_limited
+
+cache_sha1=`git-rev-cache add HEAD 2>output.err`
test_expect_success 'make cache slice' '
- git-rev-cache add HEAD 2>output.err &&
grep "final return value: 0" output.err
'
@@ -102,11 +111,141 @@ test_expect_success 'test rev-caches walker directly (unlimited)' '
test_cmp_sorted list proper_commit_list
'
+test_expect_success 'test rev-list traversal (limited)' '
+ git-rev-list HEAD --not HEAD~3 >list &&
+ test_cmp list proper_commit_list_limited
+'
+
+test_expect_success 'test rev-list traversal (unlimited)' '
+ git-rev-list HEAD >list &&
+ test_cmp list proper_commit_list
+'
+
#do the same for objects
test_expect_success 'test rev-caches walker with objects' '
git-rev-cache walk --objects HEAD >list &&
test_cmp_sorted list proper_object_list
'
-test_done
+test_expect_success 'test rev-list with objects (topo order)' '
+ git-rev-list --topo-order --objects HEAD >list &&
+ test_cmp_sorted list proper_object_list
+'
+
+test_expect_success 'test rev-list with objects (no order)' '
+ git-rev-list --objects HEAD >list &&
+ test_cmp_sorted list proper_object_list
+'
+
+#verify age limiting
+test_expect_success 'test rev-list date limiting (topo order)' '
+ git-rev-list --topo-order --max-age=$min_date --min-age=$max_date HEAD >list &&
+ test_cmp_sorted list proper_list_date_limited
+'
+
+test_expect_success 'test rev-list date limiting (no order)' '
+ git-rev-list --max-age=$min_date --min-age=$max_date HEAD >list &&
+ test_cmp_sorted list proper_list_date_limited
+'
+
+#check partial cache slice
+test_expect_success 'saving old cache and generating partial slice' '
+ cp ".git/rev-cache/$cache_sha1" .git/rev-cache/.old &&
+ rm ".git/rev-cache/$cache_sha1" .git/rev-cache/index &&
+
+ git-rev-cache add HEAD~2 2>output.err &&
+ grep "final return value: 0" output.err
+'
+
+test_expect_success 'rev-list with wholly interesting partial slice' '
+ git-rev-list --topo-order HEAD >list &&
+ test_cmp list proper_commit_list
+'
+
+test_expect_success 'rev-list with partly uninteresting partial slice' '
+ git-rev-list --topo-order HEAD --not HEAD~3 >list &&
+ test_cmp list proper_commit_list_limited
+'
+
+test_expect_success 'rev-list with wholly uninteresting partial slice' '
+ git-rev-list --topo-order HEAD --not HEAD~2 >list &&
+ test_cmp list proper_commit_list_limited2
+'
+
+#try out index generation and fuse (note that --all == HEAD in this case)
+#probably should make a test for that too...
+test_expect_success 'test (non-)fusion of one slice' '
+ git-rev-cache fuse >output.err &&
+ grep "nothing to fuse" output.err
+'
+test_expect_success 'make fresh slice' '
+ git-rev-cache add --all --fresh 2>output.err &&
+ grep "final return value: 0" output.err
+'
+
+test_expect_success 'check dual slices' '
+ git-rev-list --topo-order HEAD~2 HEAD >list &&
+ test_cmp list proper_commit_list
+'
+
+test_expect_success 'regenerate index' '
+ rm .git/rev-cache/index &&
+ git-rev-cache index 2>output.err &&
+ grep "final return value: 0" output.err
+'
+
+test_expect_success 'fuse slices' '
+ test -e .git/rev-cache/.old &&
+ git-rev-cache fuse 2>output.err &&
+ grep "final return value: 0" output.err &&
+ test_cmp .git/rev-cache/$cache_sha1 .git/rev-cache/.old
+'
+
+#make sure we can smoothly handle corrupted caches
+test_expect_success 'corrupt slice' '
+ echo bla >.git/rev-cache/$cache_sha1
+'
+
+test_expect_success 'test rev-list traversal (limited) (corrupt slice)' '
+ git-rev-list --topo-order HEAD --not HEAD~3 >list &&
+ test_cmp list proper_commit_list_limited
+'
+
+test_expect_success 'test rev-list traversal (unlimited) (corrupt slice)' '
+ git-rev-list HEAD >list &&
+ test_cmp_sorted list proper_commit_list
+'
+
+test_expect_success 'corrupt index' '
+ echo blu >.git/rev-cache/index
+'
+
+test_expect_success 'test rev-list traversal (limited) (corrupt index)' '
+ git-rev-list --topo-order HEAD --not HEAD~3 >list &&
+ test_cmp list proper_commit_list_limited
+'
+
+test_expect_success 'test rev-list traversal (unlimited) (corrupt index)' '
+ git-rev-list HEAD >list &&
+ test_cmp_sorted list proper_commit_list
+'
+
+#test --ignore-size in fuse
+rm .git/rev-cache/*
+cache_sha1=`git-rev-cache add HEAD~2 2>output.err`
+
+test_expect_success 'make fragmented slices' '
+ git-rev-cache add HEAD~1 --not HEAD~2 2>>output.err &&
+ git-rev-cache add HEAD --fresh 2>>output.err &&
+ test `grep "final return value: 0" output.err | wc -l` -eq 3
+'
+
+cache_size=`wc -c .git/rev-cache/$cache_sha1 | grep -o "[0-9]*"`
+test_expect_success 'test --ignore-size function in fuse' '
+ git-rev-cache fuse --ignore-size=$cache_size 2>output.err &&
+ grep "final return value: 0" output.err &&
+ test -e .git/rev-cache/$cache_sha1
+'
+
+test_done
--
tg: (e9374fc..) t/revcache/integration (depends on: t/revcache/misc)
^ permalink raw reply related
* Re: [PATCH 6/6 (v4)] support for path name caching in rev-cache
From: Nick Edelen @ 2009-09-07 14:11 UTC (permalink / raw)
To: Nick Edelen, Junio C Hamano, Nicolas Pitre, Johannes Schindelin,
Sam Vilain
In-Reply-To: <op.uyzwyho5tdk399@sirnot>
An update to caching mechanism, allowing path names to be cached for blob and
tree objects. A list of names appearing in each cache slice is appended to the
end of the slice, which is referenced by variable-sized indexes per entry.
This allows pack-objects to more intelligently schedule unpacked/poorly packed
object, and enables proper duplication of rev-list's behaivor.
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
builtin-rev-cache.c | 3 +-
rev-cache.c | 331 +++++++++++++++++++++++++++++++++++++--------
rev-cache.h | 16 ++-
revision.h | 6 +-
t/t6017-rev-cache-list.sh | 8 +-
5 files changed, 299 insertions(+), 65 deletions(-)
diff --git a/builtin-rev-cache.c b/builtin-rev-cache.c
index 8f41123..4c1766d 100644
--- a/builtin-rev-cache.c
+++ b/builtin-rev-cache.c
@@ -177,13 +177,14 @@ static int handle_walk(int argc, const char *argv[])
fprintf(stderr, "pending:\n");
for (i = 0; i < revs.pending.nr; i++) {
struct object *obj = revs.pending.objects[i].item;
+ const char *name = revs.pending.objects[i].name;
/* unfortunately, despite our careful generation, object duplication *is* a possibility...
* (eg. same object introduced into two different branches) */
if (obj->flags & SEEN)
continue;
- printf("%s\n", sha1_to_hex(revs.pending.objects[i].item->sha1));
+ printf("%s %s\n", sha1_to_hex(revs.pending.objects[i].item->sha1), name);
obj->flags |= SEEN;
}
diff --git a/rev-cache.c b/rev-cache.c
index 6becd4b..3595f66 100644
--- a/rev-cache.c
+++ b/rev-cache.c
@@ -17,6 +17,14 @@ struct bad_slice {
struct bad_slice *next;
};
+struct name_list {
+ unsigned char sha1[20];
+ unsigned int len;
+ struct name_list *next;
+
+ char buf[FLEX_ARRAY];
+};
+
struct cache_slice_pointer {
char signature[8]; /* REVCOPTR */
char version;
@@ -29,10 +37,13 @@ static uint32_t fanout[0xff + 2];
static unsigned char *idx_map;
static int idx_size;
static struct rc_index_header idx_head;
-static char no_idx, add_to_pending;
-static struct bad_slice *bad_slices;
+static char no_idx, add_to_pending, add_names;
static unsigned char *idx_caches;
+static struct bad_slice *bad_slices;
+static struct name_list *name_lists, *cur_name_list;
+
+static struct strbuf *acc_name_buffer;
static struct strbuf *acc_buffer;
#define SLOP 5
@@ -79,7 +90,7 @@ struct rc_object_entry *from_disked_rc_object_entry(struct rc_object_entry_ondis
if (!dst)
dst = &entry[cur++ & 0x3];
- dst->type = src->flags >> 5;
+ dst->type = src->flags >> 5 & 0x03;
dst->is_end = !!(src->flags & 0x10);
dst->is_start = !!(src->flags & 0x08);
dst->uninteresting = !!(src->flags & 0x04);
@@ -90,8 +101,9 @@ struct rc_object_entry *from_disked_rc_object_entry(struct rc_object_entry_ondis
dst->merge_nr = src->merge_nr;
dst->split_nr = src->split_nr;
- dst->size_size = src->sizes >> 5;
- dst->padding = src->sizes & 0x1f;
+ dst->size_size = src->sizes >> 5 & 0x03;
+ dst->name_size = src->sizes >> 2 & 0x03;
+ dst->padding = src->sizes & 0x02;
dst->date = ntohl(src->date);
dst->path = ntohs(src->path);
@@ -120,6 +132,7 @@ struct rc_object_entry_ondisk *to_disked_rc_object_entry(struct rc_object_entry
dst->split_nr = src->split_nr;
dst->sizes = (unsigned char)src->size_size << 5;
+ dst->sizes |= (unsigned char)src->name_size << 2;
dst->sizes |= (unsigned char)src->padding;
dst->date = htonl(src->date);
@@ -192,6 +205,12 @@ static void cleanup_cache_slices(void)
idx_map = 0;
}
+ while (name_lists) {
+ struct name_list *nl = name_lists->next;
+ free(name_lists);
+ name_lists = nl;
+ }
+
}
static int init_index(void)
@@ -324,7 +343,7 @@ static void handle_noncommit(struct rev_info *revs, struct commit *commit, unsig
struct blob *blob;
struct tree *tree;
struct object *obj;
- unsigned long size;
+ unsigned long size, name_index;
size = decode_size(ptr + RC_ENTRY_SIZE_OFFSET(entry), entry->size_size);
switch (entry->type) {
@@ -357,9 +376,22 @@ static void handle_noncommit(struct rev_info *revs, struct commit *commit, unsig
return;
}
+ if (add_names && cur_name_list) {
+ name_index = decode_size(ptr + RC_ENTRY_NAME_OFFSET(entry), entry->name_size);
+
+ if (name_index >= cur_name_list->len)
+ name_index = 0;
+ } else name_index = 0;
+
obj->flags |= FACE_VALUE;
- if (add_to_pending)
- add_pending_object(revs, obj, "");
+ if (add_to_pending) {
+ char *name = "";
+
+ if (name_index)
+ name = cur_name_list->buf + name_index;
+
+ add_pending_object(revs, obj, name);
+ }
}
static int setup_traversal(struct rc_slice_header *head, unsigned char *map, struct commit *commit, struct commit_list **work,
@@ -714,15 +746,44 @@ end:
return retval;
}
-static int get_cache_slice_header(unsigned char *cache_sha1, unsigned char *map, int len, struct rc_slice_header *head)
+static struct name_list *get_cache_slice_name_list(struct rc_slice_header *head, int fd)
+{
+ struct name_list *nl = name_lists;
+
+ while (nl) {
+ if (!hashcmp(nl->sha1, head->sha1))
+ break;
+ nl = nl->next;
+ }
+
+ if (nl)
+ return nl;
+
+ nl = xcalloc(1, sizeof(struct name_list) + head->name_size);
+ nl->len = head->name_size;
+ hashcpy(nl->sha1, head->sha1);
+
+ lseek(fd, head->size, SEEK_SET);
+ read_in_full(fd, nl->buf, head->name_size);
+
+ nl->next = name_lists;
+ name_lists = nl;
+
+ return nl;
+}
+
+static int get_cache_slice_header(int fd, unsigned char *cache_sha1, int len, struct rc_slice_header *head)
{
int t;
- memcpy(head, map, sizeof(struct rc_slice_header));
+ if (xread(fd, head, sizeof(struct rc_slice_header)) != sizeof(struct rc_slice_header))
+ return -1;
+
head->ofs_objects = ntohl(head->ofs_objects);
head->object_nr = ntohl(head->object_nr);
head->size = ntohl(head->size);
head->path_nr = ntohs(head->path_nr);
+ head->name_size = ntohl(head->name_size);
if (memcmp(head->signature, "REVCACHE", 8))
return -1;
@@ -731,10 +792,10 @@ static int get_cache_slice_header(unsigned char *cache_sha1, unsigned char *map,
if (hashcmp(head->sha1, cache_sha1))
return -3;
t = sizeof(struct rc_slice_header);
- if (t != head->ofs_objects || t >= len)
+ if (t != head->ofs_objects)
return -4;
-
- head->size = len;
+ if (head->size + head->name_size != len)
+ return -5;
return 0;
}
@@ -786,7 +847,7 @@ int traverse_cache_slice(struct rev_info *revs,
unsigned long *date_so_far, int *slop_so_far,
struct commit_list ***queue, struct commit_list **work)
{
- int fd = -1, retval = -3;
+ int fd = -1, t, retval;
struct stat fi;
struct rc_slice_header head;
struct rev_cache_info *rci;
@@ -802,26 +863,31 @@ int traverse_cache_slice(struct rev_info *revs,
/* load options */
rci = &revs->rev_cache_info;
add_to_pending = rci->add_to_pending;
+ add_names = rci->add_names;
memset(&head, 0, sizeof(struct rc_slice_header));
+# define ERROR(x) do { retval = (x); goto end; } while (0);
fd = open_cache_slice(cache_sha1, O_RDONLY);
if (fd == -1)
- goto end;
+ ERROR(-1);
if (fstat(fd, &fi) || fi.st_size < sizeof(struct rc_slice_header))
- goto end;
+ ERROR(-2);
+
+ if ((t = get_cache_slice_header(fd, cache_sha1, fi.st_size, &head)) < 0)
+ ERROR(-t);
+ if (add_names)
+ cur_name_list = get_cache_slice_name_list(&head, fd);
- map = xmmap(0, fi.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
+ map = xmmap(0, head.size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
if (map == MAP_FAILED)
- goto end;
- if (get_cache_slice_header(cache_sha1, map, fi.st_size, &head))
- goto end;
+ ERROR(-3);
retval = traverse_cache_slice_1(&head, map, revs, commit, date_so_far, slop_so_far, queue, work);
end:
if (map != MAP_FAILED)
- munmap(map, fi.st_size);
+ munmap(map, head.size);
if (fd != -1)
close(fd);
@@ -829,6 +895,7 @@ end:
if (retval)
mark_bad_slice(cache_sha1);
+# undef ERROR
return retval;
}
@@ -1113,23 +1180,110 @@ static unsigned long decode_size(unsigned char *str, int len)
return size;
}
+
+#define NL_HASH_TABLE_SIZE (0xffff + 1)
+#define NL_HASH_NUMBER (NL_HASH_TABLE_SIZE >> 3)
+
+struct name_list_hash {
+ int ind;
+ struct name_list_hash *next;
+};
+
+static struct name_list_hash **nl_hash_table;
+static unsigned char *nl_hashes;
+
+/* FNV-1a hash */
+static unsigned int hash_name(const char *name)
+{
+ unsigned int hash = 2166136261ul;
+ const char *p = name;
+
+ while (*p) {
+ hash ^= *p++;
+ hash *= 16777619ul;
+ }
+
+ return hash & 0xffff;
+}
+
+static int name_in_list(const char *name)
+{
+ unsigned int h = hash_name(name);
+ struct name_list_hash *entry = nl_hash_table[h];
+
+ while (entry && strcmp(acc_name_buffer->buf + entry->ind, name))
+ entry = entry->next;
+
+ if (entry)
+ return entry->ind;
+
+ /* add name to buffer and create hash reference */
+ entry = xcalloc(1, sizeof(struct name_list_hash));
+ entry->ind = acc_name_buffer->len;
+ strbuf_add(acc_name_buffer, name, strlen(name) + 1);
+
+ entry->next = nl_hash_table[h];
+ nl_hash_table[h] = entry;
+
+ nl_hashes[h / 8] |= h % 8;
+
+ return entry->ind;
+}
+
+static void init_name_list_hash(void)
+{
+ nl_hash_table = xcalloc(NL_HASH_TABLE_SIZE, sizeof(struct name_list_hash));
+ nl_hashes = xcalloc(NL_HASH_NUMBER, 1);
+}
+
+static void cleanup_name_list_hash(void)
+{
+ int i;
+
+ for (i = 0; i < NL_HASH_NUMBER; i++) {
+ int j, ind = nl_hashes[i];
+
+ if (!ind)
+ continue;
+
+ for (j = 0; j < 8; j++) {
+ struct name_list_hash **entryp;
+
+ if (!(ind & 1 << j))
+ continue;
+
+ entryp = &nl_hash_table[i * 8 + j];
+ while (*entryp) {
+ struct name_list_hash *t = (*entryp)->next;
+
+ free(*entryp);
+ *entryp = t;
+ }
+ }
+ } /* code overhang! */
+
+ free(nl_hashes);
+ free(nl_hash_table);
+}
+
static void add_object_entry(const unsigned char *sha1, struct rc_object_entry *entryp,
- struct strbuf *merge_str, struct strbuf *split_str)
+ struct strbuf *merge_str, struct strbuf *split_str, char *name, unsigned long size)
{
struct rc_object_entry entry;
- unsigned char size_str[7];
- unsigned long size;
+ unsigned char size_str[7], name_str[7];
enum object_type type;
void *data;
if (entryp)
sha1 = entryp->sha1;
- /* retrieve size data */
- data = read_sha1_file(sha1, &type, &size);
+ if (!size) {
+ /* retrieve size data */
+ data = read_sha1_file(sha1, &type, &size);
- if (data)
- free(data);
+ if (data)
+ free(data);
+ }
/* initialize! */
if (!entryp) {
@@ -1147,6 +1301,9 @@ static void add_object_entry(const unsigned char *sha1, struct rc_object_entry *
entryp->size_size = encode_size(size, size_str);
+ if (name)
+ entryp->name_size = encode_size(name_in_list(name), name_str);
+
/* write the muvabitch */
strbuf_add(acc_buffer, to_disked_rc_object_entry(entryp, 0), sizeof(struct rc_object_entry_ondisk));
@@ -1156,25 +1313,36 @@ static void add_object_entry(const unsigned char *sha1, struct rc_object_entry *
strbuf_add(acc_buffer, split_str->buf, split_str->len);
strbuf_add(acc_buffer, size_str, entryp->size_size);
+
+ if (name)
+ strbuf_add(acc_buffer, name_str, entryp->name_size);
}
/* returns non-zero to continue parsing, 0 to skip */
typedef int (*dump_tree_fn)(const unsigned char *, const char *, unsigned int); /* sha1, path, mode */
/* we need to walk the trees by hash, so unfortunately we can't use traverse_trees in tree-walk.c */
-static int dump_tree(struct tree *tree, dump_tree_fn fn)
+static int dump_tree(struct tree *tree, dump_tree_fn fn, char *base)
{
struct tree_desc desc;
struct name_entry entry;
struct tree *subtree;
- int r;
+ char concatpath[PATH_MAX];
+ int r, baselen;
if (parse_tree(tree))
return -1;
+ baselen = strlen(base);
+ strcpy(concatpath, base);
+
init_tree_desc(&desc, tree->buffer, tree->size);
while (tree_entry(&desc, &entry)) {
- switch (fn(entry.sha1, entry.path, entry.mode)) {
+ if (baselen + strlen(entry.path) + 1 >= PATH_MAX)
+ die("we have a problem: %s%s is too big for me to handle", base, entry.path);
+ strcpy(concatpath + baselen, entry.path);
+
+ switch (fn(entry.sha1, concatpath, entry.mode)) {
case 0 :
goto continue_loop;
default :
@@ -1186,7 +1354,8 @@ static int dump_tree(struct tree *tree, dump_tree_fn fn)
if (!subtree)
return -2;
- if ((r = dump_tree(subtree, fn)) < 0)
+ strcat(concatpath, "/");
+ if ((r = dump_tree(subtree, fn, concatpath)) < 0)
return r;
}
@@ -1200,6 +1369,9 @@ continue_loop:
static int dump_tree_callback(const unsigned char *sha1, const char *path, unsigned int mode)
{
strbuf_add(acc_buffer, sha1, 20);
+ strbuf_add(acc_buffer, (char *)&acc_name_buffer->len, sizeof(size_t));
+
+ strbuf_add(acc_name_buffer, path, strlen(path) + 1);
return 1;
}
@@ -1213,6 +1385,9 @@ static void tree_addremove(struct diff_options *options,
return;
strbuf_add(acc_buffer, sha1, 20);
+ strbuf_add(acc_buffer, (char *)&acc_name_buffer->len, sizeof(size_t));
+
+ strbuf_add(acc_name_buffer, concatpath, strlen(concatpath) + 1);
}
static void tree_change(struct diff_options *options,
@@ -1225,12 +1400,15 @@ static void tree_change(struct diff_options *options,
return;
strbuf_add(acc_buffer, new_sha1, 20);
+ strbuf_add(acc_buffer, (char *)&acc_name_buffer->len, sizeof(size_t));
+
+ strbuf_add(acc_name_buffer, concatpath, strlen(concatpath) + 1);
}
static int add_unique_objects(struct commit *commit)
{
struct commit_list *list;
- struct strbuf os, ost, *orig_buf;
+ struct strbuf os, ost, names, *orig_name_buf, *orig_buf;
struct diff_options opts;
int i, j, next;
char is_first = 1;
@@ -1238,13 +1416,17 @@ static int add_unique_objects(struct commit *commit)
/* ...no, calculate unique objects */
strbuf_init(&os, 0);
strbuf_init(&ost, 0);
+ strbuf_init(&names, 0);
orig_buf = acc_buffer;
+ orig_name_buf = acc_name_buffer;
+ acc_name_buffer = &names;
diff_setup(&opts);
DIFF_OPT_SET(&opts, RECURSIVE);
DIFF_OPT_SET(&opts, TREE_IN_RECURSIVE);
opts.change = tree_change;
opts.add_remove = tree_addremove;
+# define ENTRY_SIZE (20 + sizeof(size_t))
/* this is only called for non-ends (ie. all parents interesting) */
for (list = commit->parents; list; list = list->next) {
@@ -1255,20 +1437,20 @@ static int add_unique_objects(struct commit *commit)
strbuf_setlen(acc_buffer, 0);
diff_tree_sha1(list->item->tree->object.sha1, commit->tree->object.sha1, "", &opts);
- qsort(acc_buffer->buf, acc_buffer->len / 20, 20, (int (*)(const void *, const void *))hashcmp);
+ qsort(acc_buffer->buf, acc_buffer->len / ENTRY_SIZE, ENTRY_SIZE, (int (*)(const void *, const void *))hashcmp);
/* take intersection */
if (!is_first) {
- for (next = i = j = 0; i < os.len; i += 20) {
+ for (next = i = j = 0; i < os.len; i += ENTRY_SIZE) {
while (j < ost.len && hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)) < 0)
- j += 20;
+ j += ENTRY_SIZE;
if (j >= ost.len || hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)))
continue;
if (next != i)
- memcpy(os.buf + next, os.buf + i, 20);
- next += 20;
+ memcpy(os.buf + next, os.buf + i, ENTRY_SIZE);
+ next += ENTRY_SIZE;
}
if (next != i)
@@ -1280,29 +1462,37 @@ static int add_unique_objects(struct commit *commit)
/* no parents (!) */
if (is_first) {
acc_buffer = &os;
- dump_tree(commit->tree, dump_tree_callback);
+ dump_tree(commit->tree, dump_tree_callback, "");
}
/* the ordering of non-commit objects dosn't really matter, so we're not gonna bother */
acc_buffer = orig_buf;
- for (i = 0; i < os.len; i += 20)
- add_object_entry((unsigned char *)(os.buf + i), 0, 0, 0);
+ acc_name_buffer = orig_name_buf;
+ for (i = 0; i < os.len; i += ENTRY_SIZE)
+ add_object_entry((unsigned char *)(os.buf + i), 0, 0, 0, names.buf + *(size_t *)(os.buf + i + 20), 0);
/* last but not least, the main tree */
- add_object_entry(commit->tree->object.sha1, 0, 0, 0);
+ add_object_entry(commit->tree->object.sha1, 0, 0, 0, 0, 0);
- return i / 20 + 1;
+ strbuf_release(&ost);
+ strbuf_release(&os);
+ strbuf_release(&names);
+
+ return i / ENTRY_SIZE + 1;
+# undef ENTRY_SIZE
}
static int add_objects_verbatim_1(struct rev_cache_slice_map *mapping, int *index)
{
- unsigned char *map = mapping->map;
int i = *index, object_nr = 0;
+ unsigned char *map = mapping->map;
struct rc_object_entry *entry = RC_OBTAIN_OBJECT_ENTRY(map + i);
+ unsigned long size;
i += RC_ACTUAL_OBJECT_ENTRY_SIZE(entry);
while (i < mapping->size) {
- int pos = i;
+ char *name;
+ int name_index, pos = i;
entry = RC_OBTAIN_OBJECT_ENTRY(map + i);
i += RC_ACTUAL_OBJECT_ENTRY_SIZE(entry);
@@ -1312,7 +1502,15 @@ static int add_objects_verbatim_1(struct rev_cache_slice_map *mapping, int *inde
return object_nr;
}
- strbuf_add(acc_buffer, map + pos, i - pos);
+ name_index = decode_size(map + pos + RC_ENTRY_NAME_OFFSET(entry), entry->name_size);
+ if (name_index && name_index < mapping->name_size)
+ name = mapping->names + name_index;
+ else
+ name = 0;
+
+ size = decode_size(map + pos + RC_ENTRY_SIZE_OFFSET(entry), entry->size_size);
+
+ add_object_entry(0, entry, 0, 0, name, size);
object_nr++;
}
@@ -1397,6 +1595,7 @@ void init_rev_cache_info(struct rev_cache_info *rci)
rci->overwrite_all = 0;
rci->add_to_pending = 1;
+ rci->add_names = 1;
rci->ignore_size = 0;
}
@@ -1421,9 +1620,9 @@ int make_cache_slice(struct rev_cache_info *rci,
struct rc_slice_header head;
struct commit *commit;
unsigned char sha1[20];
- struct strbuf merge_paths, split_paths;
+ struct strbuf merge_paths, split_paths, namelist;
int object_nr, total_sz, fd;
- char file[PATH_MAX], *newfile;
+ char file[PATH_MAX], null, *newfile;
struct rev_cache_info *trci;
git_SHA_CTX ctx;
@@ -1438,7 +1637,13 @@ int make_cache_slice(struct rev_cache_info *rci,
strbuf_init(&endlist, 0);
strbuf_init(&merge_paths, 0);
strbuf_init(&split_paths, 0);
+ strbuf_init(&namelist, 0);
acc_buffer = &buffer;
+ acc_name_buffer = &namelist;
+
+ null = 0;
+ strbuf_add(&namelist, &null, 1);
+ init_name_list_hash();
if (!revs) {
revs = &therevs;
@@ -1469,6 +1674,7 @@ int make_cache_slice(struct rev_cache_info *rci,
trci = &revs->rev_cache_info;
init_rev_cache_info(trci);
trci->add_to_pending = 0;
+ trci->add_names = 0;
setup_revisions(0, 0, revs, 0);
if (prepare_revision_walk(revs))
@@ -1506,7 +1712,7 @@ int make_cache_slice(struct rev_cache_info *rci,
commit->indegree = 0;
- add_object_entry(0, &object, &merge_paths, &split_paths);
+ add_object_entry(0, &object, &merge_paths, &split_paths, 0, 0);
object_nr++;
if (rci->objects && !object.is_end) {
@@ -1532,10 +1738,16 @@ int make_cache_slice(struct rev_cache_info *rci,
total_sz += buffer.len;
}
+ /* write path name lookup list */
+ head.name_size = htonl(namelist.len);
+ write_in_full(fd, namelist.buf, namelist.len);
+
/* go ahead a free some stuff... */
strbuf_release(&buffer);
strbuf_release(&merge_paths);
strbuf_release(&split_paths);
+ strbuf_release(&namelist);
+ cleanup_name_list_hash();
if (path_sz)
free(paths);
while (path_track_alloc)
@@ -1993,6 +2205,7 @@ int fuse_cache_slices(struct rev_cache_info *rci, struct rev_info *revs)
for (i = idx_head.cache_nr - 1; i >= 0; i--) {
struct rev_cache_slice_map *map = rci->maps + i;
struct stat fi;
+ struct rc_slice_header head;
int fd;
if (!map->size)
@@ -2005,13 +2218,20 @@ int fuse_cache_slices(struct rev_cache_info *rci, struct rev_info *revs)
continue;
if (fi.st_size < sizeof(struct rc_slice_header))
continue;
+ if (get_cache_slice_header(fd, idx_caches + i * 20, fi.st_size, &head))
+ continue;
- map->map = xmmap(0, fi.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ map->map = xmmap(0, head.size, PROT_READ, MAP_PRIVATE, fd, 0);
if (map->map == MAP_FAILED)
continue;
+ lseek(fd, head.size, SEEK_SET);
+ map->names = xcalloc(head.name_size, 1);
+ read_in_full(fd, map->names, head.name_size);
+
close(fd);
- map->size = fi.st_size;
+ map->size = head.size;
+ map->name_size = head.name_size;
}
rci->make_index = 0;
@@ -2028,6 +2248,7 @@ int fuse_cache_slices(struct rev_cache_info *rci, struct rev_info *revs)
if (!map->size)
continue;
+ free(map->names);
munmap(map->map, map->size);
}
free(rci->maps);
@@ -2049,7 +2270,6 @@ static int verify_cache_slice(const char *slice_path, unsigned char *sha1)
{
struct rc_slice_header head;
int fd, len, retval = -1;
- unsigned char *map = MAP_FAILED;
struct stat fi;
len = strlen(slice_path);
@@ -2064,17 +2284,12 @@ static int verify_cache_slice(const char *slice_path, unsigned char *sha1)
if (fstat(fd, &fi) || fi.st_size < sizeof(head))
goto end;
- map = xmmap(0, sizeof(head), PROT_READ, MAP_PRIVATE, fd, 0);
- if (map == MAP_FAILED)
- goto end;
- if (get_cache_slice_header(sha1, map, fi.st_size, &head))
+ if (get_cache_slice_header(fd, sha1, fi.st_size, &head))
goto end;
retval = 0;
end:
- if (map != MAP_FAILED)
- munmap(map, sizeof(head));
if (fd > 0)
close(fd);
diff --git a/rev-cache.h b/rev-cache.h
index 14437d8..c88ceae 100644
--- a/rev-cache.h
+++ b/rev-cache.h
@@ -10,8 +10,14 @@
#define RC_OBTAIN_OBJECT_ENTRY(p) from_disked_rc_object_entry((struct rc_object_entry_ondisk *)(p), 0)
#define RC_OBTAIN_INDEX_ENTRY(p) from_disked_rc_index_entry((struct rc_index_entry_ondisk *)(p), 0)
-#define RC_ACTUAL_OBJECT_ENTRY_SIZE(e) (sizeof(struct rc_object_entry_ondisk) + RC_PATH_SIZE((e)->merge_nr + (e)->split_nr) + (e)->size_size)
-#define RC_ENTRY_SIZE_OFFSET(e) (RC_ACTUAL_OBJECT_ENTRY_SIZE(e) - (e)->size_size)
+#define RC_ACTUAL_OBJECT_ENTRY_SIZE(e) (\
+ sizeof(struct rc_object_entry_ondisk) + \
+ RC_PATH_SIZE((e)->merge_nr + (e)->split_nr) + \
+ (e)->size_size + \
+ (e)->name_size\
+)
+#define RC_ENTRY_SIZE_OFFSET(e) (RC_ACTUAL_OBJECT_ENTRY_SIZE(e) - (e)->name_size - (e)->size_size)
+#define RC_ENTRY_NAME_OFFSET(e) (RC_ACTUAL_OBJECT_ENTRY_SIZE(e) - (e)->name_size)
/* single index maps objects to cache files */
struct rc_index_header {
@@ -50,6 +56,8 @@ struct rc_slice_header {
uint32_t size;
unsigned char sha1[20];
+
+ uint32_t name_size;
};
struct rc_object_entry_ondisk {
@@ -76,7 +84,8 @@ struct rc_object_entry {
unsigned char merge_nr; /* : 7 */
unsigned char split_nr; /* : 7 */
unsigned size_size : 3;
- unsigned padding : 5;
+ unsigned name_size : 3;
+ unsigned padding : 2;
uint32_t date;
uint16_t path;
@@ -84,6 +93,7 @@ struct rc_object_entry {
/* merge paths */
/* split paths */
/* size */
+ /* name id */
};
struct rc_index_entry *from_disked_rc_index_entry(struct rc_index_entry_ondisk *src, struct rc_index_entry *dst);
diff --git a/revision.h b/revision.h
index cc5c259..c62e85b 100644
--- a/revision.h
+++ b/revision.h
@@ -26,6 +26,9 @@ struct rev_cache_slice_map {
unsigned char *map;
int size;
int last_index;
+
+ char *names;
+ int name_size;
};
struct rev_cache_info {
@@ -39,7 +42,8 @@ struct rev_cache_info {
unsigned overwrite_all : 1;
/* traversal flags */
- unsigned add_to_pending : 1;
+ unsigned add_to_pending : 1,
+ add_names : 1;
/* fuse options */
unsigned int ignore_size;
diff --git a/t/t6017-rev-cache-list.sh b/t/t6017-rev-cache-list.sh
index f2e34b1..3286560 100755
--- a/t/t6017-rev-cache-list.sh
+++ b/t/t6017-rev-cache-list.sh
@@ -4,8 +4,10 @@ test_description='git rev-cache tests'
. ./test-lib.sh
test_cmp_sorted() {
- grep -io "[a-f0-9]*" $1 | sort >.tmpfile1 &&
- grep -io "[a-f0-9]*" $2 | sort >.tmpfile2 &&
+# note that we're tip-toeing around the corner case of two objects/names
+# for the same SHA-1 => discrepencies between cached and non-cached walks
+ sort $1 >.tmpfile1 &&
+ sort $2 >.tmpfile2 &&
test_cmp .tmpfile1 .tmpfile2
}
@@ -15,6 +17,8 @@ test_cmp_sorted() {
# reuse
test_expect_success 'init repo' '
echo bla >file &&
+ mkdir amaindir &&
+ echo watskeburt >amaindir/file &&
git add . &&
git commit -m "bla" &&
--
tg: (716470e..) t/revcache/names (depends on: t/revcache/docs)
^ permalink raw reply related
* Re: [PATCH 4/6 (v4)] administrative functions for rev-cache, start of integration into git
From: Nick Edelen @ 2009-09-07 14:10 UTC (permalink / raw)
To: Nick Edelen, Junio C Hamano, Nicolas Pitre, Johannes Schindelin,
Sam Vilain
In-Reply-To: <op.uyzwx8lltdk399@sirnot>
This patch, fourth, contains miscellaneous (maintenance) features:
- support for cache slice fusion, index regeneration and object size caching
- non-commit object generation refactored
- porcelain updated to support feature additions
The beginnings of integration into git are present in this patch, mainly
centered on caching object size; the object generation is refactored to more
elegantly exploit this. Fusion allows smaller (incremental) slices to be
coagulated into a larger slice, reducing overhead, while index regeneration
enables repair or cleaning of the cache index.
Note that tests for these features are included in the following patch, as they
take advantage of the rev-cache's integration into the revision walker.
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
builtin-gc.c | 9 +
builtin-rev-cache.c | 77 ++++++-
rev-cache.c | 717 +++++++++++++++++++++++++++++++++++++++++++++------
rev-cache.h | 9 +-
revision.h | 16 +-
5 files changed, 749 insertions(+), 79 deletions(-)
diff --git a/builtin-gc.c b/builtin-gc.c
index 7d3e9cc..c92511a 100644
--- a/builtin-gc.c
+++ b/builtin-gc.c
@@ -22,6 +22,7 @@ static const char * const builtin_gc_usage[] = {
NULL
};
+static char do_rev_cache = 0;
static int pack_refs = 1;
static int aggressive_window = 250;
static int gc_auto_threshold = 6700;
@@ -34,9 +35,14 @@ static const char *argv_reflog[] = {"reflog", "expire", "--all", NULL};
static const char *argv_repack[MAX_ADD] = {"repack", "-d", "-l", NULL};
static const char *argv_prune[] = {"prune", "--expire", NULL, NULL};
static const char *argv_rerere[] = {"rerere", "gc", NULL};
+static const char *argv_rev_cache[] = {"rev-cache", "fuse", "--all", "--ignore-size", NULL};
static int gc_config(const char *var, const char *value, void *cb)
{
+ if (!strcmp(var, "gc.revcache")) {
+ do_rev_cache = 1;
+ return 0;
+ }
if (!strcmp(var, "gc.packrefs")) {
if (value && !strcmp(value, "notbare"))
pack_refs = -1;
@@ -244,6 +250,9 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
if (run_command_v_opt(argv_rerere, RUN_GIT_CMD))
return error(FAILED_RUN, argv_rerere[0]);
+ if (do_rev_cache && run_command_v_opt(argv_rev_cache, RUN_GIT_CMD))
+ return error(FAILED_RUN, argv_rev_cache[0]);
+
if (auto_gc && too_many_loose_objects())
warning("There are too many unreachable loose objects; "
"run 'git prune' to remove them.");
diff --git a/builtin-rev-cache.c b/builtin-rev-cache.c
index 6eb7065..b894c54 100644
--- a/builtin-rev-cache.c
+++ b/builtin-rev-cache.c
@@ -5,6 +5,8 @@
#include "revision.h"
#include "rev-cache.h"
+unsigned long default_ignore_size = 50 * 1024 * 1024; /* 50mb */
+
/* porcelain for rev-cache.c */
static int handle_add(int argc, const char *argv[]) /* args beyond this command */
{
@@ -24,7 +26,7 @@ static int handle_add(int argc, const char *argv[]) /* args beyond this command
if (!strcmp(argv[i], "--stdin"))
dostdin = 1;
else if (!strcmp(argv[i], "--fresh") || !strcmp(argv[i], "--incremental"))
- starts_from_slices(&revs, UNINTERESTING);
+ starts_from_slices(&revs, UNINTERESTING, 0, 0);
else if (!strcmp(argv[i], "--not"))
flags ^= UNINTERESTING;
else if (!strcmp(argv[i], "--legs"))
@@ -150,6 +152,57 @@ static int handle_walk(int argc, const char *argv[])
return 0;
}
+static int handle_fuse(int argc, const char *argv[])
+{
+ struct rev_info revs;
+ struct rev_cache_info rci;
+ const char *args[5];
+ int i, argn = 0;
+ char add_all = 0;
+
+ init_revisions(&revs, 0);
+ init_rev_cache_info(&rci);
+ args[argn++] = "rev-list";
+
+ for (i = 0; i < argc; i++) {
+ if (!strcmp(argv[i], "--all")) {
+ args[argn++] = "--all";
+ setup_revisions(argn, args, &revs, 0);
+ add_all = 1;
+ } else if (!strcmp(argv[i], "--no-objects"))
+ rci.objects = 0;
+ else if (!strncmp(argv[i], "--ignore-size", 13)) {
+ unsigned long sz;
+
+ if (argv[i][13] == '=')
+ git_parse_ulong(argv[i] + 14, &sz);
+ else
+ sz = default_ignore_size;
+
+ rci.ignore_size = sz;
+ } else
+ continue;
+ }
+
+ if (!add_all)
+ starts_from_slices(&revs, 0, 0, 0);
+
+ return fuse_cache_slices(&rci, &revs);
+}
+
+static int handle_index(int argc, const char *argv[])
+{
+ return regenerate_cache_index(0);
+}
+
+static int handle_alt(int argc, const char *argv[])
+{
+ if (argc < 1)
+ return -1;
+
+ return make_cache_slice_pointer(0, argv[0]);
+}
+
static int handle_help(void)
{
char *usage = "\
@@ -180,12 +233,28 @@ commands:\n\
return 0;
}
+static int rev_cache_config(const char *k, const char *v, void *cb)
+{
+ /* this could potentially be related to pack.windowmemory, but we want a max around 50mb,
+ * and .windowmemory is often >700mb, with *large* variations */
+ if (!strcmp(k, "revcache.ignoresize")) {
+ int t;
+
+ t = git_config_ulong(k, v);
+ if (t)
+ default_ignore_size = t;
+ }
+
+ return 0;
+}
+
int cmd_rev_cache(int argc, const char *argv[], const char *prefix)
{
const char *arg;
int r;
git_config(git_default_config, NULL);
+ git_config(rev_cache_config, NULL);
if (argc > 1)
arg = argv[1];
@@ -196,8 +265,14 @@ int cmd_rev_cache(int argc, const char *argv[], const char *prefix)
argv += 2;
if (!strcmp(arg, "add"))
r = handle_add(argc, argv);
+ else if (!strcmp(arg, "fuse"))
+ r = handle_fuse(argc, argv);
else if (!strcmp(arg, "walk"))
r = handle_walk(argc, argv);
+ else if (!strcmp(arg, "index"))
+ r = handle_index(argc, argv);
+ else if (!strcmp(arg, "alt"))
+ r = handle_alt(argc, argv);
else
return handle_help();
diff --git a/rev-cache.c b/rev-cache.c
index 8af8c85..8ca97d3 100644
--- a/rev-cache.c
+++ b/rev-cache.c
@@ -9,6 +9,13 @@
#include "revision.h"
#include "rev-cache.h"
#include "run-command.h"
+#include "string-list.h"
+
+struct cache_slice_pointer {
+ char signature[8]; /* REVCOPTR */
+ char version;
+ char path[PATH_MAX + 1];
+};
/* list resembles pack index format */
static uint32_t fanout[0xff + 2];
@@ -259,27 +266,45 @@ unsigned char *get_cache_slice(struct commit *commit)
/* traversal */
-static void handle_noncommit(struct rev_info *revs, unsigned char *ptr, struct rc_object_entry *entry)
+static unsigned long decode_size(unsigned char *str, int len);
+
+static void handle_noncommit(struct rev_info *revs, struct commit *commit, unsigned char *ptr, struct rc_object_entry *entry)
{
- struct object *obj = 0;
+ struct blob *blob;
+ struct tree *tree;
+ struct object *obj;
+ unsigned long size;
+ size = decode_size(ptr + RC_ENTRY_SIZE_OFFSET(entry), entry->size_size);
switch (entry->type) {
case OBJ_TREE :
- if (revs->tree_objects)
- obj = (struct object *)lookup_tree(entry->sha1);
+ if (!revs->tree_objects)
+ return;
+
+ tree = lookup_tree(entry->sha1);
+ if (!tree)
+ return;
+
+ tree->size = size;
+ commit->tree = tree;
+ obj = (struct object *)tree;
break;
+
case OBJ_BLOB :
- if (revs->blob_objects)
- obj = (struct object *)lookup_blob(entry->sha1);
- break;
- case OBJ_TAG :
- if (revs->tag_objects)
- obj = (struct object *)lookup_tag(entry->sha1);
+ if (!revs->blob_objects)
+ return;
+
+ blob = lookup_blob(entry->sha1);
+ if (!blob)
+ return;
+
+ obj = (struct object *)blob;
break;
- }
- if (!obj)
+ default :
+ /* tag objects aren't really supposed to be here */
return;
+ }
obj->flags |= FACE_VALUE;
add_pending_object(revs, obj, "");
@@ -375,7 +400,7 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
/* add extra objects if necessary */
if (entry->type != OBJ_COMMIT) {
if (consume_children)
- handle_noncommit(revs, map + index, entry);
+ handle_noncommit(revs, co, map + index, entry);
continue;
} else
@@ -409,6 +434,8 @@ static int traverse_cache_slice_1(struct rc_slice_header *head, unsigned char *m
if (last_objects[path]) {
parse_commit(last_objects[path]);
+ /* we needn't worry about the unique field; that will be valid as
+ * long as we're not a end entry */
last_objects[path]->object.flags &= ~FACE_VALUE;
last_objects[path] = 0;
}
@@ -541,6 +568,48 @@ static int get_cache_slice_header(unsigned char *cache_sha1, unsigned char *map,
return 0;
}
+int open_cache_slice(unsigned char *sha1, int flags)
+{
+ int fd;
+ char signature[8];
+
+ fd = open(git_path("rev-cache/%s", sha1_to_hex(sha1)), flags);
+ if (fd <= 0)
+ goto end;
+
+ if (read(fd, signature, 8) != 8)
+ goto end;
+
+ /* a normal revision slice */
+ if (!memcmp(signature, "REVCACHE", 8)) {
+ lseek(fd, 0, SEEK_SET);
+ return fd;
+ }
+
+ /* slice pointer */
+ if (!memcmp(signature, "REVCOPTR", 8)) {
+ struct cache_slice_pointer ptr;
+
+ lseek(fd, 0, SEEK_SET);
+ if (read_in_full(fd, &ptr, sizeof(ptr)) != sizeof(ptr))
+ goto end;
+
+ if (ptr.version != SUPPORTED_REVCOPTR_VERSION)
+ goto end;
+
+ close(fd);
+ fd = open(ptr.path, flags);
+
+ return fd;
+ }
+
+end:
+ if (fd > 0)
+ close(fd);
+
+ return -1;
+}
+
int traverse_cache_slice(struct rev_info *revs,
unsigned char *cache_sha1, struct commit *commit,
unsigned long *date_so_far, int *slop_so_far,
@@ -564,7 +633,7 @@ int traverse_cache_slice(struct rev_info *revs,
memset(&head, 0, sizeof(struct rc_slice_header));
- fd = open(git_path("rev-cache/%s", sha1_to_hex(cache_sha1)), O_RDONLY);
+ fd = open_cache_slice(cache_sha1, O_RDONLY);
if (fd == -1)
goto end;
if (fstat(fd, &fi) || fi.st_size < sizeof(struct rc_slice_header))
@@ -591,6 +660,68 @@ end:
/* generation */
+static int is_endpoint(struct commit *commit)
+{
+ struct commit_list *list = commit->parents;
+
+ while (list) {
+ if (!(list->item->object.flags & UNINTERESTING))
+ return 0;
+
+ list = list->next;
+ }
+
+ return 1;
+}
+
+/* ensures branch is self-contained: parents are either all interesting or all uninteresting */
+static void make_legs(struct rev_info *revs)
+{
+ struct commit_list *list, **plist;
+ int total = 0;
+
+ /* attach plist to end of commits list */
+ list = revs->commits;
+ while (list && list->next)
+ list = list->next;
+
+ if (list)
+ plist = &list->next;
+ else
+ return;
+
+ /* duplicates don't matter, as get_revision() ignores them */
+ for (list = revs->commits; list; list = list->next) {
+ struct commit *item = list->item;
+ struct commit_list *parents = item->parents;
+
+ if (item->object.flags & UNINTERESTING)
+ continue;
+ if (is_endpoint(item))
+ continue;
+
+ while (parents) {
+ struct commit *p = parents->item;
+ parents = parents->next;
+
+ if (!(p->object.flags & UNINTERESTING))
+ continue;
+
+ p->object.flags &= ~UNINTERESTING;
+ parse_commit(p);
+ plist = &commit_list_insert(p, plist)->next;
+
+ if (!(p->object.flags & SEEN))
+ total++;
+ }
+ }
+
+ if (total)
+ sort_in_topological_order(&revs->commits, 1);
+
+}
+
+
struct path_track {
struct commit *commit;
int path; /* for keeping track of children */
@@ -779,31 +910,76 @@ static void handle_paths(struct commit *commit, struct rc_object_entry *object,
}
-static void add_object_entry(const unsigned char *sha1, int type, struct rc_object_entry *nothisone,
+static int encode_size(unsigned long size, unsigned char *out)
+{
+ int len = 0;
+
+ while (size) {
+ *out++ = (unsigned char)(size & 0xff);
+ size >>= 8;
+ len++;
+ }
+
+ return len;
+}
+
+static unsigned long decode_size(unsigned char *str, int len)
+{
+ unsigned long size = 0;
+ int shift = 0;
+
+ while (len--) {
+ size |= (unsigned long)*str << shift;
+ shift += 8;
+ str++;
+ }
+
+ return size;
+}
+
+static void add_object_entry(const unsigned char *sha1, struct rc_object_entry *entryp,
struct strbuf *merge_str, struct strbuf *split_str)
{
- struct rc_object_entry object;
+ struct rc_object_entry entry;
+ unsigned char size_str[7];
+ unsigned long size;
+ enum object_type type;
+ void *data;
- if (!nothisone) {
- memset(&object, 0, sizeof(object));
- object.sha1 = (unsigned char *)sha1;
- object.type = type;
+ if (entryp)
+ sha1 = entryp->sha1;
+
+ /* retrieve size data */
+ data = read_sha1_file(sha1, &type, &size);
+
+ if (data)
+ free(data);
+
+ /* initialize! */
+ if (!entryp) {
+ memset(&entry, 0, sizeof(entry));
+ entry.sha1 = (unsigned char *)sha1;
+ entry.type = type;
if (merge_str)
- object.merge_nr = merge_str->len / sizeof(uint16_t);
+ entry.merge_nr = merge_str->len / sizeof(uint16_t);
if (split_str)
- object.split_nr = split_str->len / sizeof(uint16_t);
+ entry.split_nr = split_str->len / sizeof(uint16_t);
- nothisone = &object;
+ entryp = &entry;
}
- strbuf_add(acc_buffer, to_disked_rc_object_entry(nothisone, 0), sizeof(struct rc_object_entry_ondisk));
+ entryp->size_size = encode_size(size, size_str);
- if (merge_str && merge_str->len)
+ /* write the muvabitch */
+ strbuf_add(acc_buffer, to_disked_rc_object_entry(entryp, 0), sizeof(struct rc_object_entry_ondisk));
+
+ if (merge_str)
strbuf_add(acc_buffer, merge_str->buf, merge_str->len);
- if (split_str && split_str->len)
+ if (split_str)
strbuf_add(acc_buffer, split_str->buf, split_str->len);
+ strbuf_add(acc_buffer, size_str, entryp->size_size);
}
/* returns non-zero to continue parsing, 0 to skip */
@@ -847,12 +1023,7 @@ continue_loop:
static int dump_tree_callback(const unsigned char *sha1, const char *path, unsigned int mode)
{
- unsigned char data[21];
-
- hashcpy(data, sha1);
- data[20] = !!S_ISDIR(mode);
-
- strbuf_add(acc_buffer, data, 21);
+ strbuf_add(acc_buffer, sha1, 20);
return 1;
}
@@ -862,15 +1033,10 @@ static void tree_addremove(struct diff_options *options,
const unsigned char *sha1,
const char *concatpath)
{
- unsigned char data[21];
-
if (whatnow != '+')
return;
- hashcpy(data, sha1);
- data[20] = !!S_ISDIR(mode);
-
- strbuf_add(acc_buffer, data, 21);
+ strbuf_add(acc_buffer, sha1, 20);
}
static void tree_change(struct diff_options *options,
@@ -879,26 +1045,10 @@ static void tree_change(struct diff_options *options,
const unsigned char *new_sha1,
const char *concatpath)
{
- unsigned char data[21];
-
if (!hashcmp(old_sha1, new_sha1))
return;
- hashcpy(data, new_sha1);
- data[20] = !!S_ISDIR(new_mode);
-
- strbuf_add(acc_buffer, data, 21);
-}
-
-static int sort_type_hash(const void *a, const void *b)
-{
- const unsigned char *sa = (const unsigned char *)a,
- *sb = (const unsigned char *)b;
-
- if (sa[20] == sb[20])
- return hashcmp(sa, sb);
-
- return sa[20] > sb[20] ? -1 : 1;
+ strbuf_add(acc_buffer, new_sha1, 20);
}
static int add_unique_objects(struct commit *commit)
@@ -909,6 +1059,7 @@ static int add_unique_objects(struct commit *commit)
int i, j, next;
char is_first = 1;
+ /* ...no, calculate unique objects */
strbuf_init(&os, 0);
strbuf_init(&ost, 0);
orig_buf = acc_buffer;
@@ -928,20 +1079,20 @@ static int add_unique_objects(struct commit *commit)
strbuf_setlen(acc_buffer, 0);
diff_tree_sha1(list->item->tree->object.sha1, commit->tree->object.sha1, "", &opts);
- qsort(acc_buffer->buf, acc_buffer->len / 21, 21, (int (*)(const void *, const void *))hashcmp);
+ qsort(acc_buffer->buf, acc_buffer->len / 20, 20, (int (*)(const void *, const void *))hashcmp);
/* take intersection */
if (!is_first) {
- for (next = i = j = 0; i < os.len; i += 21) {
+ for (next = i = j = 0; i < os.len; i += 20) {
while (j < ost.len && hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)) < 0)
- j += 21;
+ j += 20;
if (j >= ost.len || hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)))
continue;
if (next != i)
- memcpy(os.buf + next, os.buf + i, 21);
- next += 21;
+ memcpy(os.buf + next, os.buf + i, 20);
+ next += 20;
}
if (next != i)
@@ -950,25 +1101,102 @@ static int add_unique_objects(struct commit *commit)
is_first = 0;
}
+ /* no parents (!) */
if (is_first) {
acc_buffer = &os;
dump_tree(commit->tree, dump_tree_callback);
}
- if (os.len)
- qsort(os.buf, os.len / 21, 21, sort_type_hash);
-
+ /* the ordering of non-commit objects dosn't really matter, so we're not gonna bother */
acc_buffer = orig_buf;
- for (i = 0; i < os.len; i += 21)
- add_object_entry((unsigned char *)(os.buf + i), os.buf[i + 20] ? OBJ_TREE : OBJ_BLOB, 0, 0, 0);
+ for (i = 0; i < os.len; i += 20)
+ add_object_entry((unsigned char *)(os.buf + i), 0, 0, 0);
/* last but not least, the main tree */
- add_object_entry(commit->tree->object.sha1, OBJ_TREE, 0, 0, 0);
+ add_object_entry(commit->tree->object.sha1, 0, 0, 0);
+
+ return i / 20 + 1;
+}
+
+static int add_objects_verbatim_1(struct rev_cache_slice_map *mapping, int *index)
+{
+ unsigned char *map = mapping->map;
+ int i = *index, object_nr = 0;
+ struct rc_object_entry *entry = RC_OBTAIN_OBJECT_ENTRY(map + i);
+
+ i += RC_ACTUAL_OBJECT_ENTRY_SIZE(entry);
+ while (i < mapping->size) {
+ int pos = i;
- strbuf_release(&ost);
- strbuf_release(&os);
+ entry = RC_OBTAIN_OBJECT_ENTRY(map + i;
+ i += RC_ACTUAL_OBJECT_ENTRY_SIZE(entry);
+
+ if (entry->type == OBJ_COMMIT) {
+ *index = pos;
+ return object_nr;
+ }
- return i / 21 + 1;
+ strbuf_add(acc_buffer, map + pos, i - pos);
+ object_nr++;
+ }
+
+ *index = 0;
+ return object_nr;
+}
+
+static int add_objects_verbatim(struct rev_cache_info *rci, struct commit *commit)
+{
+ struct rev_cache_slice_map *map;
+ char found = 0;
+ struct rc_index_entry *ie;
+ struct rc_object_entry *entry;
+ int object_nr, i;
+
+ if (!rci->maps)
+ return -1;
+
+ /* check if we can continue where we left off */
+ map = rci->last_map;
+ if (!map)
+ goto search_me;
+
+ i = map->last_index;
+ entry = RC_OBTAIN_OBJECT_ENTRY(map->map + i);
+ if (hashcmp(entry->sha1, commit->object.sha1))
+ goto search_me;
+
+ found = 1;
+
+search_me:
+ if (!found) {
+ ie = search_index(commit->object.sha1);
+ if (!ie || ie->cache_index >= idx_head.cache_nr)
+ return -2;
+
+ map = rci->maps + ie->cache_index;
+ if (!map->size)
+ return -3;
+
+ i = ie->pos;
+ entry = RC_OBTAIN_OBJECT_ENTRY(map->map + i);
+ if (entry->type != OBJ_COMMIT || hashcmp(entry->sha1, commit->object.sha1))
+ return -4;
+ }
+
+ /* can't handle end commits */
+ if (entry->is_end)
+ return -5;
+
+ object_nr = add_objects_verbatim_1(map, &i);
+
+ /* remember this */
+ if (i) {
+ rci->last_map = map;
+ map->last_index = i;
+ } else
+ rci->last_map = 0;
+
+ return object_nr;
}
static void init_revcache_directory(void)
@@ -983,9 +1211,14 @@ static void init_revcache_directory(void)
void init_rev_cache_info(struct rev_cache_info *rci)
{
+ memset(rci, 0, sizeof(struct rev_cache_info));
+
rci->objects = 1;
rci->legs = 0;
rci->make_index = 1;
+ rci->fuse_me = 0;
+
+ rci->overwrite_all = 0;
rci->add_to_pending = 1;
@@ -1065,9 +1298,13 @@ int make_cache_slice(struct rev_cache_info *rci,
if (prepare_revision_walk(revs))
die("died preparing revision walk");
+ if (rci->legs)
+ make_legs(revs);
+
object_nr = total_sz = 0;
while ((commit = get_revision(revs)) != 0) {
struct rc_object_entry object;
+ int t;
strbuf_setlen(&merge_paths, 0);
strbuf_setlen(&split_paths, 0);
@@ -1093,12 +1330,17 @@ int make_cache_slice(struct rev_cache_info *rci,
commit->indegree = 0;
- add_object_entry(0, 0, &object, &merge_paths, &split_paths);
+ add_object_entry(0, &object, &merge_paths, &split_paths);
object_nr++;
- /* add all unique children for this commit */
- if (rci->objects && !object.is_end)
- object_nr += add_unique_objects(commit);
+ if (rci->objects && !object.is_end) {
+ if (rci->fuse_me && (t = add_objects_verbatim(rci, commit)) >= 0)
+ /* yay! we did it! */
+ object_nr += t;
+ else
+ /* add all unique children for this commit */
+ object_nr += add_unique_objects(commit);
+ }
/* print every ~1MB or so */
if (buffer.len > 1000000) {
@@ -1223,6 +1465,8 @@ int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
unsigned char *map;
unsigned long max_date;
+ maybe_fill_with_defaults(rci);
+
if (!idx_map)
init_index();
@@ -1287,7 +1531,7 @@ int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
} else
disked_entry = search_index_1(object_entry->sha1);
- if (disked_entry && !object_entry->is_start)
+ if (disked_entry && !object_entry->is_start && !rci->overwrite_all)
continue;
else if (disked_entry) {
/* mmm, pointer arithmetic... tasty */ /* (entry - idx_map = offset, so cast is valid) */
@@ -1341,8 +1585,7 @@ int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
}
-/* add start-commits from each cache slice (uninterestingness will be propogated) */
-void starts_from_slices(struct rev_info *revs, unsigned int flags)
+void starts_from_slices(struct rev_info *revs, unsigned int flags, unsigned char *which, int n)
{
struct commit *commit;
int i;
@@ -1358,6 +1601,18 @@ void starts_from_slices(struct rev_info *revs, unsigned int flags)
if (!entry->is_start)
continue;
+ /* only include entries in 'which' slices */
+ if (n) {
+ int j;
+
+ for (j = 0; j < n; j++)
+ if (!hashcmp(idx_caches + entry->cache_index * 20, which + j * 20))
+ break;
+
+ if (j == n)
+ continue;
+ }
+
commit = lookup_commit(entry->sha1);
if (!commit)
continue;
@@ -1367,3 +1622,313 @@ void starts_from_slices(struct rev_info *revs, unsigned int flags)
}
}
+
+
+struct slice_fd_time {
+ unsigned char sha1[20];
+ int fd;
+ struct stat fi;
+};
+
+int slice_time_sort(const void *a, const void *b)
+{
+ unsigned long at, bt;
+
+ at = ((struct slice_fd_time *)a)->fi.st_ctime;
+ bt = ((struct slice_fd_time *)b)->fi.st_ctime;
+
+ if (at == bt)
+ return 0;
+
+ return at > bt ? 1 : -1;
+}
+
+int regenerate_cache_index(struct rev_cache_info *rci)
+{
+ DIR *dirh;
+ int i;
+ struct slice_fd_time info;
+ struct strbuf slices;
+
+ /* first remove old index if it exists */
+ unlink_or_warn(git_path("rev-cache/index"));
+
+ strbuf_init(&slices, 0);
+
+ dirh = opendir(git_path("rev-cache"));
+ if (dirh) {
+ struct dirent *de;
+ struct stat fi;
+ int fd;
+ unsigned char sha1[20];
+
+ while ((de = readdir(dirh))) {
+ if (de->d_name[0] == '.')
+ continue;
+
+ if (get_sha1_hex(de->d_name, sha1))
+ continue;
+
+ /* open with RDWR because of mmap call in make_cache_index() */
+ fd = open_cache_slice(sha1, O_RDONLY);
+ if (fd < 0 || fstat(fd, &fi)) {
+ warning("bad cache found [%s]; fuse recommended", de->d_name);
+ if (fd > 0)
+ close(fd);
+ continue;
+ }
+
+ hashcpy(info.sha1, sha1);
+ info.fd = fd;
+ memcpy(&info.fi, &fi, sizeof(struct stat));
+
+ strbuf_add(&slices, &info, sizeof(info));
+ }
+
+ closedir(dirh);
+ }
+
+ /* we want oldest first -> upon overlap, older slices are more likely to have a larger section,
+ * as of the overlapped commit */
+ qsort(slices.buf, slices.len / sizeof(info), sizeof(info), slice_time_sort);
+
+ for (i = 0; i < slices.len; i += sizeof(info)) {
+ struct slice_fd_time *infop = (struct slice_fd_time *)(slices.buf + i);
+ struct stat *fip = &infop->fi;
+ int fd = infop->fd;
+
+ if (make_cache_index(rci, infop->sha1, fd, fip->st_size) < 0)
+ die("error writing cache");
+
+ close(fd);
+ }
+
+ strbuf_release(&slices);
+
+ return 0;
+}
+
+static int add_slices_for_fuse(struct rev_cache_info *rci, struct string_list *files, struct strbuf *ignore)
+{
+ unsigned char sha1[20];
+ char base[PATH_MAX];
+ int baselen, i, slice_nr = 0;
+ struct stat fi;
+ DIR *dirh;
+ struct dirent *de;
+
+ strncpy(base, git_path("rev-cache"), sizeof(base));
+ baselen = strlen(base);
+
+ dirh = opendir(base);
+ if (!dirh)
+ return 0;
+
+ while ((de = readdir(dirh))) {
+ if (de->d_name[0] == '.')
+ continue;
+
+ base[baselen] = '/';
+ strncpy(base + baselen + 1, de->d_name, sizeof(base) - baselen - 1);
+
+ if (get_sha1_hex(de->d_name, sha1)) {
+ /* whatever it is, we don't need it... */
+ string_list_insert(base, files);
+ continue;
+ }
+
+ /* _theoretically_ it is possible a slice < ignore_size to map objects not covered by, yet reachable from,
+ * a slice >= ignore_size, meaning that we could potentially delete an 'unfused' slice; but if that
+ * ever *did* happen their cache structure'd be so fucked up they might as well refuse the entire thing.
+ * and at any rate the worst it'd do is make rev-list revert to standard walking in that (small) bit.
+ */
+ if (rci->ignore_size) {
+ if (stat(base, &fi))
+ warning("can't query file %s\n", base);
+ else if (fi.st_size >= rci->ignore_size) {
+ strbuf_add(ignore, sha1, 20);
+ continue;
+ }
+ } else {
+ /* check if a pointer */
+ struct cache_slice_pointer ptr;
+ int fd = open(base, O_RDONLY);
+
+ if (fd < 0)
+ goto dont_save;
+ if (sizeof(ptr) != read_in_full(fd, &ptr, sizeof(ptr)))
+ goto dont_save;
+
+ close(fd);
+ if (!strcmp(ptr.signature, "REVCOPTR")) {
+ strbuf_add(ignore, sha1, 20);
+ continue;
+ }
+ }
+
+dont_save:
+ for (i = idx_head.cache_nr - 1; i >= 0; i--) {
+ if (!hashcmp(idx_caches + i * 20, sha1))
+ break;
+ }
+
+ if (i >= 0)
+ rci->maps[i].size = 1;
+
+ string_list_insert(base, files);
+ slice_nr++;
+ }
+
+ closedir(dirh);
+
+ return slice_nr;
+}
+
+/* the most work-intensive attributes in the cache are the unique objects and size, both
+ * of which can be re-used. although path structures will be isomorphic, path generation is
+ * not particularly expensive, and at any rate we need to re-sort the commits */
+int fuse_cache_slices(struct rev_cache_info *rci, struct rev_info *revs)
+{
+ unsigned char cache_sha1[20];
+ struct string_list files = {0, 0, 0, 1}; /* dup */
+ struct strbuf ignore;
+ int i;
+
+ maybe_fill_with_defaults(rci);
+
+ if (!idx_map)
+ init_index();
+ if (!idx_map)
+ return -1;
+
+ strbuf_init(&ignore, 0);
+ rci->maps = xcalloc(idx_head.cache_nr, sizeof(struct rev_cache_slice_map));
+ if (add_slices_for_fuse(rci, &files, &ignore) <= 1) {
+ printf("nothing to fuse\n");
+ return 1;
+ }
+
+ if (ignore.len) {
+ starts_from_slices(revs, UNINTERESTING, (unsigned char *)ignore.buf, ignore.len / 20);
+ strbuf_release(&ignore);
+ }
+
+ /* initialize mappings */
+ for (i = idx_head.cache_nr - 1; i >= 0; i--) {
+ struct rev_cache_slice_map *map = rci->maps + i;
+ struct stat fi;
+ int fd;
+
+ if (!map->size)
+ continue;
+ map->size = 0;
+
+ /* pointers are never fused, so we can use open directly */
+ fd = open(git_path("rev-cache/%s", sha1_to_hex(idx_caches + i * 20)), O_RDONLY);
+ if (fd <= 0 || fstat(fd, &fi))
+ continue;
+ if (fi.st_size < sizeof(struct rc_slice_header))
+ continue;
+
+ map->map = xmmap(0, fi.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ if (map->map == MAP_FAILED)
+ continue;
+
+ close(fd);
+ map->size = fi.st_size;
+ }
+
+ rci->make_index = 0;
+ rci->fuse_me = 1;
+ if (make_cache_slice(rci, revs, 0, 0, cache_sha1) < 0)
+ die("can't make cache slice");
+
+ printf("%s\n", sha1_to_hex(cache_sha1));
+
+ /* clean up time! */
+ for (i = idx_head.cache_nr - 1; i >= 0; i--) {
+ struct rev_cache_slice_map *map = rci->maps + i;
+
+ if (!map->size)
+ continue;
+
+ munmap(map->map, map->size);
+ }
+ free(rci->maps);
+ cleanup_cache_slices();
+
+ for (i = 0; i < files.nr; i++) {
+ char *name = files.items[i].string;
+
+ fprintf(stderr, "removing %s\n", name);
+ unlink_or_warn(name);
+ }
+
+ string_list_clear(&files, 0);
+
+ return regenerate_cache_index(rci);
+}
+
+static int verify_cache_slice(const char *slice_path, unsigned char *sha1)
+{
+ struct rc_slice_header head;
+ int fd, len, retval = -1;
+ unsigned char *map = MAP_FAILED;
+ struct stat fi;
+
+ len = strlen(slice_path);
+ if (len < 40)
+ return -2;
+ if (get_sha1_hex(slice_path + len - 40, sha1))
+ return -3;
+
+ fd = open(slice_path, O_RDONLY);
+ if (fd == -1)
+ goto end;
+ if (fstat(fd, &fi) || fi.st_size < sizeof(head))
+ goto end;
+
+ map = xmmap(0, sizeof(head), PROT_READ, MAP_PRIVATE, fd, 0);
+ if (map == MAP_FAILED)
+ goto end;
+ if (get_cache_slice_header(sha1, map, fi.st_size, &head))
+ goto end;
+
+ retval = 0;
+
+end:
+ if (map != MAP_FAILED)
+ munmap(map, sizeof(head));
+ if (fd > 0)
+ close(fd);
+
+ return retval;
+}
+
+int make_cache_slice_pointer(struct rev_cache_info *rci, const char *slice_path)
+{
+ struct cache_slice_pointer ptr;
+ int fd;
+ unsigned char sha1[20];
+
+ maybe_fill_with_defaults(rci);
+ rci->overwrite_all = 1;
+
+ if (verify_cache_slice(slice_path, sha1) < 0)
+ return -1;
+
+ strcpy(ptr.signature, "REVCOPTR");
+ ptr.version = SUPPORTED_REVCOPTR_VERSION;
+ strcpy(ptr.path, make_nonrelative_path(slice_path));
+
+ fd = open(git_path("rev-cache/%s", sha1_to_hex(sha1)), O_RDWR | O_CREAT | O_TRUNC, 0666);
+ if (fd < 0)
+ return -2;
+
+ write_in_full(fd, &ptr, sizeof(ptr));
+ make_cache_index(rci, sha1, fd, sizeof(ptr));
+
+ close(fd);
+
+ return 0;
+}
diff --git a/rev-cache.h b/rev-cache.h
index a471fbf..14437d8 100644
--- a/rev-cache.h
+++ b/rev-cache.h
@@ -3,6 +3,7 @@
#define SUPPORTED_REVCACHE_VERSION 1
#define SUPPORTED_REVINDEX_VERSION 1
+#define SUPPORTED_REVCOPTR_VERSION 1
#define RC_PATH_SIZE(x) (sizeof(uint16_t) * (x))
@@ -10,6 +11,7 @@
#define RC_OBTAIN_INDEX_ENTRY(p) from_disked_rc_index_entry((struct rc_index_entry_ondisk *)(p), 0)
#define RC_ACTUAL_OBJECT_ENTRY_SIZE(e) (sizeof(struct rc_object_entry_ondisk) + RC_PATH_SIZE((e)->merge_nr + (e)->split_nr) + (e)->size_size)
+#define RC_ENTRY_SIZE_OFFSET(e) (RC_ACTUAL_OBJECT_ENTRY_SIZE(e) - (e)->size_size)
/* single index maps objects to cache files */
struct rc_index_header {
@@ -90,6 +92,7 @@ struct rc_object_entry *from_disked_rc_object_entry(struct rc_object_entry_ondis
struct rc_object_entry_ondisk *to_disked_rc_object_entry(struct rc_object_entry *src, struct rc_object_entry_ondisk *dst);
extern unsigned char *get_cache_slice(struct commit *commit);
+extern int open_cache_slice(unsigned char *sha1, int flags);
extern int traverse_cache_slice(struct rev_info *revs,
unsigned char *cache_sha1, struct commit *commit,
unsigned long *date_so_far, int *slop_so_far,
@@ -102,6 +105,10 @@ extern int make_cache_slice(struct rev_cache_info *rci,
extern int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
int fd, unsigned int size);
-extern void starts_from_slices(struct rev_info *revs, unsigned int flags);
+extern void starts_from_slices(struct rev_info *revs, unsigned int flags, unsigned char *which, int n);
+extern int fuse_cache_slices(struct rev_cache_info *rci, struct rev_info *revs);
+extern int regenerate_cache_index(struct rev_cache_info *rci);
+extern int make_cache_slice_pointer(struct rev_cache_info *rci, const char *slice_path);
#endif
+
diff --git a/revision.h b/revision.h
index 7db4b9e..cc5c259 100644
--- a/revision.h
+++ b/revision.h
@@ -22,17 +22,31 @@
struct rev_info;
struct log_info;
+struct rev_cache_slice_map {
+ unsigned char *map;
+ int size;
+ int last_index;
+};
+
struct rev_cache_info {
/* generation flags */
unsigned objects : 1,
legs : 1,
- make_index : 1;
+ make_index : 1,
+ fuse_me : 1;
+
+ /* index inclusion */
+ unsigned overwrite_all : 1;
/* traversal flags */
unsigned add_to_pending : 1;
/* fuse options */
unsigned int ignore_size;
+
+ /* reserved */
+ struct rev_cache_slice_map *maps,
+ *last_map;
};
struct rev_info {
--
tg: (e434cf0..) t/revcache/misc (depends on: t/revcache/objects)
^ permalink raw reply related
* Re: [PATCH 1/6 (v4)] man page and technical discussion for rev-cache
From: Nick Edelen @ 2009-09-07 14:10 UTC (permalink / raw)
To: Nick Edelen, Junio C Hamano, Nicolas Pitre, Johannes Schindelin,
Sam Vilain
In-Reply-To: <op.uyzwxpmbtdk399@sirnot>
Before any code is introduced the full documentation is put forth. This
provides a man page for the porcelain, and a technical doc in technical/. The
latter describes the API, and discusses rev-cache's design, file format and
mechanics.
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
Slight clean-up of man page.
Documentation/git-rev-cache.txt | 190 ++++++++++
Documentation/technical/rev-cache.txt | 634 +++++++++++++++++++++++++++++++++
2 files changed, 824 insertions(+), 0 deletions(-)
diff --git a/Documentation/git-rev-cache.txt b/Documentation/git-rev-cache.txt
new file mode 100644
index 0000000..5a713ad
--- /dev/null
+++ b/Documentation/git-rev-cache.txt
@@ -0,0 +1,190 @@
+git-rev-cache(1)
+================
+
+NAME
+----
+git-rev-cache - Add, walk and maintain revision cache slices
+
+SYNOPSIS
+--------
+'git-rev-cache' COMMAND [options] [<commit>...]
+
+DESCRIPTION
+-----------
+The revision cache ('rev-cache') provides a mechanism for significantly
+speeding up revision traversals. It does this by creating an efficient
+database (cache) of commits, their related objects and topological relations.
+Independant of packs and the object store, this database is composed of
+rev-cache "slices" -- each a different file storing a given segment of commit
+history. To map commits to their respective slices, a single index file is
+kept for the rev-cache.
+
+'git-rev-cache' provides a front-end for the rev-cache mechanism, intended for
+updating and maintaining rev-cache slices in the current repository. New cache
+slice files can be 'add'ed, to keep the cache up-to-date; individual slices can
+be traversed; smaller slices can be 'fuse'd into a larger slice; and the
+rev-cache index can be regenerated.
+
+COMMANDS
+--------
+
+add
+~~~
+Add revisions to the cache by creating a new cache slice. Reads a revision
+list from the command line, formatted as: `START START ... \--not END END ...`
+
+Options
+^^^^^^^
+
+\--all::
+ Include all refs in the new cache slice, like the \--all option in
+ 'rev-list'.
+
+\--fresh/\--incremental::
+ Exclude everything already in the revision cache, analogous to
+ \--incremental in 'pack-objects'.
+
+\--stdin::
+ Read newline-seperated revisions from the standard input. Use \--not
+ to exclude commits, as on the command line.
+
+\--legs::
+ Ensure newly-generated cache slice has no partial ends. This means that
+ no commit has partially cached parents, in that all its parents are
+ cached or none of them are. 99.9% of users can ignore this command.
++
+\--legs will cause 'rev-cache' to expand potential slice end-points (creating
+"legs") until this condition is met, simplifying the cache slice structure.
+'rev-cache' itself does not care if a slice has legs or not, but the condition
+may reduce the required complexity of other applications that might use the
+revision cache.
+
+\--no-objects::
+ Non-commit objects are normally included along with the commit with
+ which they were introduced. This is obviously very benificial, but can
+ take longer in cache slice generation. Using this option will disable
+ non-commit object caching.
++
+\--no-objects is mainly intended for debugging or development purposes, but may
+find use in special situations (e.g. common traversal of only commits).
+
+Output
+^^^^^^
+
+On `stderr` 'add' outputs general information about the generated slice,
+including the number of objects and paths, and the start/end commits (prefix S
+indicates start, E an end). Through `stdout` it emits only the SHA-1 of the
+slice.
+
+walk
+~~~~
+Analogous to a slice-oriented 'rev-list', 'walk' will traverse a region in a
+particular cache slice. Interesting and uninteresting (delimited, as with
+'rev-list', with \--not) are specified on the command line, and output is the
+same as vanilla 'rev-list'.
+
+Options
+^^^^^^^
+
+\--objects::
+ Like 'rev-list', 'walk' will normally only list commits. Use this
+ option to list non-commit objects as well, if they are present in the
+ cache slice.
+
+Output
+^^^^^^
+
+'walk' will simply dump the contents of the output commit list, work list, and
+pending object array. The headers are outputed on `stderr`, the object hashes
+and names on `stdout`.
+
+fuse
+~~~~
+Merge several cache slices into a single large slice, like 'repack' for
+'rev-cache'. On each invocation of 'add' a new file ("slice") is added to the
+revision cache directory, and after several additions the directory may become
+populated with many, relatively small slices. Numerous smaller slices will
+yield poorer performance than a one or two large ones, because of the overhead
+of loading new slices into memory.
+
+Running 'fuse' every once in a while will solve this problem by coalescing all
+the cache slices into one larger slice. For very large projects, using
+\--ignore-size is advisable to prevent overly large cache slices. This can be
+set to run on garbage collection; see 'Automation' for more info.
+
+Note that 'fuse' uses the internal revision walker, so the options used in
+fusion override those of the cache slices upon which it operates. For example,
+if some slices were generated with \--no-objects, yet 'fuse' was performed with
+non-commit objects, the resulting slice would still contain objects but would
+take longer to generate.
+
+Options
+^^^^^^^
+
+\--all::
+ Normally fuse will only include everything that's already in the
+ revision cache. \--all tells it to start walking from the branch
+ heads, effectively a `add --all --fresh; fuse`
+ (pseudo-revcache-command).
+
+\--no-objects::
+ As in 'add', this option disables inclusion of non-commit objects. If
+ some cache slices do contain such objects, the information will be lost.
+
+\--ignore-size[=N]::
+ Do not merge cache slices of size >=N (be aware that slices must be
+ mapped to memory). N can have a suffix of "k" or "m", denoting N as
+ kilobytes and megabytes, respectively. If N is not provided 'fuse'
+ will default to a size specified in `revcache.ignoresize`, or ~25MB if
+ the config var is not set.
+
+Output
+^^^^^^
+
+This command prints the SHA-1 of the new slice on `stdout`, and information
+about its work on `stderr` -- specifically which files it's removing.
+
+Automation
+^^^^^^^^^^
+
+Set the git configuration variable `gc.revcache` to run 'fuse' on garbage
+collection. The arguments passed are `fuse \--all \--ignore-size`; i.e. 'gc'
+will keep everything cached into size-regulated slices.
+
+index
+~~~~~
+Regenerate the revision cache index. If the rev-cache index file associating
+objects with cache slices gets corrupted, lost, or otherwise becomes unusable,
+'index' will quickly regenerate the file. It's most likely that this won't be
+needed in every day use, as it is targeted towards debugging and development.
+
+alt
+~~~
+Create a cache slice pointer to another slice, identified by its full path:
+`fuse path/to/other/slice`
+
+This command is useful if you have several repositories sharing a common
+history. Although space requirements for rev-cache are slim anyway, you can in
+this situation reduce it further by using slice pointers, pointing to relavant
+slices in other repositories. Note that only one level of redirection is
+allowed, and the slice pointer will break if the original slice is removed.
+'fuse' will not touch slice pointers.
+
+NOTES
+-----
+In certain circumstances there may be some inconsistencies with object names
+between cached and non-cached walks. Specifically, if two objects in a commit
+tree have the same content (= same SHA-1); or if objects of the same SHA-1 are
+introduced independantly in parallel branches.
+
+In the first case rev-cache will use the name of the youngest file, while
+vanilla rev-list will return the name of the entry first encountered in walking
+the tree. The latter case is a result of rev-cache's internal topological
+ordering: the difference is the same between sorted and unsorted revision walks.
+
+See 'Discussion' for the underlying reasons for the discrepencies.
+
+DISCUSSION
+----------
+For an explanation of the API and its inner workings, see
+link:technical/rev-cache.txt[technical info on rev-cache].
diff --git a/Documentation/technical/rev-cache.txt b/Documentation/technical/rev-cache.txt
new file mode 100644
index 0000000..91fce8b
--- /dev/null
+++ b/Documentation/technical/rev-cache.txt
@@ -0,0 +1,634 @@
+rev-cache
+=========
+
+The revision cache API ('rev-cache') provides a method for efficiently storing
+and accessing commit branch sections. Such branch slices are defined by a
+series of start/top (interesting) and end/bottom (uninteresting) commits. Each
+slice contains information on commits in topological order. Recorded with each
+commit is:
+
+* All intra-slice topological relations, encoded into path "channels" (see
+ 'Mechanics' for full explanation).
+* Object meta-data: type, SHA-1, size, date (for commits).
+* Objects introduced by that commit, not present in the its cached parents.
+
+In addition to the API, basic structures are exported for the possibility of
+direct access.
+
+The API
+-------
+You can find the function prototypes in `revision.h`.
+
+Data Structures
+~~~~~~~~~~~~~~~
+The `rev_cache_info` struct holds all the options and flags for the API.
+
+----
+struct rev_cache_info {
+ /* generation flags */
+ unsigned objects : 1,
+ legs : 1,
+ make_index : 1,
+ fuse_me : 1;
+
+ /* index inclusion */
+ unsigned overwrite_all : 1;
+
+ /* traversal flags */
+ unsigned add_to_pending : 1;
+
+ /* fuse options */
+ unsigned int ignore_size;
+
+ /* reserved */
+ struct rev_cache_slice_map *maps,
+ *last_map;
+};
+----
+
+The fields:
+
+`objects`::
+ Add non-commit objects to slice.
+
+`legs`::
+ Ensure end/bottom commits have no children.
+
+`make_index`::
+ Integrate newly-made slice into index.
+
+`fuse_me`::
+ This is specified if a fuse is occuring, and slices are to be reused.
+ This option requires `maps` and `last_maps` to be initialized.
+
+`overwrite_all`::
+ When a cache slice is added to the index, sometimes overlap occures
+ between it and other slices. Normally, original index entries are kept
+ unless the new entry represents a start commit (older entries are more
+ likely to lead to greater in-slice traversals). This options overrides
+ that, and updates all entries of the new slice.
+
+`add_to_pending`::
+ Append unique non-commit objects to the `pending` object list in the
+ passed `rev_info` instance.
+
+`add_names`::
+ Include non-commit object names in the pending object entries if
+ `add_to_pending` is set.
+
+`ignore_size`::
+ If non-zero, ignore slices with size greater or equal to this during
+fusion.
+
+`maps`/`last_map`::
+ An array of slice mappings, indexed by their id in the slice index
+ header, to be re-used with `fuse_me`. `last_map` points to the last
+ mapping used, and should be initialized to 0.
+
+Functions
+~~~~~~~~~
+
+init_rev_cache
+^^^^^^^^^^^^^^
+----
+void init_rev_cache_info(
+ struct rev_cache_info *rci OUT
+)
+----
+
+Initialize `rci` to default options.
+
+make_cache_slice
+^^^^^^^^^^^^^^^^
+----
+int make_cache_slice(
+ struct rev_cache_info *rci IN,
+ struct rev_info *revs IN,
+ struct commit_list **starts IN/OUT,
+ struct commit_list **ends IN/OUT,
+ unsigned char *cache_sha1 OUT
+)
+----
+
+Create a cache slice based on either `revs` (if non-NULL) *or* the `starts` and
+`ends` lists. The actual list of start and end commits of the slice may be
+different from the parameters, based on what defines the branch segment, and
+this actual list is passed back through `starts` and `ends`.
+
+The cache slice is identified via a SHA-1 generated from the actual start/end
+commit lists. `cache_sha1`, if non-NULL, can recieve the cache slice name.
+`rci` is used to specify generation options, but can be NULL if you want
+`make_cache_slice` to fall back on defaults. Returns 0 on success, non-zero on
+failure.
+
+make_cache_index
+^^^^^^^^^^^^^^^^
+----
+int make_cache_index(
+ struct rev_cache_info *rci IN,
+ unsigned char *cache_sha1 IN,
+ int fd IN,
+ unsigned int size IN
+)
+----
+
+Add a slice to the rev-cache index. `cache_sha1` is the identity hash of the
+cache slice; `fd` is a file descriptor of the cache slice opened with
+read/write privileges (the slice is not actually modified); `size` is the size
+of the cache slice. Although there are currently no options for index
+updating, `rci` is a placeholder in case of future options. Note that this
+function is normally called by `make_cache_slice`. Returns 0 on success,
+non-zero on failure.
+
+open_cache_slice
+^^^^^^^^^^^^^^^^
+----
+int open_cache_slice(
+ unsigned char *sha1 IN,
+ int flags IN
+)
+----
+
+Returns a file descriptor to a cache slice described by `sha1` hash, using
+`flags` as the access mode. This will follow cache slice pointers to one level
+of indirection.
+
+get_cache_slice
+^^^^^^^^^^^^^^^
+----
+unsigned char *get_cache_slice(
+ struct commit *commit IN
+)
+----
+
+Given a commit object `get_cache_slice` will search the revision cache index
+and return, if found, the cache slice SHA-1.
+
+traverse_cache_slice
+^^^^^^^^^^^^^^^^^^^^
+----
+int traverse_cache_slice(
+ struct rev_info *revs IN/OUT,
+ unsigned char *cache_sha1 IN,
+ struct commit *commit IN,
+ unsigned long *date_so_far IN/OUT,
+ int *slop_so_far IN/OUT,
+ struct commit_list ***queue OUT,
+ struct commit_list **work IN/OUT
+)
+----
+
+Traverse a specified cache slice. An explanation of the each field:
+
+`revs`::
+ The revision walk instance. `traverse_cache_slice` uses this for
+ general options (e.g. which objects are included) and slice traversal
+ options (in the `rev_cache_info` field). If the `add_to_pending`
+ option is specified, non-commit objects are appended to the `pending`
+ object list field.
+
+`cache_sha1`::
+ SHA-1 identifying the cache slice to use. This can be taken directly
+ from `get_cache_slice`.
+
+`commit`::
+ The current commit object in the revision walk, i.e. the commit which
+ inspired this slice traversal. Although theoretically redundant in
+ view of the `work` list, this simplifies interaction with normal
+ revision walks, which pop commits from `work` before analyzing them.
+
+`date_so_far`::
+ The date of the oldest encountered interesting commit. Passing NULL
+ will let `traverse_cache_slice` use defaults.
+
+`slop_so_far`::
+ The `slop` value, a la revision.c. This is a counter used to determine
+ when to stop traversing, based on how many extra uninteresting commits
+ should be encountered. NULL will enable defaults, as above.
+
+`queue`::
+ Refers to a pointer to the head of a FIFO commit list, recieving the
+ commits we've seen and added.
+
+`work`::
+ A date-ordered list of commits that have yet to be processed (i.e. seen
+ but not added). Commits from here present in the slice are removed
+ (and, obviously, used as starting places for traversal), and any end
+ commits encountered are inserted.
+
+starts_from_slices
+^^^^^^^^^^^^^^^^^^
+----
+void starts_from_slices(
+ struct rev_info *revs OUT,
+ unsigned int flags IN,
+ unsigned char *which IN,
+ int n IN
+)
+----
+
+Will mark start-commits in certain rev-cache slices with `flag`, and added them
+to the pending list of `revs`. If `n` is zero, `starts_from_slices` will use
+all slices. Otherwise `which` will specify an *unseperated* list of cache
+SHA-1s to use (20 bytes each), and `n` will contain the number of slices (i.e.
+20 * `n` = size of `which`).
+
+fuse_cache_slices
+^^^^^^^^^^^^^^^^^
+----
+int fuse_cache_slices(
+ struct rev_cache_info *rci IN,
+ struct rev_info *revs IN
+)
+----
+
+Generate a slice based on `revs`, replacing all encountered slices with one
+(larger) slice. The `ignore_size` field in `rci`, if non-zero, will dictate
+which cache slice sizes to ignore in both traversal and replacement.
+
+regenerate_cache_index
+^^^^^^^^^^^^^^^^^^^^^^
+----
+int regenerate_cache_index(
+ struct rev_cache_info *rci IN
+)
+----
+
+Remake the revision cache index, including all the slices. Currently no
+options in `rci` exist for index (re)generation, but some may develop in the
+future.
+
+to/from_disked_rc_object/index_entry
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+----
+struct rc_object/index_entry *from_disked_rc_object/index_entry(
+ struct rc_object/index_entry_ondisk *src IN,
+ struct rc_object/index_entry *dst OUT
+)
+
+struct rc_object/index_entry_ondisk *to_disked_rc_object/index_entry(
+ struct rc_object/index_entry *src IN,
+ struct rc_object/index_entry_ondisk *dst OUT
+)
+----
+
+Functions to convert between the internal and storage (`_ondisk`) versions of
+object and index entry structures. These are necessary for direct access to
+the cache slices. If NULL is provided for `dst` a statically allocated
+structure is used, and a pointer to the struct is returned. Otherwise the
+functions return `dst`.
+
+Example Usage
+-------------
+
+A few examples to demonstrate usage:
+
+.Creating a slice
+----
+/* pretend you're a porcelain for rev-cache reading from the command line */
+struct rev_info revs;
+struct rev_cache_info rci;
+
+init_revisions(&revs, 0);
+init_rci(&rci);
+
+flags = 0;
+for (i = 1; i < argc; i++) {
+ if (!strcmp(argv[i], "--not"))
+ flags ^= UNINTERESTING;
+ else if(!strcmp(argv[i], "--fresh"))
+ starts_from_slices(&revs, UNINTERESTING, 0, 0);
+ else
+ handle_revision_arg(argv[i], &revs, flags, 1);
+}
+
+/* we want to explicitly set certain options */
+rci.objects = 0;
+
+if (!make_cache_slice(&rci, &revs, 0, 0, cache_sha1))
+ printf("made slice! it's called %s\n", sha1_to_hex(cache_sha1));
+----
+
+.Traversing a slice
+----
+/* let's say you're walking the tree with a 'work' list of current heads and a
+ * FILO output list 'out' */
+out = 0;
+outp = &out;
+
+while (work) {
+ struct commit *commit = pop_commit(&work);
+ struct object *object = &commit->object;
+ unsigned char *cache_sha1;
+
+ if (cache_sha1 = get_cache_slice(object->sha1)) {
+ /* note that this will instatiate any topo-relations
+ * as it goes */
+ if (traverse_cache_slice(&revs, cache_sha1,
+ commit, 0, 0, /* use defaults */
+ &outp, &work) < 0)
+ die("I'm overreacting to a non-fatal cache error");
+ } else {
+ struct commit_list *parents = commit->parents;
+
+ while (parents) {
+ struct commit *p = parents->item;
+ struct object *po = &p->object;
+
+ parents = parents->next;
+ if (po->flags & UNINTERESTING)
+ continue;
+
+ if (object->flags & UNINTERESTING)
+ po->flags |= UNINTERESTING;
+ else if (po->flags & SEEN)
+ continue;
+
+ if (!po->parsed)
+ parse_commit(p);
+ insert_by_date(p, &work);
+ }
+
+ if (object->flags & (SEEN | UNINTERESTING) == 0)
+ outp = &commit_list_insert(commit, outp)->next;
+ object->flags |= SEEN;
+ }
+}
+----
+
+Some Internals
+--------------
+For more advanced usage, the slice and index file(s) may be accessed directly.
+Relavant structures are availabe in `rev-cache.h`.
+
+File Formats
+~~~~~~~~~~~~
+
+Cache Slices
+^^^^^^^^^^^^
+A slice has a basic fixed-size header, followed by a certain number of object
+entries, then a NULL-seperated list of object names. Commits are sorted in
+topo-order, and each commit entry is followed by the objects added in that
+commit.
+
+----
+ -- +--------------------------------+
+header | object number, etc... |
+ -- +--------------------------------+
+commit | commit info |
+entry | path data |
+ +--------------------------------+
+ | tree/blob info |
+ +--------------------------------+
+ | tree/blob info |
+ +--------------------------------+
+ | ... |
+ -- +--------------------------------+
+commit | commit info |
+entry | path data |
+ +--------------------------------+
+ | tree/blob info |
+ +--------------------------------+
+ | ... |
+ -- +--------------------------------+
+... ...
+ -- +--------------------------------+
+name list | \0some_file_name\0 |
+(note +--------------------------------+
+preceeding | another_file\0 |
+null) ... |
+ +--------------------------------+
+----
+
+Here is the header:
+
+----
+struct rc_cache_slice_header {
+ char signature[8]; /* REVCACHE */
+ unsigned char version;
+ uint32_t ofs_objects;
+
+ uint32_t object_nr;
+ uint16_t path_nr;
+ uint32_t size;
+
+ unsigned char sha1[20];
+
+ uint32_t names_size;
+};
+----
+
+Explanations:
+
+`signature`::
+ The identifying signature of cache slice file. Always "REVCACHE".
+`version`::
+ The version number, currently 1.
+`ofs_objects`::
+ The byte offset at which the commit/object listing starts. Always
+ present at the 10th byte, regardless of file version.
+`object_nr`::
+ The total number of objects (commit + non-commit objects) present in
+ the slice.
+`path_nr`::
+ The total number of paths/channels used in encoding the topological
+ data. Note that paths are reused (see 'Mechanics'), so there will
+ never be more than a few hundred paths (if that) used.
+`size`::
+ The size of the slice *excluding* the name list. In other words, the
+ size of the portion mapped to memory.
+`sha1`::
+ The cache slice SHA-1.
+`names_size`::
+ The size of the name list. `size` + `names_size` = size of slice
+
+Revision Cache Index
+^^^^^^^^^^^^^^^^^^^^
+The index is a single file that associates SHA-1s with cache slices and file
+positions. It is somewhat similar to pack-file indexes, containing a fanout
+table and a list of index entries sorted by hash.
+
+----
+ -- +--------------------------------+
+header | object #, cache #, etc. |
+ -- +--------------------------------+
+sha1s of | SHA-1 |
+slices | ... |
+ -- +--------------------------------+
+fanout | fanout[0x00] |
+table ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ | fanout[0xff] |
+ -- +--------------------------------+
+index | SHA-1 of object |
+entries | index of cache slice SHA-1 |
+ | position in cache slice |
+ +--------------------------------+
+ | |
+ ...
+ +--------------------------------+
+----
+
+The header:
+
+----
+struct rc_index_header {
+ char signature[8]; /* REVINDEX */
+ unsigned char version;
+ uint32_t ofs_objects;
+
+ uint32_t object_nr;
+ unsigned char cache_nr;
+
+ uint32_t max_date;
+};
+----
+
+Explanations:
+
+`signature`::
+ Always "REVINDEX".
+`version`::
+ Version number, currently 1.
+`ofs_objects`::
+ Offset at which the entry objects begin. This is more obviously useful
+ in the index because the list of slice SHA-1s is variably-sized.
+`object_nr`::
+ Number of index entry objects present.
+`cache_nr`::
+ Number of cache slices to which the index maps, and hence the number of
+slice SHA-1s listed.
+`max_date`::
+ The oldest commit represented in the index. This is used to help speed
+up lookup times by knowing what range of commits we definitely don't have
+cached. Normal usage of 'rev-cache' would leave no "holes" in its coverage of
+commit history -- once a commit is cached, everything reachable from it should
+be cached as well. Most of the time refs are added to rev-cache simultaneous
+as well. This means that in most situations almost everything <= `max_date`
+will be cached.
+
+Mechanics
+~~~~~~~~~
+
+The most important part of rev-cache is its method of encoding topological
+relations. To ensure fluid traversal and reconstruction, commits are related
+through high-level "streams"/"channels" rather than individual
+interconnections. Intuitively, rev-cache stores history the way gitk shows it:
+commits strung up on lines, which interconnect at merges and branches.
+
+Each commit is associated to a given channel/path via a 'path id', and
+variable-length fields govern which paths (if any) are closed or opened at that
+object. This means that topo-data can be preserved in only a few bytes extra
+per object entry. Other information stored per entry is the sha-1 hash, type,
+date, size, name, and status in cache slice. Here is format of an object
+entry, both on-disk and in-memory:
+
+----
+struct object_entry {
+ unsigned type : 3;
+ unsigned is_end : 1;
+ unsigned is_start : 1;
+ unsigned uninteresting : 1;
+ unsigned include : 1;
+ unsigned flags : 1;
+ unsigned char sha1[20];
+
+ unsigned char merge_nr;
+ unsigned char split_nr;
+ unsigned size_size : 3;
+ unsigned name_size : 3;
+
+ uint32_t date;
+ uint16_t path;
+
+ /* merge paths */
+ /* split paths */
+ /* size */
+ /* name index */
+};
+----
+
+An explanation of each field:
+
+`type`::
+ Object type
+`is_end`::
+ The commit has some parents outside the cache slice (all if slice has
+ legs)
+`is_start`::
+ The commit has no children in cache slice
+`uninteresting`::
+ Run-time flag, used in traversal
+`include`::
+ Run-time flag, used in traversal (initialization)
+`flags`::
+ Currently unused, extra bit
+`sha1`::
+ Object SHA-1 hash
+
+`merge_nr`::
+ The number of paths the current channel diverges into; the current path
+ ends upon any merge.
+`split_nr`::
+ The number of paths this commit ends; used on both merging and
+ branching.
+`size_size`::
+ Number of bytes the object size takes up.
+`name_size`::
+ Number of bytes the name index takes up.
+
+`date`::
+ The date of the commit.
+`path`::
+ The path ID of the channel with which this commit is associated.
+
+merge paths::
+ The path IDs (16-bit) that are to be created. Overflow is not a
+ problem as path IDs are reused, leaving even complicated projects to
+ consume no more than a few hundred IDs.
+split paths::
+ The path IDs (16-bit) that are to be ended.
+size::
+ The size split into the minimum number of bytes. That is, 1-8 bytes
+ representing the size, least-significant byte first.
+name index::
+ An offset for the null-seperated, object name list at the end of the
+ cache slice. Also split into the minimum number of bytes.
+
+Each path ID refers to an index in a 'path array', which stores the current
+status (eg. active, interestingness) of each channel.
+
+Due to topo-relations and boundary tracking, all of a commit's parents must be
+encountered before the path is reallocated. This is achieved by using a
+counter system per merge: starting at the parent number, the counter is
+decremented as each parent is encountered (dictated by 'split paths'); at 0 the
+path is cleared.
+
+Boundary tracking is necessary because non-commits are stored relative to the
+commit in which they were introduced. If a series of commits is not included
+in the output, the last interesting commit must be parsed manually to ensure
+all objects are accounted for.
+
+To prevent list-objects from recursing into trees that we've already taken care
+of, the flag `FACE_VALUE` is introduced. An object with this flag is not
+explored (= "taken at face value"), significantly reducing I/O and processing
+time.
+
+Notes
+~~~~~
+
+Due to rev-cache's internal storage format, walking may lead to some
+discrepencies between cached and uncached repositories. Although noticeable to
+users directly calling rev-list, these are unused or corner cases and
+internally a non-issue.
+
+First note that rev-cache records commits in topological order. Large portions
+of commit history will already be sorted topologically in the revision walk,
+yielding a different output for unsorted calls to rev-list. A more obscure
+consquence occurs when two objects of the same SHA-1, but different name, are
+introduced seperately in parallel branches: different names might be shown for
+that object depending on which object entry was encountered first.
+
+A similar disparity arises when two objects of same SHA-1/different name are
+present in the same tree structure. rev-cache, walking objects as they were
+introduced, lists the youngest file's name; rev-list, walking the full trees
+each commit, shows the first file encountered.
--
tg: (0130fb5..) t/revcache/docs (depends on: t/revcache/integration)
^ permalink raw reply related
* Re: how to skip branches on git svn clone/fetch when there are errors
From: Daniele Segato @ 2009-09-07 13:53 UTC (permalink / raw)
To: Git Mailing List
In-Reply-To: <9accb4400909070634oee46b78g9270586a2b0eb4b9@mail.gmail.com>
On Mon, Sep 7, 2009 at 3:34 PM, Daniele Segato<daniele.bilug@gmail.com> wrote:
> + $SVN::Error::handler = sub {
> + (my $err) = @_;
> + my $errno = $err->apr_err();
> + my $err_key = $err->expanded_message;
> + if ($errno == 175007) {
> + warn "W: Ignoring error from SVN, path probably ",
> + "does not exist: ($errno): ",
> + $err->expanded_message,"\n";
> + }
> + return;
ups..
this should have been return inside the if and die otherwise
and the error is 170001 (wrong copy/paste)
please ignore the patch I posted before
this is the "right" one:
From a17dfb0e268e11ce70587ccb48c359348f22ad99 Mon Sep 17 00:00:00 2001
From: Daniele Segato <daniele.bilug@gmail.com>
Date: Mon, 7 Sep 2009 15:30:14 +0200
Subject: [PATCH] Ignore err:170001 authorizationfailed on checkpath
I don't know if this is the best solution to solve the issue but it does
let me download the repo skipping the problematics paths
---
git-svn.perl | 17 +++++++++++++++++
1 files changed, 17 insertions(+), 0 deletions(-)
diff --git a/git-svn.perl b/git-svn.perl
index a366c89..80f958d 100755
--- a/git-svn.perl
+++ b/git-svn.perl
@@ -3756,7 +3756,24 @@ sub check_path {
return $cache->{data}->{$path};
}
my $pool = SVN::Pool->new;
+ my $err_handler = $SVN::Error::handler;
+ $SVN::Error::handler = sub {
+ (my $err) = @_;
+ my $errno = $err->apr_err();
+ my $err_key = $err->expanded_message;
+ if ($errno == 170001) {
+ warn "W: Ignoring error from SVN, path probably ",
+ "does not exist: ($errno): ",
+ $err->expanded_message,"\n";
+ return;
+ }
+ die "Error from SVN, ($errno): ", $err->expanded_message,"\n";
+ };
+
my $t = $self->SUPER::check_path($path, $r, $pool);
+
+ $SVN::Error::handler = $err_handler;
+
$pool->clear;
if ($r != $cache->{r}) {
%{$cache->{data}} = ();
--
1.5.6.5
^ permalink raw reply related
* Re: how to skip branches on git svn clone/fetch when there are errors
From: Daniele Segato @ 2009-09-07 13:34 UTC (permalink / raw)
To: Git Mailing List
In-Reply-To: <9accb4400909070230n413c6ecfqef8238422dd5d3b@mail.gmail.com>
On Mon, Sep 7, 2009 at 11:30 AM, Daniele Segato<daniele.bilug@gmail.com> wrote:
> more info on the error (enabled the confess instead of croak to the
> Core.pm library)
>
> Use of uninitialized value in concatenation (.) or string at
> /usr/lib/perl5/SVN/Core.pm line 585.
> Authorization failed: at /usr/lib/perl5/SVN/Core.pm line 654
> SVN::Error::confess_on_error('_p_svn_error_t=SCALAR(0x9492a50)')
> called at /usr/lib/perl5/SVN/Ra.pm line 492
> SVN::Ra::AUTOLOAD('Git::SVN::Ra=HASH(0x945dae8)',
> 'alfresco-enterprise-mirror/alfresco/BRANCHES/V2.1-A/root', 7738,
> 'SVN::Pool=REF(0x9492bc0)') called at /usr/bin/git-svn line 3760
> Git::SVN::Ra::check_path('Git::SVN::Ra=HASH(0x945dae8)',
> 'alfresco-enterprise-mirror/alfresco/BRANCHES/V2.1-A/root', 7738)
> called at /usr/bin/git-svn line 4045
> Git::SVN::Ra::get_dir_check('Git::SVN::Ra=HASH(0x945dae8)',
> 'HASH(0x92131e0)', 'HASH(0x9388050)', 7738) called at /usr/bin/git-svn
> line 4062
> Git::SVN::Ra::match_globs('Git::SVN::Ra=HASH(0x945dae8)',
> 'HASH(0x92131e0)', 'HASH(0x9463c00)', 'ARRAY(0x90bbc00)', 7738) called
> at /usr/bin/git-svn line 3985
> Git::SVN::Ra::gs_fetch_loop_common('Git::SVN::Ra=HASH(0x945dae8)',
> 7737, 16113, 'ARRAY(0x90bbbe0)', 'ARRAY(0x90bbc00)') called at
> /usr/bin/git-svn line 1415
> Git::SVN::fetch_all('svn', 'HASH(0x9464250)') called at
> /usr/bin/git-svn line 372
> main::cmd_fetch() called at /usr/bin/git-svn line 253
> eval {...} called at /usr/bin/git-svn line 251
>
>
> I'll keep looking at it to see if I can figure out a way to "skip" the
> error myself and, eventually, provide a patch
I played a little with perl and modified the code
I attach the patch I created...
it probably sucks and doesn't take cares of a lot of thing that I
didn't thought about...
After applying it I was able to continue the git svn fetch from the
point I left skipping those problematics paths...
Still I get a lot of warnings with:
Use of uninitialized value in concatenation (.) or string at
/usr/lib/perl5/SVN/Core.pm line 584
I just want you to take a look at it
I will not be disappointed if you place this patch in the recycle bean :)
I cloned the git repo, checked out the v1.5.6.5 tag (which is my
current version) and modified the git-svn.perl file.
It probably wont help but I think I should share it anyway
(patch below my signature)
Bye,
Daniele
From e8a1a12e83b3f0b18ce842190d8fc8eddaa77f68 Mon Sep 17 00:00:00 2001
From: Daniele Segato <daniele.bilug@gmail.com>
Date: Mon, 7 Sep 2009 15:30:14 +0200
Subject: [PATCH] Ignore error 175007 authorization failed on checkpath
I don't know if this is the best solution to solve the issue but it does
let me download the repo skipping the problematics paths
---
git-svn.perl | 16 ++++++++++++++++
1 files changed, 16 insertions(+), 0 deletions(-)
diff --git a/git-svn.perl b/git-svn.perl
index a366c89..0ab6453 100755
--- a/git-svn.perl
+++ b/git-svn.perl
@@ -3756,7 +3756,23 @@ sub check_path {
return $cache->{data}->{$path};
}
my $pool = SVN::Pool->new;
+ my $err_handler = $SVN::Error::handler;
+ $SVN::Error::handler = sub {
+ (my $err) = @_;
+ my $errno = $err->apr_err();
+ my $err_key = $err->expanded_message;
+ if ($errno == 175007) {
+ warn "W: Ignoring error from SVN, path probably ",
+ "does not exist: ($errno): ",
+ $err->expanded_message,"\n";
+ }
+ return;
+ };
+
my $t = $self->SUPER::check_path($path, $r, $pool);
+
+ $SVN::Error::handler = $err_handler;
+
$pool->clear;
if ($r != $cache->{r}) {
%{$cache->{data}} = ();
--
1.5.6.5
^ permalink raw reply related
* Re: [PATCH] git-rebase-interactive: avoid breaking when GREP_OPTIONS="-H"
From: Dave Rodgman @ 2009-09-07 13:25 UTC (permalink / raw)
To: Carlo Marcelo Arenas Belon, git
In-Reply-To: <1252328160-4359-1-git-send-email-carenas@sajinet.com.pe>
On Mon, 07 Sep 2009 05:56 -0700, "Carlo Marcelo Arenas Belon"
<carenas@sajinet.com.pe> wrote:
> if GREP_OPTIONS is set and includes -H, using `grep -c` will fail
> to generate a numeric count and result in the following error :
>
> /usr/libexec/git-core/git-rebase--interactive: line 110: (standard
> input):1+(standard input):0: missing `)' (error token is
> "input):1+(standard input):0")
I think in my case, grep is being confused by colours being enabled - I
have this wrapper script
for grep:
#!/bin/bash
echo $@
`which -a grep|/bin/grep -v $0|head -n 1` --color=auto $@
your patch fixes it though.
thanks
Dave
>
> instead of grep counting use `wc -l` to return the line count.
>
> Signed-off-by: Carlo Marcelo Arenas Belon <carenas@sajinet.com.pe>
> ---
> git-rebase--interactive.sh | 4 ++--
> 1 files changed, 2 insertions(+), 2 deletions(-)
>
> diff --git a/git-rebase--interactive.sh b/git-rebase--interactive.sh
> index 23ded48..c12d980 100755
> --- a/git-rebase--interactive.sh
> +++ b/git-rebase--interactive.sh
> @@ -106,8 +106,8 @@ mark_action_done () {
> sed -e 1q < "$TODO" >> "$DONE"
> sed -e 1d < "$TODO" >> "$TODO".new
> mv -f "$TODO".new "$TODO"
> - count=$(grep -c '^[^#]' < "$DONE")
> - total=$(($count+$(grep -c '^[^#]' < "$TODO")))
> + count=$(grep '^[^#]' < "$DONE" | wc -l)
> + total=$(($count+$(grep '^[^#]' < "$TODO" | wc -l)))
> if test "$last_count" != "$count"
> then
> last_count=$count
> --
> 1.6.3.3
>
> --
> To unsubscribe from this list: send the line "unsubscribe git" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* [PATCH] git-rebase-interactive: avoid breaking when GREP_OPTIONS="-H"
From: Carlo Marcelo Arenas Belon @ 2009-09-07 12:56 UTC (permalink / raw)
To: git
if GREP_OPTIONS is set and includes -H, using `grep -c` will fail
to generate a numeric count and result in the following error :
/usr/libexec/git-core/git-rebase--interactive: line 110: (standard
input):1+(standard input):0: missing `)' (error token is
"input):1+(standard input):0")
instead of grep counting use `wc -l` to return the line count.
Signed-off-by: Carlo Marcelo Arenas Belon <carenas@sajinet.com.pe>
---
git-rebase--interactive.sh | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/git-rebase--interactive.sh b/git-rebase--interactive.sh
index 23ded48..c12d980 100755
--- a/git-rebase--interactive.sh
+++ b/git-rebase--interactive.sh
@@ -106,8 +106,8 @@ mark_action_done () {
sed -e 1q < "$TODO" >> "$DONE"
sed -e 1d < "$TODO" >> "$TODO".new
mv -f "$TODO".new "$TODO"
- count=$(grep -c '^[^#]' < "$DONE")
- total=$(($count+$(grep -c '^[^#]' < "$TODO")))
+ count=$(grep '^[^#]' < "$DONE" | wc -l)
+ total=$(($count+$(grep '^[^#]' < "$TODO" | wc -l)))
if test "$last_count" != "$count"
then
last_count=$count
--
1.6.3.3
^ permalink raw reply related
* Re: git rebase --interactive problems
From: Carlo Marcelo Arenas Belon @ 2009-09-07 13:04 UTC (permalink / raw)
To: Dave Rodgman; +Cc: git
In-Reply-To: <1252326716.7497.1333578429@webmail.messagingengine.com>
On Mon, Sep 07, 2009 at 01:31:56PM +0100, Dave Rodgman wrote:
>
> Am I doing something wrong, or is this a bug? I am using git 1.6.4.2
can you do before running the rebase (assuming a bourne shell) :
unset GREP_OPTIONS
this looks like the bug I just send a patch to the list to fix and
which seems to be also reported in Ubuntu as :
https://bugs.launchpad.net/ubuntu/+source/git-core/+bug/398393
Carlo
^ permalink raw reply
* Re: git rebase --interactive problems
From: Jeff Epler @ 2009-09-07 13:03 UTC (permalink / raw)
To: Dave Rodgman; +Cc: git
In-Reply-To: <1252326716.7497.1333578429@webmail.messagingengine.com>
It looks like something is going wrong with the 'grep -c' in
mark_action_done () {
sed -e 1q < "$TODO" >> "$DONE"
sed -e 1d < "$TODO" >> "$TODO".new
mv -f "$TODO".new "$TODO"
count=$(grep -c '^[^#]' < "$DONE")
total=$(($count+$(grep -c '^[^#]' < "$TODO")))
if test "$last_count" != "$count"
then
last_count=$count
printf "Rebasing (%d/%d)\r" $count $total
test -z "$VERBOSE" || echo
fi
}
are you using an unusual platform or have an unusual /bin/grep or
/bin/sh?
Jeff
^ permalink raw reply
* [PATCH] gitweb: Add 'show-sizes' feature to show blob sizes in tree view
From: Jakub Narebski @ 2009-09-07 12:40 UTC (permalink / raw)
To: git
Add support for 'show-sizes' feature to show (in separate column,
between mode and filename) the size of blobs (files) in the 'tree'
view. It passes '-l' option to "git ls-tree" invocation.
For the 'tree' and 'commit' (submodule) entries, '-' is shown in place
of size; for generated '..' "up directory" entry nothing is shown.
The 'show-sizes' feature is enabled by default.
Signed-off-by: Jakub Narebski <jnareb@gmail.com>
---
I am sorry if this patch was sent twice to git mailing list
I don't quite like 'show-sizes' as a name for this feature.
Does anyone has any better ideas?
Not tested extensively (just that it works, and looks correct),
in particular I didn't test that having 'gitweb.showsizes' as
config variable name while 'show-sizes' as feature name works
as expected.
Previous version of this idea was presented in
http://thread.gmane.org/gmane.comp.version-control.git/54335/focus=54334
It required to pass '-l' to 'tree' action via 'opt' (extra options)
parameter. This one uses features mechanism.
This patch might have @extra_options passed unnecessary...
gitweb/gitweb.css | 6 +++++
gitweb/gitweb.perl | 69 +++++++++++++++++++++++++++++++++++++++++-----------
2 files changed, 60 insertions(+), 15 deletions(-)
diff --git a/gitweb/gitweb.css b/gitweb/gitweb.css
index 8f68fe3..d60bfc1 100644
--- a/gitweb/gitweb.css
+++ b/gitweb/gitweb.css
@@ -341,6 +341,12 @@ td.mode {
font-family: monospace;
}
+/* format of (optional) objects size in 'tree' view */
+td.size {
+ font-family: monospace;
+ text-align: right;
+}
+
/* styling of diffs (patchsets): commitdiff and blobdiff views */
div.diff.header,
div.diff.extended_header {
diff --git a/gitweb/gitweb.perl b/gitweb/gitweb.perl
index 24b2193..7b1c60e 100755
--- a/gitweb/gitweb.perl
+++ b/gitweb/gitweb.perl
@@ -297,6 +297,19 @@ our %feature = (
'override' => 0,
'default' => [1]},
+ # Enable showing size of blobs in a 'tree' view, in a separate
+ # column, similar to what 'ls -l' does. This cost a bit of IO.
+
+ # To disable system wide have in $GITWEB_CONFIG
+ # $feature{'show-sizes'}{'default'} = [0];
+ # To have project specific config enable override in $GITWEB_CONFIG
+ # $feature{'show-sizes'}{'override'} = 1;
+ # and in project config gitweb.showsizes = 0|1;
+ 'show-sizes' => {
+ 'sub' => sub { feature_bool('showsizes', @_) },
+ 'override' => 0,
+ 'default' => [1]},
+
# Make gitweb use an alternative format of the URLs which can be
# more readable and natural-looking: project name is embedded
# directly in the path and the query string contains other
@@ -2764,16 +2777,31 @@ sub parse_ls_tree_line {
my %opts = @_;
my %res;
- #'100644 blob 0fa3f3a66fb6a137f6ec2c19351ed4d807070ffa panic.c'
- $line =~ m/^([0-9]+) (.+) ([0-9a-fA-F]{40})\t(.+)$/s;
+ if ($opts{'-l'}) {
+ #'100644 blob 0fa3f3a66fb6a137f6ec2c19351ed4d807070ffa 16717 panic.c'
+ $line =~ m/^([0-9]+) (.+) ([0-9a-fA-F]{40}) +(-|[0-9]+)\t(.+)$/s;
- $res{'mode'} = $1;
- $res{'type'} = $2;
- $res{'hash'} = $3;
- if ($opts{'-z'}) {
- $res{'name'} = $4;
+ $res{'mode'} = $1;
+ $res{'type'} = $2;
+ $res{'hash'} = $3;
+ $res{'size'} = $4;
+ if ($opts{'-z'}) {
+ $res{'name'} = $5;
+ } else {
+ $res{'name'} = unquote($5);
+ }
} else {
- $res{'name'} = unquote($4);
+ #'100644 blob 0fa3f3a66fb6a137f6ec2c19351ed4d807070ffa panic.c'
+ $line =~ m/^([0-9]+) (.+) ([0-9a-fA-F]{40})\t(.+)$/s;
+
+ $res{'mode'} = $1;
+ $res{'type'} = $2;
+ $res{'hash'} = $3;
+ if ($opts{'-z'}) {
+ $res{'name'} = $4;
+ } else {
+ $res{'name'} = unquote($4);
+ }
}
return wantarray ? %res : \%res;
@@ -3564,6 +3592,9 @@ sub git_print_tree_entry {
# and link is the action links of the entry.
print "<td class=\"mode\">" . mode_str($t->{'mode'}) . "</td>\n";
+ if (exists $t->{'size'}) {
+ print "<td class=\"size\">$t->{'size'}</td>\n";
+ }
if ($t->{'type'} eq "blob") {
print "<td class=\"list\">" .
$cgi->a({-href => href(action=>"blob", hash=>$t->{'hash'},
@@ -3609,12 +3640,14 @@ sub git_print_tree_entry {
} elsif ($t->{'type'} eq "tree") {
print "<td class=\"list\">";
print $cgi->a({-href => href(action=>"tree", hash=>$t->{'hash'},
- file_name=>"$basedir$t->{'name'}", %base_key)},
+ file_name=>"$basedir$t->{'name'}",
+ %base_key)},
esc_path($t->{'name'}));
print "</td>\n";
print "<td class=\"link\">";
print $cgi->a({-href => href(action=>"tree", hash=>$t->{'hash'},
- file_name=>"$basedir$t->{'name'}", %base_key)},
+ file_name=>"$basedir$t->{'name'}",
+ %base_key)},
"tree");
if (defined $hash_base) {
print " | " .
@@ -5088,10 +5121,14 @@ sub git_tree {
}
die_error(404, "No such tree") unless defined($hash);
+ my $show_sizes = gitweb_check_feature('show-sizes');
+ my $have_blame = gitweb_check_feature('blame');
+
my @entries = ();
{
local $/ = "\0";
- open my $fd, "-|", git_cmd(), "ls-tree", '-z', $hash
+ open my $fd, "-|", git_cmd(), "ls-tree", '-z',
+ ($show_sizes ? '-l' : ()), @extra_options, $hash
or die_error(500, "Open git-ls-tree failed");
@entries = map { chomp; $_ } <$fd>;
close $fd
@@ -5102,7 +5139,6 @@ sub git_tree {
my $ref = format_ref_marker($refs, $hash_base);
git_header_html();
my $basedir = '';
- my $have_blame = gitweb_check_feature('blame');
if (defined $hash_base && (my %co = parse_commit($hash_base))) {
my @views_nav = ();
if (defined $file_name) {
@@ -5118,7 +5154,8 @@ sub git_tree {
# FIXME: Should be available when we have no hash base as well.
push @views_nav, $snapshot_links;
}
- git_print_page_nav('tree','', $hash_base, undef, undef, join(' | ', @views_nav));
+ git_print_page_nav('tree','', $hash_base, undef, undef,
+ join(' | ', @views_nav));
git_print_header_div('commit', esc_html($co{'title'}) . $ref, $hash_base);
} else {
undef $hash_base;
@@ -5151,8 +5188,10 @@ sub git_tree {
undef $up unless $up;
# based on git_print_tree_entry
print '<td class="mode">' . mode_str('040000') . "</td>\n";
+ print '<td class="size"> </td>'."\n" if $show_sizes;
print '<td class="list">';
- print $cgi->a({-href => href(action=>"tree", hash_base=>$hash_base,
+ print $cgi->a({-href => href(action=>"tree",
+ hash_base=>$hash_base,
file_name=>$up)},
"..");
print "</td>\n";
@@ -5161,7 +5200,7 @@ sub git_tree {
print "</tr>\n";
}
foreach my $line (@entries) {
- my %t = parse_ls_tree_line($line, -z => 1);
+ my %t = parse_ls_tree_line($line, -z => 1, -l => $show_sizes);
if ($alternate) {
print "<tr class=\"dark\">\n";
^ permalink raw reply related
* [PATCH resend] git-pull: fix fetch-options.txt to not document --quiet and --verbose twice in git-pull.txt
From: Emmanuel Trillaud @ 2009-09-07 12:34 UTC (permalink / raw)
To: Git Mailing List
Hello all,
In git-pull(1) we can read :
OPTIONS
-q, --quiet
Operate quietly.
-v, --verbose
Be verbose.
...
-q, --quiet
Pass --quiet to git-fetch-pack and silence any other
internally used git
commands.
-v, --verbose
Be verbose.
The first part is included by merge-option.txt and the second by
fetch-options.txt.
I choose to "suppress" the fetch-options part because IMHO we don't
need that level
of precision. But if you prefer, I can provide a patch to "ifndef" the
merge-options.txt part.
Best regard
git-pull.txt includes fetch-options.txt and merge-options.txt which both
document the --quiet and --verbose parameters. So we supress the
--quiet and --verbose paragraphs if fetch-options.txt is included by
git-pull.txt
Signed-off-by: Emmanuel Trillaud <etrillaud@gmail.com>
---
Documentation/fetch-options.txt | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
index ea3b1bc..5eb2b0e 100644
--- a/Documentation/fetch-options.txt
+++ b/Documentation/fetch-options.txt
@@ -1,3 +1,4 @@
+ifndef::git-pull[]
-q::
--quiet::
Pass --quiet to git-fetch-pack and silence any other internally
@@ -6,6 +7,7 @@
-v::
--verbose::
Be verbose.
+endif::git-pull[]
-a::
--append::
--
1.6.4.2.253.g0b1fac
^ permalink raw reply related
* git rebase --interactive problems
From: Dave Rodgman @ 2009-09-07 12:31 UTC (permalink / raw)
To: git
When I try to do even the simplest rebase --interactive, I get a cryptic
error message:
$ git rebase -i HEAD~2
<do nothing in the editor that appears, just save>
I then get:
/usr/lib/git-core/git-rebase--interactive: 1: arithmetic expression:
expecting primary: "-c ^[^#]
2+-c ^[^#]
0"
and all I can do is git rebase --abort
Am I doing something wrong, or is this a bug? I am using git 1.6.4.2
thanks
Dave
^ permalink raw reply
* Re: [PATCH 14/14] Add README and gitignore file for MSVC build
From: Erik Faye-Lund @ 2009-09-07 12:26 UTC (permalink / raw)
To: Frank Li
Cc: Thiago Farina, Marius Storm-Olsen, Reece Dunn,
Johannes.Schindelin, msysgit, git
In-Reply-To: <1976ea660908250732q1e1fc153g663f3a9c13f1c902@mail.gmail.com>
On Tue, Aug 25, 2009 at 4:32 PM, Frank Li<lznuaa@gmail.com> wrote:
> I update http://repo.or.cz/w/gitbuild.git.
> Add create_command.bat to create common-cmds.h.
I just gave this script a try, but if I simply click the .bat-file, I
get the following crash in sh.exe: "Unhandled exception at 0x77ba8e7c
in sh.exe: 0xC0000005: Access violation writing location 0x00000014."
Here's the output I get on the console:
--->8---
C:\Users\Erik\src\git-msvc\gitbuild>setlocal
C:\Users\Erik\src\git-msvc\gitbuild>set tools=C:\Users\Erik\src\git-msvc\gitbuil
d\tools
C:\Users\Erik\src\git-msvc\gitbuild>echo C:\Users\Erik\src\git-msvc\gitbuild\too
ls
C:\Users\Erik\src\git-msvc\gitbuild\tools
C:\Users\Erik\src\git-msvc\gitbuild>set path=C:\Users\Erik\src\git-msvc\gitbuild
\tools
C:\Users\Erik\src\git-msvc\gitbuild>cd ext\git
C:\Users\Erik\src\git-msvc\gitbuild\ext\git>sh generate-cmdlist.sh 1>common-cmd
s.h
abnormal program termination
--->8---
Since the path is set to only point to the tools, my guess is that
other msys installations (I've got two) shouldn't affect this, no?
I'm running Vista 64bit.
--
Erik "kusma" Faye-Lund
kusmabite@gmail.com
(+47) 986 59 656
^ permalink raw reply
* Re: Use case I don't know how to address
From: Alan Chandler @ 2009-09-07 12:08 UTC (permalink / raw)
To: git
In-Reply-To: <4AA4F632.3070206@chandlerfamily.org.uk>
Alan Chandler wrote:
> Alan Chandler wrote:
>> Junio C Hamano wrote:
>>> Alan Chandler <alan@chandlerfamily.org.uk> writes:
>>>
>>>> 2' - 2a - 3' - 4' ----------------- 6' SITE
>>>> / / / /
>>>> 1 - 2 ------ 3 - 4 ------------6'''- 6a TEST
>>>> \ /
>>>> 5 ------ 6 MASTER
>>>> \ \
>>>> 5''- 5a- 6'' DEMO
>>>>
>>>>
>>>> What will happen is the changes made in 4->5 will get applied to the
>>>> (now) Test branch as part of the 6->6'' merge, and I will be left
>>>> having to add a new commit, 6a, to undo them all again. Given this is
>>>> likely to be quite a substantial change I want to try and avoid it if
>>>> possible.
>>>
>>> I presume 6'''-6a has the revert of 4-5? If so, the next merge should
>>> work just fine.
>>
>>
>> I think you missed the issue - Yes 6'''-6a is the revert, but the
>> problem is this could be large and complicated, and I wanted to find
>> an automated way rather than manual
>>
>> Sort of like doing a diff of 4-5 and somehow applying it backwards.
>>
>>
>
> I just discovered that git-apply has the -R flag. Is that what I am
> looking for?
>
2' - 2a - 3' - 4' ------------ 6' SITE
/ / / /
1 - 2 ------ 3 - 4 --5'''--5b---6''' TEST
\ / /
5 ------ 6 MASTER
\ \
5''- 5a- 6'' DEMO
Just to be clear - if I do a diff of 4->5 and then immediately merge it
back to 4 as 5'' (which fast forwards 4) and then 5b is the diff of 4-5
applied with git apply -R.
Is that what the -R flag does on git apply?
--
Alan Chandler
http://www.chandlerfamily.org.uk
^ permalink raw reply
* Re: Use case I don't know how to address
From: Alan Chandler @ 2009-09-07 12:01 UTC (permalink / raw)
Cc: git
In-Reply-To: <4AA3AEA1.7030107@chandlerfamily.org.uk>
Alan Chandler wrote:
> Junio C Hamano wrote:
>> Alan Chandler <alan@chandlerfamily.org.uk> writes:
>>
>>> 2' - 2a - 3' - 4' ----------------- 6' SITE
>>> / / / /
>>> 1 - 2 ------ 3 - 4 ------------6'''- 6a TEST
>>> \ /
>>> 5 ------ 6 MASTER
>>> \ \
>>> 5''- 5a- 6'' DEMO
>>>
>>>
>>> What will happen is the changes made in 4->5 will get applied to the
>>> (now) Test branch as part of the 6->6'' merge, and I will be left
>>> having to add a new commit, 6a, to undo them all again. Given this is
>>> likely to be quite a substantial change I want to try and avoid it if
>>> possible.
>>
>> I presume 6'''-6a has the revert of 4-5? If so, the next merge should
>> work just fine.
>
>
> I think you missed the issue - Yes 6'''-6a is the revert, but the
> problem is this could be large and complicated, and I wanted to find an
> automated way rather than manual
>
> Sort of like doing a diff of 4-5 and somehow applying it backwards.
>
>
I just discovered that git-apply has the -R flag. Is that what I am
looking for?
--
Alan Chandler
http://www.chandlerfamily.org.uk
^ permalink raw reply
* [PATCH/RFC] gitweb: Add 'show-sizes' feature to show blob sizes in tree view
From: Jakub Narebski @ 2009-09-07 12:00 UTC (permalink / raw)
To: git, git; +Cc: Jakub Narebski
Add support for 'show-sizes' feature to show (in separate column,
between mode and filename) the size of blobs (files) in the 'tree'
view. It passes '-l' option to "git ls-tree" invocation.
For the 'tree' and 'commit' (submodule) entries, '-' is shown in place
of size; for generated '..' "up directory" entry nothing is shown.
The 'show-sizes' feature is enabled by default.
Signed-off-by: Jakub Narebski <jnareb@gmail.com>
---
I don't quite like 'show-sizes' as a name for this feature.
Does anyone has any better ideas?
Not tested extensively (just that it works, and looks correct),
in particular I didn't test that having 'gitweb.showsizes' as
config variable name while 'show-sizes' as feature name works
as expected.
Previous version of this idea was presented in
http://thread.gmane.org/gmane.comp.version-control.git/54335/focus=54334
It required to pass '-l' to 'tree' action via 'opt' (extra options)
parameter. This one uses features mechanism.
This patch might have @extra_options passed unnecessary...
gitweb/gitweb.css | 6 +++++
gitweb/gitweb.perl | 69 +++++++++++++++++++++++++++++++++++++++++-----------
2 files changed, 60 insertions(+), 15 deletions(-)
diff --git a/gitweb/gitweb.css b/gitweb/gitweb.css
index 8f68fe3..d60bfc1 100644
--- a/gitweb/gitweb.css
+++ b/gitweb/gitweb.css
@@ -341,6 +341,12 @@ td.mode {
font-family: monospace;
}
+/* format of (optional) objects size in 'tree' view */
+td.size {
+ font-family: monospace;
+ text-align: right;
+}
+
/* styling of diffs (patchsets): commitdiff and blobdiff views */
div.diff.header,
div.diff.extended_header {
diff --git a/gitweb/gitweb.perl b/gitweb/gitweb.perl
index 24b2193..7b1c60e 100755
--- a/gitweb/gitweb.perl
+++ b/gitweb/gitweb.perl
@@ -297,6 +297,19 @@ our %feature = (
'override' => 0,
'default' => [1]},
+ # Enable showing size of blobs in a 'tree' view, in a separate
+ # column, similar to what 'ls -l' does. This cost a bit of IO.
+
+ # To disable system wide have in $GITWEB_CONFIG
+ # $feature{'show-sizes'}{'default'} = [0];
+ # To have project specific config enable override in $GITWEB_CONFIG
+ # $feature{'show-sizes'}{'override'} = 1;
+ # and in project config gitweb.showsizes = 0|1;
+ 'show-sizes' => {
+ 'sub' => sub { feature_bool('showsizes', @_) },
+ 'override' => 0,
+ 'default' => [1]},
+
# Make gitweb use an alternative format of the URLs which can be
# more readable and natural-looking: project name is embedded
# directly in the path and the query string contains other
@@ -2764,16 +2777,31 @@ sub parse_ls_tree_line {
my %opts = @_;
my %res;
- #'100644 blob 0fa3f3a66fb6a137f6ec2c19351ed4d807070ffa panic.c'
- $line =~ m/^([0-9]+) (.+) ([0-9a-fA-F]{40})\t(.+)$/s;
+ if ($opts{'-l'}) {
+ #'100644 blob 0fa3f3a66fb6a137f6ec2c19351ed4d807070ffa 16717 panic.c'
+ $line =~ m/^([0-9]+) (.+) ([0-9a-fA-F]{40}) +(-|[0-9]+)\t(.+)$/s;
- $res{'mode'} = $1;
- $res{'type'} = $2;
- $res{'hash'} = $3;
- if ($opts{'-z'}) {
- $res{'name'} = $4;
+ $res{'mode'} = $1;
+ $res{'type'} = $2;
+ $res{'hash'} = $3;
+ $res{'size'} = $4;
+ if ($opts{'-z'}) {
+ $res{'name'} = $5;
+ } else {
+ $res{'name'} = unquote($5);
+ }
} else {
- $res{'name'} = unquote($4);
+ #'100644 blob 0fa3f3a66fb6a137f6ec2c19351ed4d807070ffa panic.c'
+ $line =~ m/^([0-9]+) (.+) ([0-9a-fA-F]{40})\t(.+)$/s;
+
+ $res{'mode'} = $1;
+ $res{'type'} = $2;
+ $res{'hash'} = $3;
+ if ($opts{'-z'}) {
+ $res{'name'} = $4;
+ } else {
+ $res{'name'} = unquote($4);
+ }
}
return wantarray ? %res : \%res;
@@ -3564,6 +3592,9 @@ sub git_print_tree_entry {
# and link is the action links of the entry.
print "<td class=\"mode\">" . mode_str($t->{'mode'}) . "</td>\n";
+ if (exists $t->{'size'}) {
+ print "<td class=\"size\">$t->{'size'}</td>\n";
+ }
if ($t->{'type'} eq "blob") {
print "<td class=\"list\">" .
$cgi->a({-href => href(action=>"blob", hash=>$t->{'hash'},
@@ -3609,12 +3640,14 @@ sub git_print_tree_entry {
} elsif ($t->{'type'} eq "tree") {
print "<td class=\"list\">";
print $cgi->a({-href => href(action=>"tree", hash=>$t->{'hash'},
- file_name=>"$basedir$t->{'name'}", %base_key)},
+ file_name=>"$basedir$t->{'name'}",
+ %base_key)},
esc_path($t->{'name'}));
print "</td>\n";
print "<td class=\"link\">";
print $cgi->a({-href => href(action=>"tree", hash=>$t->{'hash'},
- file_name=>"$basedir$t->{'name'}", %base_key)},
+ file_name=>"$basedir$t->{'name'}",
+ %base_key)},
"tree");
if (defined $hash_base) {
print " | " .
@@ -5088,10 +5121,14 @@ sub git_tree {
}
die_error(404, "No such tree") unless defined($hash);
+ my $show_sizes = gitweb_check_feature('show-sizes');
+ my $have_blame = gitweb_check_feature('blame');
+
my @entries = ();
{
local $/ = "\0";
- open my $fd, "-|", git_cmd(), "ls-tree", '-z', $hash
+ open my $fd, "-|", git_cmd(), "ls-tree", '-z',
+ ($show_sizes ? '-l' : ()), @extra_options, $hash
or die_error(500, "Open git-ls-tree failed");
@entries = map { chomp; $_ } <$fd>;
close $fd
@@ -5102,7 +5139,6 @@ sub git_tree {
my $ref = format_ref_marker($refs, $hash_base);
git_header_html();
my $basedir = '';
- my $have_blame = gitweb_check_feature('blame');
if (defined $hash_base && (my %co = parse_commit($hash_base))) {
my @views_nav = ();
if (defined $file_name) {
@@ -5118,7 +5154,8 @@ sub git_tree {
# FIXME: Should be available when we have no hash base as well.
push @views_nav, $snapshot_links;
}
- git_print_page_nav('tree','', $hash_base, undef, undef, join(' | ', @views_nav));
+ git_print_page_nav('tree','', $hash_base, undef, undef,
+ join(' | ', @views_nav));
git_print_header_div('commit', esc_html($co{'title'}) . $ref, $hash_base);
} else {
undef $hash_base;
@@ -5151,8 +5188,10 @@ sub git_tree {
undef $up unless $up;
# based on git_print_tree_entry
print '<td class="mode">' . mode_str('040000') . "</td>\n";
+ print '<td class="size"> </td>'."\n" if $show_sizes;
print '<td class="list">';
- print $cgi->a({-href => href(action=>"tree", hash_base=>$hash_base,
+ print $cgi->a({-href => href(action=>"tree",
+ hash_base=>$hash_base,
file_name=>$up)},
"..");
print "</td>\n";
@@ -5161,7 +5200,7 @@ sub git_tree {
print "</tr>\n";
}
foreach my $line (@entries) {
- my %t = parse_ls_tree_line($line, -z => 1);
+ my %t = parse_ls_tree_line($line, -z => 1, -l => $show_sizes);
if ($alternate) {
print "<tr class=\"dark\">\n";
^ permalink raw reply related
* Re: [PATCH 3/4] push: make non-fast-forward help message configurable
From: Matthieu Moy @ 2009-09-07 11:20 UTC (permalink / raw)
To: Jeff King; +Cc: Nanako Shiraishi, Junio C Hamano, Teemu Likonen, Git
In-Reply-To: <20090907085405.GA17968@coredump.intra.peff.net>
Jeff King <peff@peff.net> writes:
> I'm not sure it solves the problem. The point of "message.all" was to
> easily say "I'm an expert, so turn off useless advice". But now I would
> have to manually re-enable any messages that I _do_ want to see. And of
> course I don't see them to know that I want them, so I have to read
> through the config documentation and decide on each one.
Well, if it was _that_ important, I'd go for your suggestion of a
message hierarchy message.advice.foo, message.info.bar and so, with
the possibility of enabling/disabling a subhierarchy with a config
option. Now, I really get the feeling that this is overkill...
> So I think "be verbose, but let the user quiet us" is probably
> better than "be quiet, but let the user make us louder", because it is
> easier to discover verbose things. Which implies to me that
> "message.all", if it exists at all, should be limited in scope to just
> advice.
Yup, you convinced me for the last implication.
Otherwise, one setting "message.all = false" would never even notice
that another very cool informative message was added to Git in its
latest version.
--
Matthieu Moy
http://www-verimag.imag.fr/~moy/
^ permalink raw reply
* Re: [PATCH] Do not scramble password read from .cvspass
From: Dirk Hörner @ 2009-09-07 10:39 UTC (permalink / raw)
To: Junio C Hamano; +Cc: git list, pascal
In-Reply-To: <7vvdjyn0j3.fsf@alter.siamese.dyndns.org>
Hi Junio,
On Fri, Sep 4, 2009 at 6:21 PM, Junio C Hamano<gitster@pobox.com> wrote:
> Dirk, does the patch look Ok to you?
The patch looks fine :-)
Ciao,
Dirk
>
>> git-cvsimport.perl | 7 ++++---
>> 1 files changed, 4 insertions(+), 3 deletions(-)
>>
>> diff --git a/git-cvsimport.perl b/git-cvsimport.perl
>> index 593832d..c5cdcae 100755
>> --- a/git-cvsimport.perl
>> +++ b/git-cvsimport.perl
>> @@ -238,7 +238,10 @@ sub conn {
>> }
>> my $rr = ":pserver:$user\@$serv:$port$repo";
>>
>> - unless ($pass) {
>> + if ($pass) {
>> + $pass = $self->_scramble($pass);
>> + } else
>> + {
>> open(H,$ENV{'HOME'}."/.cvspass") and do {
>> #
>> :pserver:cvs@mea.tmt.tele.fi:/cvsroot/zmailer Ah<Z
>> while (<H>) {
>> @@ -253,8 +256,6 @@ sub conn {
>> };
>> }
>>
>> - $pass = $self->_scramble($pass);
>> -
>> my ($s, $rep);
>> if ($proxyhost) {
>>
>> --
>> 1.6.4.2.253.g0b1fac
>
^ permalink raw reply
* Re: tracking branch for a rebase
From: Michael J Gruber @ 2009-09-07 9:53 UTC (permalink / raw)
To: Johannes Schindelin
Cc: Jeff King, Junio C Hamano, Björn Steinbrink, Pete Wyckoff,
git
In-Reply-To: <alpine.DEB.1.00.0909071126040.8306@pacific.mpi-cbg.de>
Johannes Schindelin venit, vidit, dixit 07.09.2009 11:29:
> Hi,
>
> On Mon, 7 Sep 2009, Jeff King wrote:
>
>> On Sun, Sep 06, 2009 at 10:05:21PM -0700, Junio C Hamano wrote:
>>
>>> ref@{number} -- nth reflog entry
>>> ref@{time} -- ref back then
>>> @{-number} -- nth branch switching
>>>
>>> So perhaps ref@{upstream}, or any string that is not a number and cannot
>>> be time, can trigger the magic operation on the ref with ref@{magic}
>>> syntax?
>>
>> I think using @{} is a reasonable extension format.
>
> Sorry to enter this thread that late, but I did not realize that it
> touches my %<branch> work.
>
> Your proposal leads to something like "master@{upstream}@{2.days.ago}",
> which looks ugly. And it is much more to type.
>
> I still think that it is not too-much asked for to require the
> "refs/heads/" prefix if somebody starts her branch names with "%".
>
> Or did I miss something (as I do not have time to read long mails these
> days, I tend to read only the short, to-the-point ones; I allowed myself
> to only skim over the rest of your mail)?
Solution needs to be:
- practical (short, clear)
- not too brutal (on existing users of legal "exotic" refnames)
- extensible (for later uses)
Michael
152 chars for Dscho :)
^ permalink raw reply
* Re: how to skip branches on git svn clone/fetch when there are errors
From: Daniele Segato @ 2009-09-07 9:30 UTC (permalink / raw)
To: Git Mailing List
In-Reply-To: <9accb4400908310126v15b08c7fr425c9daff26012f3@mail.gmail.com>
On Mon, Aug 31, 2009 at 10:26 AM, Daniele
Segato<daniele.bilug@gmail.com> wrote:>
> git init
> git svn init svn://svn.mydomain.com/path/to/repo -T HEAD -b BRANCHES -t TAGS
> $ git svn fetch
> Use of uninitialized value in concatenation (.) or string at
> /usr/lib/perl5/SVN/Core.pm line 584.
> Authorization failed: at /usr/bin/git-svn line 1415
>
more info on the error (enabled the confess instead of croak to the
Core.pm library)
Use of uninitialized value in concatenation (.) or string at
/usr/lib/perl5/SVN/Core.pm line 585.
Authorization failed: at /usr/lib/perl5/SVN/Core.pm line 654
SVN::Error::confess_on_error('_p_svn_error_t=SCALAR(0x9492a50)')
called at /usr/lib/perl5/SVN/Ra.pm line 492
SVN::Ra::AUTOLOAD('Git::SVN::Ra=HASH(0x945dae8)',
'alfresco-enterprise-mirror/alfresco/BRANCHES/V2.1-A/root', 7738,
'SVN::Pool=REF(0x9492bc0)') called at /usr/bin/git-svn line 3760
Git::SVN::Ra::check_path('Git::SVN::Ra=HASH(0x945dae8)',
'alfresco-enterprise-mirror/alfresco/BRANCHES/V2.1-A/root', 7738)
called at /usr/bin/git-svn line 4045
Git::SVN::Ra::get_dir_check('Git::SVN::Ra=HASH(0x945dae8)',
'HASH(0x92131e0)', 'HASH(0x9388050)', 7738) called at /usr/bin/git-svn
line 4062
Git::SVN::Ra::match_globs('Git::SVN::Ra=HASH(0x945dae8)',
'HASH(0x92131e0)', 'HASH(0x9463c00)', 'ARRAY(0x90bbc00)', 7738) called
at /usr/bin/git-svn line 3985
Git::SVN::Ra::gs_fetch_loop_common('Git::SVN::Ra=HASH(0x945dae8)',
7737, 16113, 'ARRAY(0x90bbbe0)', 'ARRAY(0x90bbc00)') called at
/usr/bin/git-svn line 1415
Git::SVN::fetch_all('svn', 'HASH(0x9464250)') called at
/usr/bin/git-svn line 372
main::cmd_fetch() called at /usr/bin/git-svn line 253
eval {...} called at /usr/bin/git-svn line 251
I'll keep looking at it to see if I can figure out a way to "skip" the
error myself and, eventually, provide a patch
Bye,
Daniele
^ permalink raw reply
* Re: tracking branch for a rebase
From: Johannes Schindelin @ 2009-09-07 9:29 UTC (permalink / raw)
To: Jeff King
Cc: Junio C Hamano, Björn Steinbrink, Michael J Gruber,
Pete Wyckoff, git
In-Reply-To: <20090907084324.GB17997@coredump.intra.peff.net>
Hi,
On Mon, 7 Sep 2009, Jeff King wrote:
> On Sun, Sep 06, 2009 at 10:05:21PM -0700, Junio C Hamano wrote:
>
> > ref@{number} -- nth reflog entry
> > ref@{time} -- ref back then
> > @{-number} -- nth branch switching
> >
> > So perhaps ref@{upstream}, or any string that is not a number and cannot
> > be time, can trigger the magic operation on the ref with ref@{magic}
> > syntax?
>
> I think using @{} is a reasonable extension format.
Sorry to enter this thread that late, but I did not realize that it
touches my %<branch> work.
Your proposal leads to something like "master@{upstream}@{2.days.ago}",
which looks ugly. And it is much more to type.
I still think that it is not too-much asked for to require the
"refs/heads/" prefix if somebody starts her branch names with "%".
Or did I miss something (as I do not have time to read long mails these
days, I tend to read only the short, to-the-point ones; I allowed myself
to only skim over the rest of your mail)?
Ciao,
Dscho
^ permalink raw reply
* Re: Issue 323 in msysgit: Can't clone over http
From: Tay Ray Chuan @ 2009-09-07 9:27 UTC (permalink / raw)
To: Junio C Hamano; +Cc: git, msysgit, Tom Preston-Werner
In-Reply-To: <7vocpn44dg.fsf@alter.siamese.dyndns.org>
Hi,
On Mon, Sep 7, 2009 at 3:10 PM, Junio C Hamano<gitster@pobox.com> wrote:
> We couldn't perform the check, and then what happens? We continue as if
> everything is peachy, assuming that the *.idx file we thought we were
> going to get describe what objects are in the corresponding pack, and barf
> when we try to read the *.idx file that we failed to download even though
> the server said we should be able to get it?
We didn't check for the *.idx, only the corresponding *.pack file.
About barfing on a failed *.idx file transfer: in 48188c2 and later, we
still do have ample checks here and there in the HTTP api. http_get_file()
returns a HTTP_ERROR if we fail to get a file (a *.idx file here).
The issue is that the check on the file (a HEAD request) fails, but not
its actual transfer (a GET request). There's no compromise on error
handling; we can live with a bad response to HEAD requests, but we will
still fail if we can't GET the file.
> Ahh, v1.6.3 ships with fetch_index() that checks CURLE_OK and returns an
> error(), but that is about .idx file, and it did not have the "do they
> have the corresponding .pack file?" check introduced by 48188c2
> (http-walker: verify remote packs, 2009-06-06), which is what makes the
> server give us 500 error.
Just to clarify: the "check" of CURLE_OK in http-walker.c::fetch_index()
in v1.6.3 is fundamentally different from the check we have in 48188c2
(http-walker: verify remote packs, 2009-06-06).
The first "check" is a full-blooded GET request, and we do get back
actual data (in this case, the pack index file). The second check isn't
a GET request, just an inconsequential HEAD request; we don't get back
any real data.
> Before that check, we ignored a request to
> fetch_index() if we already had one.
48188c2 didn't really remove the check (for the presence of the *.idx
locally), it just pushed it further down.
But being placed so late makes it rather ineffective and useless.
> Why do we even call fetch_index() when we have one? After all, we are
> talking about "git clone" here, so it is not about "we failed once and the
> previous attempt left .idx files we fetched". Why
>
> should we even have .idx file to begin with, that would have protected
> v1.6.3 clients from getting this error?
>
> Unless we are calling fetch_index() on the same .idx file twice from our
> own stupidity, that is.
>
> The same logic now is in fetch_pack_index(), which is called from
> fetch_and_setup_pack_index(). I do not still see how we end up calling
> the function for the same .idx file twice, though.
We should bump up this check before the verify-remote-pack one, I think
there's no mysterious bug here to find.
> In the meantime, I do not think it is a good idea to loosen the error
> checking on our side to accomodate a server in such a (hopefully
> temporary) broken state, however popular it is.
Agreed. I didn't intend my patch to loosen up error checking, merely to
be clearer about what we're looking for. If read in another context
(separate from fixing cloning over github.com), my patch can be seen as
one that clarifies the verify-remote-pack check:
Case 1: A 404 is received. The pack file is not available, so we stop.
Case 2: Our check failed, due to some reason (request failed,
unauthorized, etc). Nothing conclusive about availabilty of
file. Continue anyway.
PS. I wrote 48188c2 with the aim of having http-push.c and http-walker.c
fetch behaviour match each other as closely as possible, and later move
these out into http. At that time, I toyed with the idea of removing
the check. I wondered if the overhead incurred by the HEAD request was
useful, since if there's going to be an error, we would be hit anyway
with a GET. We already do this for almost every other http request
(info/refs, objects/info/packs, etc). But the patch series was meant to
be a refactor job, not a feature/behaviour change, so there you have it.
--
Cheers,
Ray Chuan
^ permalink raw reply
* Re: tracking branch for a rebase
From: Michael J Gruber @ 2009-09-07 9:06 UTC (permalink / raw)
To: Jeff King; +Cc: Junio C Hamano, Björn Steinbrink, Pete Wyckoff, git
In-Reply-To: <20090907084455.GC17997@coredump.intra.peff.net>
Jeff King venit, vidit, dixit 07.09.2009 10:44:
> On Mon, Sep 07, 2009 at 01:25:38AM -0700, Junio C Hamano wrote:
>
>>> @{^}
>>
>> This _could_ work, although it is rather cryptic.
>
> But an identifier composed entirely of punctuation? It might help
> git catch on with perl programmers. ;)
Right, that was the true agenda hidden only partially behind the plain
association between ^ and up ;)
Michael (having exactly zero perl fu)
P.S.: @ may not be reserved, and neither @^, but 1.7 should allow us to
reserve some symbols/combinations with a "low damage probability".
People may have a@b refnames (git-svn users for sure), but @^?
^ permalink raw reply
* [PATCHv2 2/2] Add url.<base>.pushInsteadOf: URL rewriting for push only
From: Josh Triplett @ 2009-09-07 8:56 UTC (permalink / raw)
To: git, gitster
In-Reply-To: <cover.1252313313.git.josh@joshtriplett.org>
This configuration option allows systematically rewriting fetch-only
URLs to push-capable URLs when used with push. For instance:
[url "ssh://example.org/"]
pushInsteadOf = "git://example.org/"
This will allow clones of "git://example.org/path/to/repo" to
subsequently push to "ssh://example.org/path/to/repo", without manually
configuring pushurl for that remote.
Includes documentation for the new option, bash completion updates, and
test cases (both that pushInsteadOf applies to push and that it does
*not* apply to fetch).
Signed-off-by: Josh Triplett <josh@joshtriplett.org>
---
Documentation/config.txt | 13 +++++++++++
Documentation/urls.txt | 18 ++++++++++++++++
contrib/completion/git-completion.bash | 2 +-
remote.c | 36 ++++++++++++++++++++++++--------
t/t5516-fetch-push.sh | 31 +++++++++++++++++++++++++++
5 files changed, 90 insertions(+), 10 deletions(-)
diff --git a/Documentation/config.txt b/Documentation/config.txt
index 5256c7f..38c7086 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -1500,6 +1500,19 @@ url.<base>.insteadOf::
never-before-seen repository on the site. When more than one
insteadOf strings match a given URL, the longest match is used.
+url.<base>.pushInsteadOf::
+ Any URL that starts with this value will not be pushed to;
+ instead, it will be rewritten to start with <base>, and the
+ resulting URL will be pushed to. In cases where some site serves
+ a large number of repositories, and serves them with multiple
+ access methods, some of which do not allow push, this feature
+ allows people to specify a pull-only URL and have git
+ automatically use an appropriate URL to push, even for a
+ never-before-seen repository on the site. When more than one
+ pushInsteadOf strings match a given URL, the longest match is
+ used. If a remote has an explicit pushurl, git will ignore this
+ setting for that remote.
+
user.email::
Your email address to be recorded in any newly created commits.
Can be overridden by the 'GIT_AUTHOR_EMAIL', 'GIT_COMMITTER_EMAIL', and
diff --git a/Documentation/urls.txt b/Documentation/urls.txt
index 5355ebc..d813ceb 100644
--- a/Documentation/urls.txt
+++ b/Documentation/urls.txt
@@ -67,3 +67,21 @@ For example, with this:
a URL like "work:repo.git" or like "host.xz:/path/to/repo.git" will be
rewritten in any context that takes a URL to be "git://git.host.xz/repo.git".
+If you want to rewrite URLs for push only, you can create a
+configuration section of the form:
+
+------------
+ [url "<actual url base>"]
+ pushInsteadOf = <other url base>
+------------
+
+For example, with this:
+
+------------
+ [url "ssh://example.org/"]
+ pushInsteadOf = git://example.org/
+------------
+
+a URL like "git://example.org/path/to/repo.git" will be rewritten to
+"ssh://example.org/path/to/repo.git" for pushes, but pulls will still
+use the original URL.
diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash
index bf688e1..9859204 100755
--- a/contrib/completion/git-completion.bash
+++ b/contrib/completion/git-completion.bash
@@ -1532,7 +1532,7 @@ _git_config ()
url.*.*)
local pfx="${cur%.*}."
cur="${cur##*.}"
- __gitcomp "insteadof" "$pfx" "$cur"
+ __gitcomp "insteadOf pushInsteadOf" "$pfx" "$cur"
return
;;
esac
diff --git a/remote.c b/remote.c
index ff8e71f..73d33f2 100644
--- a/remote.c
+++ b/remote.c
@@ -47,6 +47,7 @@ static const char *default_remote_name;
static int explicit_default_remote_name;
static struct rewrites rewrites;
+static struct rewrites rewrites_push;
#define BUF_SIZE (2048)
static char buffer[BUF_SIZE];
@@ -104,17 +105,25 @@ static void add_url(struct remote *remote, const char *url)
remote->url[remote->url_nr++] = url;
}
-static void add_url_alias(struct remote *remote, const char *url)
-{
- add_url(remote, alias_url(url, &rewrites));
-}
-
static void add_pushurl(struct remote *remote, const char *pushurl)
{
ALLOC_GROW(remote->pushurl, remote->pushurl_nr + 1, remote->pushurl_alloc);
remote->pushurl[remote->pushurl_nr++] = pushurl;
}
+static void add_pushurl_alias(struct remote *remote, const char *url)
+{
+ const char *pushurl = alias_url(url, &rewrites_push);
+ if (pushurl != url)
+ add_pushurl(remote, pushurl);
+}
+
+static void add_url_alias(struct remote *remote, const char *url)
+{
+ add_url(remote, alias_url(url, &rewrites));
+ add_pushurl_alias(remote, url);
+}
+
static struct remote *make_remote(const char *name, int len)
{
struct remote *ret;
@@ -358,8 +367,13 @@ static int handle_config(const char *key, const char *value, void *cb)
subkey = strrchr(name, '.');
if (!subkey)
return 0;
- rewrite = make_rewrite(&rewrites, name, subkey - name);
if (!strcmp(subkey, ".insteadof")) {
+ rewrite = make_rewrite(&rewrites, name, subkey - name);
+ if (!value)
+ return config_error_nonbool(key);
+ add_instead_of(rewrite, xstrdup(value));
+ } else if (!strcmp(subkey, ".pushinsteadof")) {
+ rewrite = make_rewrite(&rewrites_push, name, subkey - name);
if (!value)
return config_error_nonbool(key);
add_instead_of(rewrite, xstrdup(value));
@@ -433,14 +447,18 @@ static void alias_all_urls(void)
{
int i, j;
for (i = 0; i < remotes_nr; i++) {
+ int add_pushurl_aliases;
if (!remotes[i])
continue;
- for (j = 0; j < remotes[i]->url_nr; j++) {
- remotes[i]->url[j] = alias_url(remotes[i]->url[j], &rewrites);
- }
for (j = 0; j < remotes[i]->pushurl_nr; j++) {
remotes[i]->pushurl[j] = alias_url(remotes[i]->pushurl[j], &rewrites);
}
+ add_pushurl_aliases = remotes[i]->pushurl_nr == 0;
+ for (j = 0; j < remotes[i]->url_nr; j++) {
+ if (add_pushurl_aliases)
+ add_pushurl_alias(remotes[i], remotes[i]->url[j]);
+ remotes[i]->url[j] = alias_url(remotes[i]->url[j], &rewrites);
+ }
}
}
diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh
index 2d2633f..8f455c7 100755
--- a/t/t5516-fetch-push.sh
+++ b/t/t5516-fetch-push.sh
@@ -122,6 +122,23 @@ test_expect_success 'fetch with insteadOf' '
)
'
+test_expect_success 'fetch with pushInsteadOf (should not rewrite)' '
+ mk_empty &&
+ (
+ TRASH=$(pwd)/ &&
+ cd testrepo &&
+ git config "url.trash/.pushInsteadOf" "$TRASH" &&
+ git config remote.up.url "$TRASH." &&
+ git config remote.up.fetch "refs/heads/*:refs/remotes/origin/*" &&
+ git fetch up &&
+
+ r=$(git show-ref -s --verify refs/remotes/origin/master) &&
+ test "z$r" = "z$the_commit" &&
+
+ test 1 = $(git for-each-ref refs/remotes/origin | wc -l)
+ )
+'
+
test_expect_success 'push without wildcard' '
mk_empty &&
@@ -162,6 +179,20 @@ test_expect_success 'push with insteadOf' '
)
'
+test_expect_success 'push with pushInsteadOf' '
+ mk_empty &&
+ TRASH="$(pwd)/" &&
+ git config "url.$TRASH.pushInsteadOf" trash/ &&
+ git push trash/testrepo refs/heads/master:refs/remotes/origin/master &&
+ (
+ cd testrepo &&
+ r=$(git show-ref -s --verify refs/remotes/origin/master) &&
+ test "z$r" = "z$the_commit" &&
+
+ test 1 = $(git for-each-ref refs/remotes/origin | wc -l)
+ )
+'
+
test_expect_success 'push with matching heads' '
mk_test heads/master &&
--
1.6.3.3
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox