* [PATCH] reduce git-pack-objects memory usage a little more
@ 2007-07-12 21:07 Nicolas Pitre
2007-07-13 1:42 ` Brian Downing
0 siblings, 1 reply; 5+ messages in thread
From: Nicolas Pitre @ 2007-07-12 21:07 UTC (permalink / raw)
To: Junio C Hamano; +Cc: git
The delta depth doesn't have to be stored in the global object array
structure since it is only used during the deltification pass.
Signed-off-by: Nicolas Pitre <nico@cam.org>
---
diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index b4f3e7c..55609f3 100644
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
@@ -25,9 +25,6 @@ git-pack-objects [{ -q | --progress | --all-progress }] [--max-pack-size=N] \n\
struct object_entry {
struct pack_idx_entry idx;
unsigned long size; /* uncompressed size */
-
- unsigned int hash; /* name hint hash */
- unsigned int depth; /* delta depth */
struct packed_git *in_pack; /* already in pack */
off_t in_pack_offset;
struct object_entry *delta; /* delta base object */
@@ -37,6 +34,7 @@ struct object_entry {
*/
void *delta_data; /* cached delta (uncompressed) */
unsigned long delta_size; /* delta data size (uncompressed) */
+ unsigned int hash; /* name hint hash */
enum object_type type;
enum object_type in_pack_type; /* could be delta */
unsigned char in_pack_header_size;
@@ -1270,6 +1268,7 @@ struct unpacked {
struct object_entry *entry;
void *data;
struct delta_index *index;
+ unsigned depth;
};
static int delta_cacheable(struct unpacked *trg, struct unpacked *src,
@@ -1328,7 +1327,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
return 0;
/* Let's not bust the allowed depth. */
- if (src_entry->depth >= max_depth)
+ if (src->depth >= max_depth)
return 0;
/* Now some size filtering heuristics. */
@@ -1338,9 +1337,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
ref_depth = 1;
} else {
max_size = trg_entry->delta_size;
- ref_depth = trg_entry->depth;
+ ref_depth = trg->depth;
}
- max_size = max_size * (max_depth - src_entry->depth) /
+ max_size = max_size * (max_depth - src->depth) /
(max_depth - ref_depth + 1);
if (max_size == 0)
return 0;
@@ -1379,17 +1378,17 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
if (trg_entry->delta_data) {
/* Prefer only shallower same-sized deltas. */
if (delta_size == trg_entry->delta_size &&
- src_entry->depth + 1 >= trg_entry->depth) {
+ src->depth + 1 >= trg->depth) {
free(delta_buf);
return 0;
}
delta_cache_size -= trg_entry->delta_size;
free(trg_entry->delta_data);
+ trg_entry->delta_data = NULL;
}
- trg_entry->delta_data = 0;
trg_entry->delta = src_entry;
trg_entry->delta_size = delta_size;
- trg_entry->depth = src_entry->depth + 1;
+ trg->depth = src->depth + 1;
if (delta_cacheable(src, trg, src_size, trg_size, delta_size)) {
trg_entry->delta_data = xrealloc(delta_buf, delta_size);
@@ -1484,7 +1483,7 @@ static void find_deltas(struct object_entry **list, int window, int depth)
* depth, leaving it in the window is pointless. we
* should evict it first.
*/
- if (entry->delta && depth <= entry->depth)
+ if (entry->delta && depth <= n->depth)
continue;
next:
^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [PATCH] reduce git-pack-objects memory usage a little more
2007-07-12 21:07 [PATCH] reduce git-pack-objects memory usage a little more Nicolas Pitre
@ 2007-07-13 1:42 ` Brian Downing
2007-07-13 1:45 ` Brian Downing
0 siblings, 1 reply; 5+ messages in thread
From: Brian Downing @ 2007-07-13 1:42 UTC (permalink / raw)
To: Nicolas Pitre; +Cc: Junio C Hamano, git
On Thu, Jul 12, 2007 at 05:07:59PM -0400, Nicolas Pitre wrote:
> The delta depth doesn't have to be stored in the global object array
> structure since it is only used during the deltification pass.
This patch breaks pack-objects pretty horribly:
:; PATH=~/src/git:$PATH /usr/bin/time ~/src/git/git-repack -a -d -f
:; ls -l .git/objects/pack
total 153916
-r--r--r-- 1 bdowning bdowning 1312136 2007-07-12 20:39 pack-9ac926ee1f5810c434707d3f816f5ad2cbd14668.idx
-r--r--r-- 1 bdowning bdowning 156130933 2007-07-12 20:39 pack-9ac926ee1f5810c434707d3f816f5ad2cbd14668.pack
chain length = 1: 6182 objects
chain length = 2: 66 objects
chain length = 3: 27 objects
chain length = 4: 20 objects
chain length = 5: 15 objects
chain length = 6: 9 objects
chain length = 7: 5 objects
chain length = 8: 5 objects
chain length = 9: 6 objects
chain length = 10: 4 objects
chain length = 11: 6 objects
chain length = 12: 4 objects
chain length = 13: 3 objects
chain length = 14: 3 objects
chain length = 15: 2 objects
chain length = 16: 2 objects
chain length = 17: 1 object
chain length = 18: 1 object
chain length = 19: 1 object
Unfortunately I didn't notice until I tried v1.5.3-rc1...
-bcd
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH] reduce git-pack-objects memory usage a little more
2007-07-13 1:42 ` Brian Downing
@ 2007-07-13 1:45 ` Brian Downing
2007-07-13 2:27 ` [PATCH] forgot to clear the depth value Nicolas Pitre
0 siblings, 1 reply; 5+ messages in thread
From: Brian Downing @ 2007-07-13 1:45 UTC (permalink / raw)
To: Nicolas Pitre; +Cc: Junio C Hamano, git
On Thu, Jul 12, 2007 at 08:42:28PM -0500, Brian Downing wrote:
> This patch breaks pack-objects pretty horribly:
>
> :; PATH=~/src/git:$PATH /usr/bin/time ~/src/git/git-repack -a -d -f
[that's Git I'm repacking here]
> :; ls -l .git/objects/pack
> total 153916
> -r--r--r-- 1 bdowning bdowning 1312136 2007-07-12 20:39 pack-9ac926ee1f5810c434707d3f816f5ad2cbd14668.idx
> -r--r--r-- 1 bdowning bdowning 156130933 2007-07-12 20:39 pack-9ac926ee1f5810c434707d3f816f5ad2cbd14668.pack
>
> chain length = 1: 6182 objects
> chain length = 2: 66 objects
> chain length = 3: 27 objects
> chain length = 4: 20 objects
> chain length = 5: 15 objects
> chain length = 6: 9 objects
> chain length = 7: 5 objects
> chain length = 8: 5 objects
> chain length = 9: 6 objects
> chain length = 10: 4 objects
> chain length = 11: 6 objects
> chain length = 12: 4 objects
> chain length = 13: 3 objects
> chain length = 14: 3 objects
> chain length = 15: 2 objects
> chain length = 16: 2 objects
> chain length = 17: 1 object
> chain length = 18: 1 object
> chain length = 19: 1 object
-bcd
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH] forgot to clear the depth value
2007-07-13 1:45 ` Brian Downing
@ 2007-07-13 2:27 ` Nicolas Pitre
2007-07-13 3:10 ` Junio C Hamano
0 siblings, 1 reply; 5+ messages in thread
From: Nicolas Pitre @ 2007-07-13 2:27 UTC (permalink / raw)
To: Junio C Hamano, Brian Downing; +Cc: git
Commit 5a235b5e was missing this little detail. Otherwise your pack
will explode.
Problem noted by Brian Downing.
Signed-off-by: Nicolas Pitre <nico@cam.org>
---
On Thu, 12 Jul 2007, Brian Downing wrote:
> On Thu, Jul 12, 2007 at 08:42:28PM -0500, Brian Downing wrote:
> > This patch breaks pack-objects pretty horribly:
> >
> > :; PATH=~/src/git:$PATH /usr/bin/time ~/src/git/git-repack -a -d -f
>
> [that's Git I'm repacking here]
>
> > :; ls -l .git/objects/pack
> > total 153916
> > -r--r--r-- 1 bdowning bdowning 1312136 2007-07-12 20:39 pack-9ac926ee1f5810c434707d3f816f5ad2cbd14668.idx
> > -r--r--r-- 1 bdowning bdowning 156130933 2007-07-12 20:39 pack-9ac926ee1f5810c434707d3f816f5ad2cbd14668.pack
Oooooops.
diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index a43d604..5e9d1fd 100644
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
@@ -1431,6 +1431,7 @@ static void free_unpacked(struct unpacked *n)
window_memory_usage -= n->entry->size;
}
n->entry = NULL;
+ n->depth = 0;
}
static void find_deltas(struct object_entry **list, int window, int depth)
^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [PATCH] forgot to clear the depth value
2007-07-13 2:27 ` [PATCH] forgot to clear the depth value Nicolas Pitre
@ 2007-07-13 3:10 ` Junio C Hamano
0 siblings, 0 replies; 5+ messages in thread
From: Junio C Hamano @ 2007-07-13 3:10 UTC (permalink / raw)
To: Nicolas Pitre; +Cc: Brian Downing, git
Gaaah ..... thanks.
^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2007-07-13 3:11 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2007-07-12 21:07 [PATCH] reduce git-pack-objects memory usage a little more Nicolas Pitre
2007-07-13 1:42 ` Brian Downing
2007-07-13 1:45 ` Brian Downing
2007-07-13 2:27 ` [PATCH] forgot to clear the depth value Nicolas Pitre
2007-07-13 3:10 ` Junio C Hamano
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).