* [PATCH 3/3] verify-pack --stat-only: show histogram without verifying
From: Junio C Hamano @ 2009-08-08 3:36 UTC (permalink / raw)
To: git
In-Reply-To: <1249702594-7815-2-git-send-email-gitster@pobox.com>
When this option is given, the command does not verify the pack contents,
but shows the delta chain histogram. If used with --verbose, the usual
list of objects is also shown.
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
Documentation/git-verify-pack.txt | 8 +++++-
builtin-verify-pack.c | 49 +++++++++++++++++++++++++------------
2 files changed, 40 insertions(+), 17 deletions(-)
diff --git a/Documentation/git-verify-pack.txt b/Documentation/git-verify-pack.txt
index d791a80..97f7f91 100644
--- a/Documentation/git-verify-pack.txt
+++ b/Documentation/git-verify-pack.txt
@@ -25,7 +25,13 @@ OPTIONS
-v::
--verbose::
After verifying the pack, show list of objects contained
- in the pack.
+ in the pack and a histogram of delta chain length.
+
+-s::
+--stat-only::
+ Do not verify the pack contents; only show the histogram of delta
+ chain length. With `--verbose`, list of objects is also shown.
+
\--::
Do not interpret any more arguments as options.
diff --git a/builtin-verify-pack.c b/builtin-verify-pack.c
index b5bd28e..b6079ae 100644
--- a/builtin-verify-pack.c
+++ b/builtin-verify-pack.c
@@ -6,10 +6,14 @@
#define MAX_CHAIN 50
-static void show_pack_info(struct packed_git *p)
+#define VERIFY_PACK_VERBOSE 01
+#define VERIFY_PACK_STAT_ONLY 02
+
+static void show_pack_info(struct packed_git *p, unsigned int flags)
{
uint32_t nr_objects, i;
int cnt;
+ int stat_only = flags & VERIFY_PACK_STAT_ONLY;
unsigned long chain_histogram[MAX_CHAIN+1], baseobjects;
nr_objects = p->num_objects;
@@ -32,16 +36,19 @@ static void show_pack_info(struct packed_git *p)
type = packed_object_info_detail(p, offset, &size, &store_size,
&delta_chain_length,
base_sha1);
- printf("%s ", sha1_to_hex(sha1));
+ if (!stat_only)
+ printf("%s ", sha1_to_hex(sha1));
if (!delta_chain_length) {
- printf("%-6s %lu %lu %"PRIuMAX"\n",
- type, size, store_size, (uintmax_t)offset);
+ if (!stat_only)
+ printf("%-6s %lu %lu %"PRIuMAX"\n",
+ type, size, store_size, (uintmax_t)offset);
baseobjects++;
}
else {
- printf("%-6s %lu %lu %"PRIuMAX" %u %s\n",
- type, size, store_size, (uintmax_t)offset,
- delta_chain_length, sha1_to_hex(base_sha1));
+ if (!stat_only)
+ printf("%-6s %lu %lu %"PRIuMAX" %u %s\n",
+ type, size, store_size, (uintmax_t)offset,
+ delta_chain_length, sha1_to_hex(base_sha1));
if (delta_chain_length <= MAX_CHAIN)
chain_histogram[delta_chain_length]++;
else
@@ -66,10 +73,12 @@ static void show_pack_info(struct packed_git *p)
chain_histogram[0] > 1 ? "s" : "");
}
-static int verify_one_pack(const char *path, int verbose)
+static int verify_one_pack(const char *path, unsigned int flags)
{
char arg[PATH_MAX];
int len;
+ int verbose = flags & VERIFY_PACK_VERBOSE;
+ int stat_only = flags & VERIFY_PACK_STAT_ONLY;
struct packed_git *pack;
int err;
@@ -105,14 +114,19 @@ static int verify_one_pack(const char *path, int verbose)
return error("packfile %s not found.", arg);
install_packed_git(pack);
- err = verify_pack(pack);
- if (verbose) {
+ if (!stat_only)
+ err = verify_pack(pack);
+ else
+ err = open_pack_index(pack);
+
+ if (verbose || stat_only) {
if (err)
printf("%s: bad\n", pack->pack_name);
else {
- show_pack_info(pack);
- printf("%s: ok\n", pack->pack_name);
+ show_pack_info(pack, flags);
+ if (!stat_only)
+ printf("%s: ok\n", pack->pack_name);
}
}
@@ -120,17 +134,20 @@ static int verify_one_pack(const char *path, int verbose)
}
static const char * const verify_pack_usage[] = {
- "git verify-pack [-v|--verbose] <pack>...",
+ "git verify-pack [-v|--verbose] [-s|--stat-only] <pack>...",
NULL
};
int cmd_verify_pack(int argc, const char **argv, const char *prefix)
{
int err = 0;
- int verbose = 0;
+ unsigned int flags = 0;
int i;
const struct option verify_pack_options[] = {
- OPT__VERBOSE(&verbose),
+ OPT_BIT('v', "verbose", &flags, "verbose",
+ VERIFY_PACK_VERBOSE),
+ OPT_BIT('s', "stat-only", &flags, "show statistics only",
+ VERIFY_PACK_STAT_ONLY),
OPT_END()
};
@@ -140,7 +157,7 @@ int cmd_verify_pack(int argc, const char **argv, const char *prefix)
if (argc < 1)
usage_with_options(verify_pack_usage, verify_pack_options);
for (i = 0; i < argc; i++) {
- if (verify_one_pack(argv[i], verbose))
+ if (verify_one_pack(argv[i], flags))
err = 1;
discard_revindex();
}
--
1.6.4.151.g786db
^ permalink raw reply related
* Re: [PATCH 2/3] verify-pack -v: do not report "chain length 0"
From: Jeff King @ 2009-08-08 4:11 UTC (permalink / raw)
To: Junio C Hamano; +Cc: git
In-Reply-To: <1249702594-7815-2-git-send-email-gitster@pobox.com>
On Fri, Aug 07, 2009 at 08:36:33PM -0700, Junio C Hamano wrote:
> + if (baseobjects)
> + printf("non delta: %lu object%s\n",
> + baseobjects, baseobjects > 1 ? "s" : "");
Nit: this should be "non-delta" or "no delta".
-Peff
^ permalink raw reply
* Re: [PATCH 0/7] block-sha1: improved SHA1 hashing
From: Linus Torvalds @ 2009-08-08 4:16 UTC (permalink / raw)
To: Artur Skawina; +Cc: Git Mailing List
In-Reply-To: <4A7CC380.3070008@gmail.com>
On Sat, 8 Aug 2009, Artur Skawina wrote:
>
> i was seeing such large variations depending on the -mtune flags that
> i gave up and now do just -march=i686; that's what i would expect
> for generic x86 binaries.
I think I have found a way to avoid the gcc crazyness.
Lookie here:
# TIME[s] SPEED[MB/s]
rfc3174 5.094 119.8
rfc3174 5.098 119.7
linus 1.462 417.5
linusas 2.008 304
linusas2 1.878 325
mozilla 5.566 109.6
mozillaas 5.866 104.1
openssl 1.609 379.3
spelvin 1.675 364.5
spelvina 1.601 381.3
nettle 1.591 383.6
notice? I outperform all the hand-tuned asm on 32-bit too. By quite a
margin, in fact.
Now, I didn't try a P4, and it's possible that it won't do that there, but
the 32-bit code generation sure looks impressive on my Nehalem box. The
magic? I force the stores to the 512-bit hash bucket to be done in order.
That seems to help a lot.
The diff is trivial (on top of the "rename registers with cpp" patch), as
appended. And it does seem to fix the P4 issues too, although I can
obviously (once again) only test Prescott, and only in 64-bit mode:
# TIME[s] SPEED[MB/s]
rfc3174 1.662 36.73
rfc3174 1.64 37.22
linus 0.2523 241.9
linusas 0.4367 139.8
linusas2 0.4487 136
mozilla 0.9704 62.9
mozillaas 0.9399 64.94
that's some really impressive improvement. All from just saying "do the
stores in the order I told you to, dammit!" to the compiler.
Linus
---
block-sha1/sha1.c | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/block-sha1/sha1.c b/block-sha1/sha1.c
index 19dc41d..f70e1ba 100644
--- a/block-sha1/sha1.c
+++ b/block-sha1/sha1.c
@@ -93,6 +93,7 @@ void blk_SHA1_Final(unsigned char hashout[20], blk_SHA_CTX *ctx)
/* This "rolls" over the 512-bit array */
#define W(x) (array[(x)&15])
+#define setW(x, val) (*(volatile unsigned int *)&W(x) = (val))
/*
* Where do we get the source from? The first 16 iterations get it from
@@ -102,7 +103,7 @@ void blk_SHA1_Final(unsigned char hashout[20], blk_SHA_CTX *ctx)
#define SHA_MIX(t) SHA_ROL(W(t+13) ^ W(t+8) ^ W(t+2) ^ W(t), 1)
#define SHA_ROUND(t, input, fn, constant, A, B, C, D, E) do { \
- unsigned int TEMP = input(t); W(t) = TEMP; \
+ unsigned int TEMP = input(t); setW(t, TEMP); \
E += TEMP + SHA_ROL(A,5) + (fn) + (constant); \
B = SHA_ROR(B, 2); } while (0)
^ permalink raw reply related
* Re: [PATCH 0/7] block-sha1: improved SHA1 hashing
From: Artur Skawina @ 2009-08-08 5:34 UTC (permalink / raw)
To: Linus Torvalds; +Cc: Git Mailing List
In-Reply-To: <alpine.LFD.2.01.0908072107170.3288@localhost.localdomain>
Linus Torvalds wrote:
>
> I think I have found a way to avoid the gcc crazyness.
>
> Lookie here:
>
> # TIME[s] SPEED[MB/s]
> rfc3174 5.094 119.8
> rfc3174 5.098 119.7
> linus 1.462 417.5
> linusas 2.008 304
> linusas2 1.878 325
> mozilla 5.566 109.6
> mozillaas 5.866 104.1
> openssl 1.609 379.3
> spelvin 1.675 364.5
> spelvina 1.601 381.3
> nettle 1.591 383.6
>
> notice? I outperform all the hand-tuned asm on 32-bit too. By quite a
> margin, in fact.
>
> Now, I didn't try a P4, and it's possible that it won't do that there, but
> the 32-bit code generation sure looks impressive on my Nehalem box. The
> magic? I force the stores to the 512-bit hash bucket to be done in order.
> That seems to help a lot.
I named it 'linusv':
P4/i686:
# TIME[s] SPEED[MB/s]
rfc3174 1.456 41.92
rfc3174 1.445 42.22
linus 0.5865 104.1
linusph 0.5643 108.2
linusv 0.3697 165.1
linusvph 0.3618 168.7
linusp4 0.4312 141.5
linusas 0.4091 149.2
linusas2 0.4364 139.9
mozilla 1.102 55.37
mozillaas 1.297 47.07
openssl 0.261 233.9
opensslb 0.2395 254.9
spelvin 0.2653 230
nettle 0.438 139.4
and when tuning for prescott:
linus 0.6544 93.27
linusph 0.6523 93.57
linusv 0.3439 177.5
linusvph 0.3547 172.1
linusp4 0.3585 170.3
so it isn't as fast as the openssl asm ones, but it does win
in the C category.
> I outperform all the hand-tuned asm on 32-bit too. By quite a
> margin, in fact.
I've inlined the byteswapping in 'opensslb', maybe that one will
do a bit better.
http://www.src.multimo.pl/YDpqIo7Li27O0L0h/sha1bench.tar.gz
artur
^ permalink raw reply
* [PATCH] Check return value of ftruncate call in http.c. Remove possible mem leak
From: Jeff Lasslett @ 2009-08-08 5:38 UTC (permalink / raw)
To: git, gitster; +Cc: Jeff Lasslett
In new_http_object_request(), check ftruncate() call return value and
handle possible errors. Remove possible leak of mem pointed to by url.
Signed-off-by: Jeff Lasslett <jeff.lasslett@gmail.com>
---
http.c | 12 +++++++++---
1 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/http.c b/http.c
index a2720d5..34a3a35 100644
--- a/http.c
+++ b/http.c
@@ -1098,7 +1098,7 @@ struct http_object_request *new_http_object_request(const char *base_url,
char *hex = sha1_to_hex(sha1);
char *filename;
char prevfile[PATH_MAX];
- char *url;
+ char *url = NULL;
int prevlocal;
unsigned char prev_buf[PREV_BUF_SIZE];
ssize_t prev_read = 0;
@@ -1189,7 +1189,11 @@ struct http_object_request *new_http_object_request(const char *base_url,
if (prev_posn>0) {
prev_posn = 0;
lseek(freq->localfile, 0, SEEK_SET);
- ftruncate(freq->localfile, 0);
+ if (ftruncate(freq->localfile, 0) < 0) {
+ error("Couldn't truncate temporary file %s for %s: %s",
+ freq->tmpfile, freq->filename, strerror(errno));
+ goto abort;
+ }
}
}
@@ -1216,10 +1220,12 @@ struct http_object_request *new_http_object_request(const char *base_url,
CURLOPT_HTTPHEADER, range_header);
}
+ free(url);
return freq;
- free(url);
abort:
+ free(url);
+ free(freq->url);
free(filename);
free(freq);
return NULL;
--
1.6.4.59.g4d590.dirty
^ permalink raw reply related
* [PATCH v2 0/3] commit --dry-run
From: Junio C Hamano @ 2009-08-08 7:14 UTC (permalink / raw)
To: git
This is a re-roll of the commit in the jc/shortstatus series that has been
kept out of 'pu'.
First, we introduce "git commit --dry-run", which does exactly the same
thing as the current "git status". A longer term goal of this change is
so that we could make "git status" behave completely differently after an
ample warning period.
The second commit introduces a new "git status" under a tentative name
"git stat". In this step, it does not do anything interesting, other than
that it ignores the path parameters.
The third commit adds "-s" option to the new command, to give the "short"
output format.
Probably the next round should insert a commit between the second one and
the third one to take pathspec parameters that are used as path limiters,
but I am too tired for doing that myself tonight.
Junio C Hamano (3):
commit: --dry-run
git stat: the beginning
git stat -s: short status output
Documentation/git-commit.txt | 7 ++-
Makefile | 1 +
builtin-commit.c | 163 ++++++++++++++++++++++++++++++++++++------
builtin.h | 1 +
git.c | 1 +
wt-status.c | 24 ++++---
wt-status.h | 1 +
7 files changed, 165 insertions(+), 33 deletions(-)
^ permalink raw reply
* [PATCH v2 2/3] git stat: the beginning
From: Junio C Hamano @ 2009-08-08 7:14 UTC (permalink / raw)
To: git
In-Reply-To: <1249715693-9715-2-git-send-email-gitster@pobox.com>
Tentatively add "git stat" as a new command. This does not munge the
index with paths parameters before showing the status like "git status"
does. In later rounds, we will take path parameters as pathspec to limit
the output.
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
Makefile | 1 +
builtin-commit.c | 56 ++++++++++++++++++++++++++++++++++++++++++++---------
builtin.h | 1 +
git.c | 1 +
wt-status.c | 24 ++++++++++++++--------
wt-status.h | 1 +
6 files changed, 65 insertions(+), 19 deletions(-)
diff --git a/Makefile b/Makefile
index daf4296..39dd334 100644
--- a/Makefile
+++ b/Makefile
@@ -378,6 +378,7 @@ BUILT_INS += git-init$X
BUILT_INS += git-merge-subtree$X
BUILT_INS += git-peek-remote$X
BUILT_INS += git-repo-config$X
+BUILT_INS += git-stat$X
BUILT_INS += git-show$X
BUILT_INS += git-stage$X
BUILT_INS += git-status$X
diff --git a/builtin-commit.c b/builtin-commit.c
index 3a7e35d..10f9d4f 100644
--- a/builtin-commit.c
+++ b/builtin-commit.c
@@ -35,6 +35,11 @@ static const char * const builtin_status_usage[] = {
NULL
};
+static const char * const builtin_stat_usage[] = {
+ "git stat [options]",
+ NULL
+};
+
static unsigned char head_sha1[20], merge_head_sha1[20];
static char *use_message_buffer;
static const char commit_editmsg[] = "COMMIT_EDITMSG";
@@ -693,6 +698,21 @@ static const char *find_author_by_nickname(const char *name)
die("No existing author found with '%s'", name);
}
+
+static void handle_untracked_files_arg(void)
+{
+ if (!untracked_files_arg)
+ ; /* default already initialized */
+ else if (!strcmp(untracked_files_arg, "no"))
+ show_untracked_files = SHOW_NO_UNTRACKED_FILES;
+ else if (!strcmp(untracked_files_arg, "normal"))
+ show_untracked_files = SHOW_NORMAL_UNTRACKED_FILES;
+ else if (!strcmp(untracked_files_arg, "all"))
+ show_untracked_files = SHOW_ALL_UNTRACKED_FILES;
+ else
+ die("Invalid untracked files mode '%s'", untracked_files_arg);
+}
+
static int parse_and_validate_options(int argc, const char *argv[],
const char * const usage[],
const char *prefix)
@@ -795,16 +815,7 @@ static int parse_and_validate_options(int argc, const char *argv[],
else
die("Invalid cleanup mode %s", cleanup_arg);
- if (!untracked_files_arg)
- ; /* default already initialized */
- else if (!strcmp(untracked_files_arg, "no"))
- show_untracked_files = SHOW_NO_UNTRACKED_FILES;
- else if (!strcmp(untracked_files_arg, "normal"))
- show_untracked_files = SHOW_NORMAL_UNTRACKED_FILES;
- else if (!strcmp(untracked_files_arg, "all"))
- show_untracked_files = SHOW_ALL_UNTRACKED_FILES;
- else
- die("Invalid untracked files mode '%s'", untracked_files_arg);
+ handle_untracked_files_arg();
if (all && argc > 0)
die("Paths with -a does not make sense.");
@@ -814,6 +825,31 @@ static int parse_and_validate_options(int argc, const char *argv[],
return argc;
}
+int cmd_stat(int argc, const char **argv, const char *prefix)
+{
+ struct wt_status s;
+ static struct option builtin_stat_options[] = {
+ { OPTION_STRING, 'u', "untracked-files", &untracked_files_arg,
+ "mode",
+ "show untracked files, optional modes: all, normal, no. (Default: all)",
+ PARSE_OPT_OPTARG, NULL, (intptr_t)"all" },
+ OPT_END(),
+ };
+
+ argc = parse_options(argc, argv, prefix,
+ builtin_stat_options,
+ builtin_stat_usage, 0);
+ handle_untracked_files_arg();
+
+ read_cache();
+ refresh_cache(REFRESH_QUIET|REFRESH_UNMERGED);
+ wt_status_prepare(&s);
+ wt_status_collect_changes(&s);
+
+ wt_status_print_body(&s);
+ return 0;
+}
+
static int dry_run_commit(int argc, const char **argv, const char *prefix)
{
int commitable;
diff --git a/builtin.h b/builtin.h
index 20427d2..eeaf0b6 100644
--- a/builtin.h
+++ b/builtin.h
@@ -95,6 +95,7 @@ extern int cmd_send_pack(int argc, const char **argv, const char *prefix);
extern int cmd_shortlog(int argc, const char **argv, const char *prefix);
extern int cmd_show(int argc, const char **argv, const char *prefix);
extern int cmd_show_branch(int argc, const char **argv, const char *prefix);
+extern int cmd_stat(int argc, const char **argv, const char *prefix);
extern int cmd_status(int argc, const char **argv, const char *prefix);
extern int cmd_stripspace(int argc, const char **argv, const char *prefix);
extern int cmd_symbolic_ref(int argc, const char **argv, const char *prefix);
diff --git a/git.c b/git.c
index 807d875..de7fcf6 100644
--- a/git.c
+++ b/git.c
@@ -350,6 +350,7 @@ static void handle_internal_command(int argc, const char **argv)
{ "shortlog", cmd_shortlog, USE_PAGER },
{ "show-branch", cmd_show_branch, RUN_SETUP },
{ "show", cmd_show, RUN_SETUP | USE_PAGER },
+ { "stat", cmd_stat, RUN_SETUP | NEED_WORK_TREE },
{ "status", cmd_status, RUN_SETUP | NEED_WORK_TREE },
{ "stripspace", cmd_stripspace },
{ "symbolic-ref", cmd_symbolic_ref, RUN_SETUP },
diff --git a/wt-status.c b/wt-status.c
index 97fedfa..6eb32df 100644
--- a/wt-status.c
+++ b/wt-status.c
@@ -537,6 +537,20 @@ static void wt_status_print_tracking(struct wt_status *s)
color_fprintf_ln(s->fp, color(WT_STATUS_HEADER), "#");
}
+void wt_status_print_body(struct wt_status *s)
+{
+ wt_status_print_unmerged(s);
+ wt_status_print_updated(s);
+ wt_status_print_changed(s);
+ if (wt_status_submodule_summary)
+ wt_status_print_submodule_summary(s);
+ if (show_untracked_files)
+ wt_status_print_untracked(s);
+ else if (s->commitable)
+ fprintf(s->fp,
+ "# Untracked files not listed (use -u option to show untracked files)\n");
+}
+
void wt_status_print(struct wt_status *s)
{
unsigned char sha1[20];
@@ -567,15 +581,7 @@ void wt_status_print(struct wt_status *s)
color_fprintf_ln(s->fp, color(WT_STATUS_HEADER), "#");
}
- wt_status_print_unmerged(s);
- wt_status_print_updated(s);
- wt_status_print_changed(s);
- if (wt_status_submodule_summary)
- wt_status_print_submodule_summary(s);
- if (show_untracked_files)
- wt_status_print_untracked(s);
- else if (s->commitable)
- fprintf(s->fp, "# Untracked files not listed (use -u option to show untracked files)\n");
+ wt_status_print_body(s);
if (s->verbose)
wt_status_print_verbose(s);
diff --git a/wt-status.h b/wt-status.h
index f80142f..ebbf06a 100644
--- a/wt-status.h
+++ b/wt-status.h
@@ -50,6 +50,7 @@ extern int wt_status_use_color;
extern int wt_status_relative_paths;
void wt_status_prepare(struct wt_status *s);
void wt_status_print(struct wt_status *s);
+void wt_status_print_body(struct wt_status *s);
void wt_status_collect_changes(struct wt_status *s);
#endif /* STATUS_H */
--
1.6.4.159.gd0d612
^ permalink raw reply related
* [PATCH v2 1/3] commit: --dry-run
From: Junio C Hamano @ 2009-08-08 7:14 UTC (permalink / raw)
To: git
In-Reply-To: <1249715693-9715-1-git-send-email-gitster@pobox.com>
This teaches --dry-run option to "git commit".
It is the same as "git status", but in the longer term we would want to
change the semantics of "git status" not to be the preview of commit, and
this is the first step for doing so.
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
Documentation/git-commit.txt | 7 ++++++-
builtin-commit.c | 29 ++++++++++++++++-------------
2 files changed, 22 insertions(+), 14 deletions(-)
diff --git a/Documentation/git-commit.txt b/Documentation/git-commit.txt
index b5d81be..d01ff5a 100644
--- a/Documentation/git-commit.txt
+++ b/Documentation/git-commit.txt
@@ -8,7 +8,7 @@ git-commit - Record changes to the repository
SYNOPSIS
--------
[verse]
-'git commit' [-a | --interactive] [-s] [-v] [-u<mode>] [--amend]
+'git commit' [-a | --interactive] [-s] [-v] [-u<mode>] [--amend] [--dry-run]
[(-c | -C) <commit>] [-F <file> | -m <msg>]
[--allow-empty] [--no-verify] [-e] [--author=<author>]
[--cleanup=<mode>] [--] [[-i | -o ]<file>...]
@@ -198,6 +198,11 @@ specified.
--quiet::
Suppress commit summary message.
+--dry-run::
+ Do not create a commit, but show a list of paths that are
+ to be committed, paths with local changes that will be left
+ uncommitted and paths that are untracked.
+
\--::
Do not interpret any more arguments as options.
diff --git a/builtin-commit.c b/builtin-commit.c
index 6d12c2e..3a7e35d 100644
--- a/builtin-commit.c
+++ b/builtin-commit.c
@@ -51,7 +51,7 @@ static const char *template_file;
static char *edit_message, *use_message;
static char *author_name, *author_email, *author_date;
static int all, edit_flag, also, interactive, only, amend, signoff;
-static int quiet, verbose, no_verify, allow_empty;
+static int quiet, verbose, no_verify, allow_empty, dry_run;
static char *untracked_files_arg;
/*
* The default commit message cleanup mode will remove the lines
@@ -103,6 +103,7 @@ static struct option builtin_commit_options[] = {
OPT_BOOLEAN(0, "interactive", &interactive, "interactively add files"),
OPT_BOOLEAN('o', "only", &only, "commit only specified files"),
OPT_BOOLEAN('n', "no-verify", &no_verify, "bypass pre-commit hook"),
+ OPT_BOOLEAN(0, "dry-run", &dry_run, "show what would be committed"),
OPT_BOOLEAN(0, "amend", &amend, "amend previous commit"),
{ OPTION_STRING, 'u', "untracked-files", &untracked_files_arg, "mode", "show untracked files, optional modes: all, normal, no. (Default: all)", PARSE_OPT_OPTARG, NULL, (intptr_t)"all" },
OPT_BOOLEAN(0, "allow-empty", &allow_empty, "ok to record an empty change"),
@@ -813,28 +814,28 @@ static int parse_and_validate_options(int argc, const char *argv[],
return argc;
}
-int cmd_status(int argc, const char **argv, const char *prefix)
+static int dry_run_commit(int argc, const char **argv, const char *prefix)
{
- const char *index_file;
int commitable;
+ const char *index_file;
- git_config(git_status_config, NULL);
+ index_file = prepare_index(argc, argv, prefix, 1);
+ commitable = run_status(stdout, index_file, prefix, 0);
+ rollback_index_files();
+ return commitable ? 0 : 1;
+}
+
+int cmd_status(int argc, const char **argv, const char *prefix)
+{
+ git_config(git_status_config, NULL);
if (wt_status_use_color == -1)
wt_status_use_color = git_use_color_default;
-
if (diff_use_color_default == -1)
diff_use_color_default = git_use_color_default;
argc = parse_and_validate_options(argc, argv, builtin_status_usage, prefix);
-
- index_file = prepare_index(argc, argv, prefix, 1);
-
- commitable = run_status(stdout, index_file, prefix, 0);
-
- rollback_index_files();
-
- return commitable ? 0 : 1;
+ return dry_run_commit(argc, argv, prefix);
}
static void print_summary(const char *prefix, const unsigned char *sha1)
@@ -909,6 +910,8 @@ int cmd_commit(int argc, const char **argv, const char *prefix)
wt_status_use_color = git_use_color_default;
argc = parse_and_validate_options(argc, argv, builtin_commit_usage, prefix);
+ if (dry_run)
+ return dry_run_commit(argc, argv, prefix);
index_file = prepare_index(argc, argv, prefix, 0);
--
1.6.4.159.gd0d612
^ permalink raw reply related
* [PATCH v2 3/3] git stat -s: short status output
From: Junio C Hamano @ 2009-08-08 7:14 UTC (permalink / raw)
To: git
In-Reply-To: <1249715693-9715-3-git-send-email-gitster@pobox.com>
Give -s(hort) option to "git stat" that shows the status of paths in a
more concise way.
XY PATH1 -> PATH2
format to be more machine readable than output from "git status", which is
about previewing of "git commit" with the same arguments.
PATH1 is the path in the HEAD, and " -> PATH2" part is shown only when
PATH1 corresponds to a different path in the index/worktree.
For unmerged entries, X shows the status of stage #2 (i.e. ours) and Y
shows the status of stage #3 (i.e. theirs). For entries that do not have
conflicts, X shows the status of the index, and Y shows the status of the
work tree.
X Y Meaning
-------------------------------------------------
[MD] not updated
M [ MD] updated in index
A [ MD] added to index
D [ MD] deleted from index
R [ MD] renamed in index
C [ MD] copied in index
[MARC] index and work tree matches
[ MARC] M work tree changed since index
[ MARC] D deleted in work tree
D D unmerged, both deleted
A U unmerged, added by us
U D unmerged, deleted by them
U A unmerged, added by them
D U unmerged, deleted by us
A A unmerged, both added
U U unmerged, both modified
When given -z option, the records are terminated by NUL characters for
better machine readability.
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
builtin-commit.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 files changed, 79 insertions(+), 1 deletions(-)
diff --git a/builtin-commit.c b/builtin-commit.c
index 10f9d4f..abb4377 100644
--- a/builtin-commit.c
+++ b/builtin-commit.c
@@ -24,6 +24,7 @@
#include "string-list.h"
#include "rerere.h"
#include "unpack-trees.h"
+#include "quote.h"
static const char * const builtin_commit_usage[] = {
"git commit [options] [--] <filepattern>...",
@@ -825,10 +826,72 @@ static int parse_and_validate_options(int argc, const char *argv[],
return argc;
}
+#define quote_path quote_path_relative
+
+static void show_unmerged(int null_termination, struct string_list_item *it,
+ struct wt_status *s)
+{
+ struct wt_status_change_data *d = it->util;
+ const char *how = "??";
+
+ switch (d->stagemask) {
+ case 1: how = "DD"; break; /* both deleted */
+ case 2: how = "AU"; break; /* added by us */
+ case 3: how = "UD"; break; /* deleted by them */
+ case 4: how = "UA"; break; /* added by them */
+ case 5: how = "DU"; break; /* deleted by us */
+ case 6: how = "AA"; break; /* both added */
+ case 7: how = "UU"; break; /* both modified */
+ }
+ printf("%s ", how);
+ if (null_termination) {
+ fprintf(stdout, "%s%c", it->string, 0);
+ } else {
+ struct strbuf onebuf = STRBUF_INIT;
+ const char *one;
+ one = quote_path(it->string, -1, &onebuf, s->prefix);
+ printf("%s\n", one);
+ strbuf_release(&onebuf);
+ }
+}
+
+static void show_status(int null_termination, struct string_list_item *it,
+ struct wt_status *s)
+{
+ struct wt_status_change_data *d = it->util;
+
+ printf("%c%c ",
+ !d->index_status ? ' ' : d->index_status,
+ !d->worktree_status ? ' ' : d->worktree_status);
+ if (null_termination) {
+ fprintf(stdout, "%s%c", it->string, 0);
+ if (d->head_path)
+ fprintf(stdout, "%s%c", d->head_path, 0);
+ } else {
+ struct strbuf onebuf = STRBUF_INIT;
+ const char *one;
+ if (d->head_path) {
+ one = quote_path(d->head_path, -1, &onebuf, s->prefix);
+ printf("%s -> ", one);
+ strbuf_release(&onebuf);
+ }
+ one = quote_path(it->string, -1, &onebuf, s->prefix);
+ printf("%s\n", one);
+ strbuf_release(&onebuf);
+ }
+}
+
int cmd_stat(int argc, const char **argv, const char *prefix)
{
struct wt_status s;
+ static int null_termination, shortstatus;
+ int i;
+ unsigned char sha1[20];
static struct option builtin_stat_options[] = {
+ OPT_BOOLEAN('s', "short", &shortstatus,
+ "show status concicely"),
+ OPT_BOOLEAN('z', "null", &null_termination,
+ "terminate entries with NUL"),
{ OPTION_STRING, 'u', "untracked-files", &untracked_files_arg,
"mode",
"show untracked files, optional modes: all, normal, no. (Default: all)",
@@ -844,9 +907,24 @@ int cmd_stat(int argc, const char **argv, const char *prefix)
read_cache();
refresh_cache(REFRESH_QUIET|REFRESH_UNMERGED);
wt_status_prepare(&s);
+ s.is_initial = get_sha1(s.reference, sha1) ? 1 : 0;
wt_status_collect_changes(&s);
- wt_status_print_body(&s);
+ if (shortstatus) {
+ for (i = 0; i < s.change.nr; i++) {
+ struct wt_status_change_data *d;
+ struct string_list_item *it;
+
+ it = &(s.change.items[i]);
+ d = it->util;
+ if (d->stagemask)
+ show_unmerged(null_termination, it, &s);
+ else
+ show_status(null_termination, it, &s);
+ }
+ } else {
+ wt_status_print_body(&s);
+ }
return 0;
}
--
1.6.4.159.gd0d612
^ permalink raw reply related
* Re: [PATCH v2 0/3] commit --dry-run
From: Sverre Rabbelier @ 2009-08-08 7:18 UTC (permalink / raw)
To: Junio C Hamano; +Cc: git
In-Reply-To: <1249715693-9715-1-git-send-email-gitster@pobox.com>
Heya,
On Sat, Aug 8, 2009 at 00:14, Junio C Hamano<gitster@pobox.com> wrote:
> The second commit introduces a new "git status" under a tentative name
> "git stat".
That would put 'git status' at 'git statu<tab>', instead of 'git
stat<tab>', I don't know if that's a concern though.
--
Cheers,
Sverre Rabbelier
^ permalink raw reply
* Re: [PATCH 0/5] Suggested for PU: revision caching system to significantly speed up packing/walking
From: Nick Edelen @ 2009-08-08 7:27 UTC (permalink / raw)
To: Junio C Hamano
Cc: Nicolas Pitre, Johannes Schindelin, Sam Vilain, Michael J Gruber,
Jeff King, Shawn O. Pearce, Andreas Ericsson, Christian Couder,
git@vger.kernel.org
In-Reply-To: <7vfxc3ov8t.fsf@alter.siamese.dyndns.org>
> IIRC from previous discussions, kernel.org's main performance problem is
> I/O, not CPU. Are there any provisions for sharing rev-caches between
> similar repositories, as we already do for objects?
I haven't implemented a transmission protocol or anything, but it
would be perfectly possible to copy cache slices from one repo to
another. Generating the revision cache from scratch on large repos
can take several minutes, so this wouldn't be a bad idea.
> That depends primarily on how heavily the patches needed to change in
> response to review comments, but until the series lands in 'next', you
> would typically send updated series as a replacement, not incremental.
>
> Many people seemed to be interested in the series and had a volume of
> comments on it. I suspect the updated series would be quite different
> from the original, so for the next round I would suspect it would be best
> to start anew, marking them as [PATCH N/M (v2)], in a fresh thread. It
> would help reviewers if you said "this corresponds to [PATCH 3/5] in the
> original series, with the following improvements based on X and Y's
> comments" after the three-dash line.
Ok, that sounds good. I've added a new patch as well so the numbering changes.
^ permalink raw reply
* Re: [PATCH 0/5] Suggested for PU: revision caching system to significantly speed up packing/walking
From: Jeff King @ 2009-08-08 7:30 UTC (permalink / raw)
To: Nick Edelen
Cc: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber, Shawn O. Pearce, Andreas Ericsson,
Christian Couder, git@vger.kernel.org
In-Reply-To: <c77435a80908080027p6a8de468w68983bd81364cefd@mail.gmail.com>
On Sat, Aug 08, 2009 at 09:27:55AM +0200, Nick Edelen wrote:
> > IIRC from previous discussions, kernel.org's main performance problem is
> > I/O, not CPU. Are there any provisions for sharing rev-caches between
> > similar repositories, as we already do for objects?
>
> I haven't implemented a transmission protocol or anything, but it
> would be perfectly possible to copy cache slices from one repo to
> another. Generating the revision cache from scratch on large repos
> can take several minutes, so this wouldn't be a bad idea.
That might be useful, but I was thinking more of an "alternates"-like
mechanism between repos. So that the data is stored only once on disk
and in the disk cache, which is helpful for sites like kernel.org which
serve many similar repositories.
-Peff
^ permalink raw reply
* [PATCH 0/6 (v2)] Suggested for PU: revision caching system to significantly speed up packing/walking
From: Nick Edelen @ 2009-08-08 7:31 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
SUGGESTED FOR 'PU': (patchset v2)
Traversing objects is currently very costly, as every commit and tree must be
loaded and parsed. Much time and energy could be saved by caching metadata and
topological info in an efficient, easily accessible manner. Furthermore, this
could improve git's interfacing potential, by providing a condensed summary of
a repository's commit tree.
This is a series to implement such a revision caching mechanism, aptly named
rev-cache. The series will provide:
- a core API to manipulate and traverse caches
- an integration into the internal revision walker
- a porcelain front-end providing access to users and (shell) applications
- a series of tests to verify/demonstrate correctness
- documentation of the API, porcelain and core concepts
In cold starts rev-cache has sped up packing and walking by a factor of 4, and
over twice that on warm starts. Some times on slax for the linux repository:
rev-list --all --objects >/dev/null
default
cold 1:13
warm 0:43
rev-cache'd
cold 0:19
warm 0:02
pack-objects --revs --all --stdout >/dev/null
default
cold 2:44
warm 1:21
rev-cache'd
cold 0:44
warm 0:10
The mechanism is minimally intrusive: most of the changes take place in
seperate files, and only a handful of git's existing functions are modified.
Hope you find this useful.
- Nick
---
This is a revised patchset; the biggest changes are:
- elimination of python dependency
- revision of documentation
- addition of name caching (patch 6)
Documentation/git-rev-cache.txt | 96 ++
Documentation/technical/rev-cache.txt | 379 ++++++
Makefile | 2 +
blob.c | 1 +
blob.h | 2 +
builtin-rev-cache.c | 309 +++++
builtin.h | 1 +
commit.c | 3 +
commit.h | 2 +
git.c | 1 +
list-objects.c | 49 +-
rev-cache.c | 2041 +++++++++++++++++++++++++++++++++
revision.c | 89 ++-
revision.h | 47 +-
t/t6015-rev-cache-list.sh | 246 ++++
tree.h | 1 +
16 files changed, 3244 insertions(+), 25 deletions(-)
create mode 100755 Documentation/git-rev-cache.txt
create mode 100755 Documentation/technical/rev-cache.txt
create mode 100755 builtin-rev-cache.c
create mode 100755 rev-cache.c
create mode 100755 t/t6015-rev-cache-list.sh
^ permalink raw reply
* [PATCH 1/6 (v2)] revision caching documentation: man page and technical discussion
From: Nick Edelen @ 2009-08-08 7:31 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
Before any code is introduced the full documentation is put forth. This
provides a man page for the porcelain, and a technical doc in technical/. The
latter describes the API, and discusses rev-cache's design, file format and
mechanics.
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
Same as 1/5 in previous set. The man page has been flushed out, and formatted
more congruently with git's other man pages. The technical discussion has been
updated and tweaked for better formatting as well. This has been in response
to Sam's commentary.
Documentation/git-rev-cache.txt | 96 +++++++++
Documentation/technical/rev-cache.txt | 379 +++++++++++++++++++++++++++++++++
2 files changed, 475 insertions(+), 0 deletions(-)
diff --git a/Documentation/git-rev-cache.txt b/Documentation/git-rev-cache.txt
new file mode 100755
index 0000000..600cf64
--- /dev/null
+++ b/Documentation/git-rev-cache.txt
@@ -0,0 +1,96 @@
+git-rev-cache(1)
+================
+
+NAME
+----
+git-rev-cache - Add, walk and maintain revision cache slices.
+
+SYNOPSIS
+--------
+'git-rev-cache' COMMAND [options] [<commit>...]
+
+DESCRIPTION
+-----------
+A front end for the rev-cache API. It is mainly intended for cache slice
+generation and maintenance, but can also walk commits within a slice. It
+currently provides basic administrative functionality.
+
+COMMANDS
+--------
+
+add
+~~~
+Add revisions to the cache by creating a new cache slice. Reads a revision
+list from the command line, formatted as: `START START ... \--not END END ...`
+
+Options:
+
+\--all::
+ Include all branch heads in the new cache slice.
+
+\--fresh::
+ Exclude everything already in the revision cache.
+
+\--stdin::
+ Read newline-seperated revisions from the standard input. Use \--not
+ to exclude commits, as on the command line.
+
+\--legs::
+ Ensure newly-generated cache slice is self-contained. This means that
+ no commit has partially cached parents, in that all its parents are
+ cached or none of them are.
+
+\--noobjects::
+ Non-commit objects are normally included along with the commit with
+ which they were introduced. This is obviously very benificial, but can
+ take longer in cache slice generation. Use this option to disable
+ non-commit object caching.
+
+walk
+~~~~
+Walk a cache slice based on revision list, formatted like 'add'.
+
+Options:
+
+\--objects::
+ Like 'git-rev-list', 'walk' will normally only list commits. Use this
+ option to list non-commit objects as well.
+
+fuse
+~~~~
+Merge several cache slices into a single large slice. On each invocation of
+'add' a new file ("slice") is added to the revision cache directory, and after
+several additions the directory may become populated with many, relatively
+small slices. Numerous smaller slices will yield poorer performance than a one
+or two large ones, because of the overhead of loading new slices into memory.
+
+Running 'fuse' every once in a while will solve this problem by coalescing all
+the cache slices into one larger slice. For very large projects, using
+\--ignore-size is advisable to prevent overly large cache slices.
+
+Options:
+
+\--all::
+ Normally fuse will only include everything that's already in the
+ revision cache. \--add tells it to start walking from the branch
+ heads, effectively an `add --all --fresh; fuse` (pseudo-command).
+
+\--noobjects::
+ As in 'add', this option disables inclusion of non-commit objects. If
+ some cache slices do contain such objects, the information will be lost.
+
+\--ignore-size[=N]::
+ Do not merge cache slices of size >=N. If N is not provided 'fuse'
+ will default to a size of ~25MB (be aware that slices must be mapped to
+ memory).
+
+index
+~~~~~
+Regenerate the cache index. If the index file associating objects with cache
+slices gets corrupted, lost, or otherwise becomes unusable, 'index' will
+quickly regenerate the file. 'fuse' will also regenerate the index.
+
+DISCUSSION
+----------
+For an explanation of the API and its inner workings, see
+link:technical/rev-cache.txt[technical info on rev-cache].
diff --git a/Documentation/technical/rev-cache.txt b/Documentation/technical/rev-cache.txt
new file mode 100755
index 0000000..78d7218
--- /dev/null
+++ b/Documentation/technical/rev-cache.txt
@@ -0,0 +1,379 @@
+rev-cache
+=========
+
+The revision cache API ('rev-cache') provides a method for efficiently storing
+and accessing commit branch sections. Such branch slices are defined by a
+series of top (interesting) and bottom (uninteresting) commits. Each slice
+contains, per commit:
+
+* All intra-slice topological relations, encoded into path "channels" (see
+ 'Mechanics' for full explanation).
+* Object meta-data: type, SHA-1, size, date (for commits).
+* Objects introduced in that commit, relative to slice (ie. only for non-start
+ commits).
+
+Storage data structures are not exported, in part to keep git's global scope
+clean, but largely because they're pretty much useless outside of rev-cache.
+
+The API
+-------
+
+The API for 'rev-cache' is quite simple. You can find the function prototypes
+in `revision.h`.
+
+Data Structures
+~~~~~~~~~~~~~~~
+
+The `rev_cache_info` struct holds all the options and flags for the API.
+
+----
+struct rev_cache_info {
+ /* generation flags */
+ unsigned objects : 1,
+ legs : 1,
+ make_index : 1;
+
+ /* traversal flags */
+ unsigned save_unique : 1,
+ add_to_pending : 1,
+ add_names : 1;
+
+ /* fuse options */
+ unsigned int ignore_size;
+};
+----
+
+The fields:
+
+`objects`::
+ Add non-commit objects to slice.
+
+`legs`::
+ Ensure end commits have no children.
+
+`make_index`::
+ Integrate newly-made slice into index.
+
+`save_unique`::
+ Load non-commit objects into `unique` field of their associated
+ `commit` object.
+
+`add_to_pending`::
+ Append unique non-commit objects to the `pending` object list in the
+ passed `rev_info` instance.
+
+`add_names`::
+ Include non-commit object names in the pending object entries (if
+ `add_to_pending` is set), and set the `name` field of `blob` and `tree`
+ objects (if `save_unique`).
+
+`ignore_size`::
+ If non-zero, ignore slices with size greater or equal to this during
+fusion.
+
+Functions
+~~~~~~~~~
+
+`init_rci`::
+ Initiate a `rev_cache_info` struct to default options.
+
+`make_cache_slice`::
+ Create a cache based on an a `rev_info` instance or `commit_list` s of
+ "starts" and "ends" (defaulting to latter if `rev_info` pointer is
+ NULL), copying the cache SHA-1 into a passed pointer if non-zero. A
+ `rev_cache_info` struct pointer can be passed to set options, while
+ passing NULL will set default options. A last parameter can
+ optionally recieve the final cache hash.
+
+`make_cache_index`::
+ Add a slice to the cache index. Requires a file descriptor, the cache
+ hash and the file size. Note that this is normally called by
+ `make_cache_slice` under the `make_index` option.
+
+`get_cache_slice`::
+ Given a commit SHA-1 `get_cache_slice` will search the slice index and
+ return, if found, the cache-identifying SHA-1.
+
+`traverse_cache_slice`::
+ Traverse a specified cache slice based on:
+
+ * `rev_info` instance; options set in the `rev_cache_info` field.
+ * cache SHA-1 identifier
+ * a starting commit and commit work list
+ * date of oldest encountered interesting commit (passing 0 will tell
+ rev-cache to use defaults)
+ * current `slop` (this is a counter, a la `revision.c`, to determine
+ when to stop traversing; typically you can just pass 0 and rev-cache
+ will use default settings)
+
++
+The output is sent to a FILO `commit_list` "queue", while any end commits
+are passed back into the work list. If the walk is not contained within the
+slice, commit boundaries are also inserted into "work".
+
+`start_from_slices`::
+ Will mark all start-commits in the specified cache slices with a given
+ flag, and add them to the rev pending list. Will include all if no
+ slices are specified.
+
+`coagulate_cache_slices`::
+ Generate a slice based on the passed `rev_info` instance, replacing all
+ encountered slices with one (larger) slice. The `ignore_size` field in
+ `rci`, if non-zero, will dictate which cache slice sizes to ignore in
+ both traversal and replacement.
+
+`regenerate_index`::
+ Remake cache index.
+
+Example Usage
+-------------
+
+A few examples to demonstrate usage:
+
+.Creating a slice
+----
+/* pretend you're a porcelain for rev-cache reading from the command line */
+struct rev_info revs;
+struct rev_cache_info rci;
+
+init_revisions(&revs, 0);
+init_rci(&rci);
+
+flags = 0;
+for (i = 1; i < argc; i++) {
+ if (!strcmp(argv[i], "--not"))
+ flags ^= UNINTERESTING;
+ else if(!strcmp(argv[i], "--fresh"))
+ starts_from_slices(&revs, UNINTERESTING, 0, 0);
+ else
+ handle_revision_arg(argv[i], &revs, flags, 1);
+}
+
+/* we want to explicitly set certain options */
+rci.objects = 0;
+
+if (!make_cache_slice(&rci, &revs, 0, 0, cache_sha1))
+ printf("made slice! it's called %s\n", sha1_to_hex(cache_sha1));
+----
+
+.Traversing a slice
+----
+/* let's say you're walking the tree with a 'work' list of current heads and a
+ * FILO output list 'out' */
+out = 0;
+outp = &out;
+
+while (work) {
+ struct commit *commit = pop_commit(&work);
+ struct object *object = &commit->object;
+ unsigned char *cache_sha1;
+
+ if (cache_sha1 = get_cache_slice(object->sha1)) {
+ /* note that this will instatiate any topo-relations
+ * as it goes */
+ if (traverse_cache_slice(&revs, cache_sha1,
+ commit, 0, 0, /* use defaults */
+ &outp, &work) < 0)
+ die("I'm overreacting to a non-fatal cache error");
+ } else {
+ struct commit_list *parents = commit->parents;
+
+ while (parents) {
+ struct commit *p = parents->item;
+ struct object *po = &p->object;
+
+ parents = parents->next;
+ if (po->flags & UNINTERESTING)
+ continue;
+
+ if (object->flags & UNINTERESTING)
+ po->flags |= UNINTERESTING;
+ else if (po->flags & SEEN)
+ continue;
+
+ if (!po->parsed)
+ parse_commit(p);
+ insert_by_date(p, &work);
+ }
+
+ if (object->flags & (SEEN | UNINTERESTING) == 0)
+ outp = &commit_list_insert(commit, outp);
+ object->flags |= SEEN;
+ }
+}
+----
+
+Some Internals
+--------------
+
+Although you really don't need to know anything about how rev-cache actually
+does its magic shizz, a bit of background may go a long way if you're wading
+through the source.
+
+File Formats
+~~~~~~~~~~~~
+
+A slice has a basic fixed-size header, followed by a certain number of object
+entries, then a NULL-seperated list of object names. Commits are sorted in
+topo-order, and each commit entry is followed by the objects added in that
+commit.
+
+----
+ -- +--------------------------------+
+header | object number, etc... |
+ -- +--------------------------------+
+commit | commit info |
+entry | path data |
+ +--------------------------------+
+ | tree/blob info |
+ +--------------------------------+
+ | tree/blob info |
+ +--------------------------------+
+ | ... |
+ -- +--------------------------------+
+commit | commit info |
+entry | path data |
+ +--------------------------------+
+ | tree/blob info |
+ +--------------------------------+
+ | ... |
+ -- +--------------------------------+
+... ...
+ -- +--------------------------------+
+name list | \0some_file_name\0 |
+(note +--------------------------------+
+preceeding | another_file\0 |
+null) ... |
+ +--------------------------------+
+----
+
+The index is a single file that associates SHA-1s with cache slices and file
+positions. It is somewhat similar to pack-file indexes, containing a fanout
+table and a list of index entries sorted by hash.
+
+----
+ -- +--------------------------------+
+header | object #, cache #, etc. |
+ -- +--------------------------------+
+sha1s of | SHA-1 |
+slices | ... |
+ -- +--------------------------------+
+fanout | fanout[0x00] |
+table ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ | fanout[0xff] |
+ -- +--------------------------------+
+index | SHA-1 of object |
+entries | index of cache slice SHA-1 |
+ | position in cache slice |
+ +--------------------------------+
+ | |
+ ...
+ +--------------------------------+
+----
+
+All the relavant structures are readily accessible in `rev-cache.c`
+
+Mechanics
+~~~~~~~~~
+
+The most important part of rev-cache is its method of encoding topological
+relations. To ensure fluid traversal and reconstruction, commits are related
+through high-level "streams"/"channels" rather than individual
+interconnections. Intuitively, rev-cache stores history the way gitk shows it:
+commits strung up on lines, which interconnect at merges and branches.
+
+Each commit is associated to a given channel/path via a 'path id', and
+variable-length fields govern which paths (if any) are closed or opened at that
+object. This means that topo-data can be preserved in only a few bytes extra
+per object entry. Other information stored per entry is the sha-1 hash, type,
+date, size, name, and status in cache slice. Here is format of an object
+entry, both on-disk and in-memory:
+
+----
+struct object_entry {
+ unsigned type : 3;
+ unsigned is_end : 1;
+ unsigned is_start : 1;
+ unsigned uninteresting : 1;
+ unsigned include : 1;
+ unsigned flags : 1;
+ unsigned char sha1[20];
+
+ unsigned merge_nr : 6;
+ unsigned split_nr : 7;
+ unsigned size_size : 3;
+ unsigned name_size : 3;
+ unsigned what_to_do : 5;
+
+ uint32_t date;
+ uint16_t path;
+
+ /* merge paths */
+ /* split paths */
+ /* size */
+ /* name index */
+};
+----
+
+An explanation of each field:
+
+`type`::
+ Object type
+`is_end`::
+ The commit has some parents outside the cache slice (all if slice has
+ legs)
+`is_start`::
+ The commit has no children in cache slice
+`uninteresting`::
+ Run-time flag, used in traversal
+`include`::
+ Run-time flag, used in traversal (initialization)
+`flags`::
+ Currently unused, extra bit
+`sha`::
+ Object SHA-1 hash
+
+`merge_nr`::
+ The number of paths the current channel diverges into; the current path
+ ends upon any merge.
+`split_nr`::
+ The number of paths this commit ends; used on both merging and
+ branching.
+`size_size`::
+ Number of bytes the object size takes up.
+`name_size`::
+ Number of bytes the name index takes up.
+`what_to_do`::
+ Currently space-filler, to properly align the struct fields. Can
+ potentially be used for anything.
+
+merge paths::
+ The path IDs (16-bit) that are to be created.
+split paths::
+ The path IDs (16-bit) that are to be ended.
+size::
+ The size split into the minimum number of bytes.
+name index::
+ An offset for the null-seperated, object name list at the end of the
+ cache slice. Also split into the minimum number of bytes.
+
+Each path ID refers to an index in a 'path array', which stores the current
+status (eg. active, interestingness) of each channel.
+
+Due to topo-relations and boundary tracking, all of a commit's parents must be
+encountered before the path is reallocated. This is achieved by using a
+counter system per merge: starting at the parent number, the counter is
+decremented as each parent is encountered (dictated by 'split paths'); at 0 the
+path is cleared.
+
+Boundary tracking is necessary because non-commits are stored relative to the
+commit in which they were introduced. If a series of commits is not included
+in the output, the last interesting commit must be parsed manually to ensure
+all objects are accounted for.
+
+To prevent list-objects from recursing into trees that we've already taken care
+of, the flag `FACE_VALUE` is introduced. An object with this flag is not
+explored (= "taken at face value"), significantly reducing I/O and processing
+time.
+
+(NSE)
--
tg: (6f604e7..) t/revcache/docs (depends on: t/revcache/integration)
^ permalink raw reply related
* [PATCH 2/6 (v2)] bare minimum revision cache system, no integration with git
From: Nick Edelen @ 2009-08-08 7:31 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
Second in the revision cache series, this particular patch provides:
- minimal API: caching only commit topo data
- minimal porcelain: add and walk cache slices
- appropriate tests
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
Corresponds to 2/5 in previous patchset. The python test file has been
eliminated, as discussion by Johannes and Micheal.
Makefile | 2 +
builtin-rev-cache.c | 197 ++++++++
builtin.h | 1 +
commit.c | 4 +-
git.c | 1 +
rev-cache.c | 1145 +++++++++++++++++++++++++++++++++++++++++++++
revision.c | 2 +-
revision.h | 44 ++-
t/t6015-rev-cache-list.sh | 104 ++++
9 files changed, 1497 insertions(+), 3 deletions(-)
diff --git a/Makefile b/Makefile
index daf4296..386700a 100644
--- a/Makefile
+++ b/Makefile
@@ -533,6 +533,7 @@ LIB_OBJS += reflog-walk.o
LIB_OBJS += refs.o
LIB_OBJS += remote.o
LIB_OBJS += rerere.o
+LIB_OBJS += rev-cache.o
LIB_OBJS += revision.o
LIB_OBJS += run-command.o
LIB_OBJS += server-info.o
@@ -623,6 +624,7 @@ BUILTIN_OBJS += builtin-reflog.o
BUILTIN_OBJS += builtin-remote.o
BUILTIN_OBJS += builtin-rerere.o
BUILTIN_OBJS += builtin-reset.o
+BUILTIN_OBJS += builtin-rev-cache.o
BUILTIN_OBJS += builtin-rev-list.o
BUILTIN_OBJS += builtin-rev-parse.o
BUILTIN_OBJS += builtin-revert.o
diff --git a/builtin-rev-cache.c b/builtin-rev-cache.c
new file mode 100755
index 0000000..2338871
--- /dev/null
+++ b/builtin-rev-cache.c
@@ -0,0 +1,197 @@
+#include "cache.h"
+#include "object.h"
+#include "commit.h"
+#include "diff.h"
+#include "revision.h"
+
+/* porcelain for rev-cache.c */
+static int handle_add(int argc, const char *argv[]) /* args beyond this command */
+{
+ struct rev_info revs;
+ struct rev_cache_info rci;
+ char dostdin = 0;
+ unsigned int flags = 0;
+ int i, retval;
+ unsigned char cache_sha1[20];
+
+ init_revisions(&revs, 0);
+ init_rci(&rci);
+
+ for (i = 0; i < argc; i++) {
+ if (!strcmp(argv[i], "--stdin"))
+ dostdin = 1;
+ else if (!strcmp(argv[i], "--fresh"))
+ starts_from_slices(&revs, UNINTERESTING);
+ else if (!strcmp(argv[i], "--not"))
+ flags ^= UNINTERESTING;
+ else if (!strcmp(argv[i], "--legs"))
+ rci.legs = 1;
+ else if (!strcmp(argv[i], "--noobjects"))
+ rci.objects = 0;
+ else if (!strcmp(argv[i], "--all")) {
+ const char *args[2];
+ int argn = 0;
+
+ args[argn++] = "rev-list";
+ args[argn++] = "--all";
+ setup_revisions(argn, args, &revs, 0);
+ } else
+ handle_revision_arg(argv[i], &revs, flags, 1);
+ }
+
+ if (dostdin) {
+ char line[1000];
+
+ flags = 0;
+ while (fgets(line, sizeof(line), stdin)) {
+ int len = strlen(line);
+ while (len && (line[len - 1] == '\n' || line[len - 1] == '\r'))
+ line[--len] = 0;
+
+ if (!len)
+ break;
+
+ if (!strcmp(line, "--not"))
+ flags ^= UNINTERESTING;
+ else
+ handle_revision_arg(line, &revs, flags, 1);
+ }
+ }
+
+ retval = make_cache_slice(&rci, &revs, 0, 0, cache_sha1);
+ if (retval < 0)
+ return retval;
+
+ printf("%s\n", sha1_to_hex(cache_sha1));
+
+ return 0;
+}
+
+static int handle_walk(int argc, const char *argv[])
+{
+ struct commit *commit;
+ struct rev_info revs;
+ struct commit_list *queue, *work, **qp;
+ unsigned char *sha1p, *sha1pt;
+ unsigned long date = 0;
+ unsigned int flags = 0;
+ int retval, slop = 5, i;
+
+ init_revisions(&revs, 0);
+
+ for (i = 0; i < argc; i++) {
+ if (!strcmp(argv[i], "--not"))
+ flags ^= UNINTERESTING;
+ else if (!strcmp(argv[i], "--objects"))
+ revs.tree_objects = revs.blob_objects = 1;
+ else
+ handle_revision_arg(argv[i], &revs, flags, 1);
+ }
+
+ work = 0;
+ sha1p = 0;
+ for (i = 0; i < revs.pending.nr; i++) {
+ commit = lookup_commit(revs.pending.objects[i].item->sha1);
+
+ sha1pt = get_cache_slice(commit);
+ if (!sha1pt)
+ die("%s: not in a cache slice", sha1_to_hex(commit->object.sha1));
+
+ if (!i)
+ sha1p = sha1pt;
+ else if (sha1p != sha1pt)
+ die("walking porcelain is /per/ cache slice; commits cannot be spread out amoung several");
+
+ insert_by_date(commit, &work);
+ }
+
+ if (!sha1p)
+ die("nothing to traverse!");
+
+ queue = 0;
+ qp = &queue;
+ commit = pop_commit(&work);
+ retval = traverse_cache_slice(&revs, sha1p, commit, &date, &slop, &qp, &work);
+ if (retval < 0)
+ return retval;
+
+ printf("queue:\n");
+ while ((commit = pop_commit(&queue)) != 0) {
+ printf("%s\n", sha1_to_hex(commit->object.sha1));
+ }
+
+ printf("work:\n");
+ while ((commit = pop_commit(&work)) != 0) {
+ printf("%s\n", sha1_to_hex(commit->object.sha1));
+ }
+
+ printf("pending:\n");
+ for (i = 0; i < revs.pending.nr; i++) {
+ struct object *obj = revs.pending.objects[i].item;
+
+ /* unfortunately, despite our careful generation, object duplication *is* a possibility...
+ * (eg. same object introduced into two different branches) */
+ if (obj->flags & SEEN)
+ continue;
+
+ printf("%s\n", sha1_to_hex(revs.pending.objects[i].item->sha1));
+ obj->flags |= SEEN;
+ }
+
+ return 0;
+}
+
+static int handle_help(void)
+{
+ char *usage = "\
+usage:\n\
+git-rev-cache COMMAND [options] [<commit-id>...]\n\
+commands:\n\
+ add - add revisions to the cache. reads commit ids from stdin, \n\
+ formatted as: START START ... --not END END ...\n\
+ options:\n\
+ --all use all branch heads as starts\n\
+ --fresh exclude everything already in a cache slice\n\
+ --stdin also read commit ids from stdin (same form as cmd)\n\
+ --legs ensure branch is entirely self-contained\n\
+ --noobjects don't add non-commit objects to slice\n\
+ walk - walk a cache slice based on set of commits; formatted as add\n\
+ options:\n\
+ --objects include non-commit objects in traversals\n\
+ fuse - coagulate cache slices into a single cache.\n\
+ options:\n\
+ --all include all objects in repository\n\
+ --noobjects don't add non-commit objects to slice\n\
+ --ignore-size[=N] ignore slices of size >= N; defaults to ~5MB\n\
+ index - regnerate the cache index.";
+
+ puts(usage);
+
+ return 0;
+}
+
+int cmd_rev_cache(int argc, const char *argv[], const char *prefix)
+{
+ const char *arg;
+ int r;
+
+ git_config(git_default_config, NULL);
+
+ if (argc > 1)
+ arg = argv[1];
+ else
+ arg = "";
+
+ argc -= 2;
+ argv += 2;
+ if (!strcmp(arg, "add"))
+ r = handle_add(argc, argv);
+ else if (!strcmp(arg, "walk"))
+ r = handle_walk(argc, argv);
+ else
+ return handle_help();
+
+ fprintf(stderr, "final return value: %d\n", r);
+
+ return 0;
+}
diff --git a/builtin.h b/builtin.h
index 20427d2..00ecc9c 100644
--- a/builtin.h
+++ b/builtin.h
@@ -87,6 +87,7 @@ extern int cmd_remote(int argc, const char **argv, const char *prefix);
extern int cmd_config(int argc, const char **argv, const char *prefix);
extern int cmd_rerere(int argc, const char **argv, const char *prefix);
extern int cmd_reset(int argc, const char **argv, const char *prefix);
+extern int cmd_rev_cache(int argc, const char **argv, const char *prefix);
extern int cmd_rev_list(int argc, const char **argv, const char *prefix);
extern int cmd_rev_parse(int argc, const char **argv, const char *prefix);
extern int cmd_revert(int argc, const char **argv, const char *prefix);
diff --git a/commit.c b/commit.c
index e2bcbe8..65d223e 100644
--- a/commit.c
+++ b/commit.c
@@ -251,7 +251,9 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
sha1_to_hex(item->object.sha1));
item->tree = lookup_tree(parent);
bufptr += 46; /* "tree " + "hex sha1" + "\n" */
- pptr = &item->parents;
+ pptr = &item->parents;
+ while (pop_commit(pptr))
+ ; /* clear anything from cache */
graft = lookup_commit_graft(item->object.sha1);
while (bufptr + 48 < tail && !memcmp(bufptr, "parent ", 7)) {
diff --git a/git.c b/git.c
index 807d875..da3a9fb 100644
--- a/git.c
+++ b/git.c
@@ -342,6 +342,7 @@ static void handle_internal_command(int argc, const char **argv)
{ "repo-config", cmd_config },
{ "rerere", cmd_rerere, RUN_SETUP },
{ "reset", cmd_reset, RUN_SETUP },
+ { "rev-cache", cmd_rev_cache, RUN_SETUP },
{ "rev-list", cmd_rev_list, RUN_SETUP },
{ "rev-parse", cmd_rev_parse },
{ "revert", cmd_revert, RUN_SETUP | NEED_WORK_TREE },
diff --git a/rev-cache.c b/rev-cache.c
new file mode 100755
index 0000000..5d9e150
--- /dev/null
+++ b/rev-cache.c
@@ -0,0 +1,1145 @@
+#include "cache.h"
+#include "object.h"
+#include "commit.h"
+#include "tree.h"
+#include "tree-walk.h"
+#include "blob.h"
+#include "tag.h"
+#include "diff.h"
+#include "revision.h"
+#include "run-command.h"
+
+
+/* single index maps objects to cache files */
+struct index_header {
+ char signature[8]; /* REVINDEX */
+ unsigned char version;
+ uint32_t ofs_objects;
+
+ uint32_t object_nr;
+ unsigned char cache_nr;
+
+ uint32_t max_date;
+};
+
+struct index_entry {
+ unsigned char sha1[20];
+ unsigned is_start : 1;
+ unsigned cache_index : 7;
+ uint32_t pos;
+};
+
+
+/* structure for actual cache file */
+struct cache_slice_header {
+ char signature[8]; /* REVCACHE */
+ unsigned char version;
+ uint32_t ofs_objects;
+
+ uint32_t object_nr;
+ uint16_t path_nr;
+ uint32_t size;
+
+ unsigned char sha1[20];
+};
+
+struct object_entry {
+ unsigned type : 3;
+ unsigned is_end : 1;
+ unsigned is_start : 1;
+ unsigned uninteresting : 1;
+ unsigned include : 1;
+ unsigned flags : 1; /* unused */
+ unsigned char sha1[20];
+
+ unsigned merge_nr : 6;
+ unsigned split_nr : 7;
+ unsigned size_size : 3;
+
+ uint32_t date;
+ uint16_t path;
+
+ /* merge paths */
+ /* split paths */
+ /* size */
+};
+
+/* list resembles pack index format */
+static uint32_t fanout[0xff + 2];
+
+static unsigned char *idx_map;
+static int idx_size;
+static struct index_header idx_head;
+static unsigned char *idx_caches;
+static char no_idx;
+
+static struct strbuf *g_buffer;
+
+#define SUPPORTED_REVCACHE_VERSION 1
+#define SUPPORTED_REVINDEX_VERSION 1
+
+#define PATH_WIDTH sizeof(uint16_t)
+#define PATH_SIZE(x) (PATH_WIDTH * (x))
+
+#define OE_SIZE sizeof(struct object_entry)
+#define IE_SIZE sizeof(struct index_entry)
+
+#define OE_CAST(p) ((struct object_entry *)(p))
+#define IE_CAST(p) ((struct index_entry *)(p))
+
+#define ACTUAL_OBJECT_ENTRY_SIZE(e) (OE_SIZE + PATH_SIZE((e)->merge_nr + (e)->split_nr) + (e)->size_size)
+
+#define SLOP 5
+
+/* initialization */
+
+static int get_index_head(unsigned char *map, int len, struct index_header *head, uint32_t *fanout, unsigned char **caches)
+{
+ struct index_header whead;
+ int i, index = sizeof(struct index_header);
+
+ memcpy(&whead, map, sizeof(struct index_header));
+ if (memcmp(whead.signature, "REVINDEX", 8) || whead.version > SUPPORTED_REVINDEX_VERSION)
+ return -1;
+
+ memcpy(head->signature, "REVINDEX", 8);
+ head->version = whead.version;
+ head->ofs_objects = ntohl(whead.ofs_objects);
+ head->object_nr = ntohl(whead.object_nr);
+ head->cache_nr = whead.cache_nr;
+ head->max_date = ntohl(whead.max_date);
+
+ if (len < index + head->cache_nr * 20 + 0x100 * sizeof(uint32_t))
+ return -2;
+
+ *caches = xmalloc(head->cache_nr * 20);
+ memcpy(*caches, map + index, head->cache_nr * 20);
+ index += head->cache_nr * 20;
+
+ memcpy(fanout, map + index, 0x100 * sizeof(uint32_t));
+ for (i = 0; i <= 0xff; i++)
+ fanout[i] = ntohl(fanout[i]);
+ fanout[0x100] = len;
+
+ return 0;
+}
+
+/* added in init_index */
+static void cleanup_cache_slices(void)
+{
+ if (idx_map) {
+ free(idx_caches);
+ munmap(idx_map, idx_size);
+ idx_map = 0;
+ }
+
+}
+
+static int init_index(void)
+{
+ int fd;
+ struct stat fi;
+
+ fd = open(git_path("rev-cache/index"), O_RDONLY);
+ if (fd == -1 || fstat(fd, &fi))
+ goto end;
+ if (fi.st_size < sizeof(struct index_header))
+ goto end;
+
+ idx_size = fi.st_size;
+ idx_map = xmmap(0, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ close(fd);
+ if (idx_map == MAP_FAILED)
+ goto end;
+ if (get_index_head(idx_map, fi.st_size, &idx_head, fanout, &idx_caches))
+ goto end;
+
+ atexit(cleanup_cache_slices);
+
+ return 0;
+
+end:
+ idx_map = 0;
+ no_idx = 1;
+ return -1;
+}
+
+/* this assumes index is already loaded */
+static struct index_entry *search_index(unsigned char *sha1)
+{
+ int start, end, starti, endi, i, len, r;
+ struct index_entry *ie;
+
+ if (!idx_map)
+ return 0;
+
+ /* binary search */
+ start = fanout[(int)sha1[0]];
+ end = fanout[(int)sha1[0] + 1];
+ len = (end - start) / IE_SIZE;
+ if (!len || len * IE_SIZE != end - start)
+ return 0;
+
+ starti = 0;
+ endi = len - 1;
+ for (;;) {
+ i = (endi + starti) / 2;
+ ie = IE_CAST(idx_map + start + i * IE_SIZE);
+ r = hashcmp(sha1, ie->sha1);
+
+ if (r) {
+ if (starti + 1 == endi) {
+ starti++;
+ continue;
+ } else if (starti == endi)
+ break;
+
+ if (r > 0)
+ starti = i;
+ else /* r < 0 */
+ endi = i;
+ } else
+ return ie;
+ }
+
+ return 0;
+}
+
+unsigned char *get_cache_slice(struct commit *commit)
+{
+ struct index_entry *ie;
+
+ if (!idx_map) {
+ if (no_idx)
+ return 0;
+ init_index();
+ }
+
+ if (commit->date > idx_head.max_date)
+ return 0;
+
+ ie = search_index(commit->object.sha1);
+
+ if (ie && ie->cache_index < idx_head.cache_nr)
+ return idx_caches + ie->cache_index * 20;
+
+ return 0;
+}
+
+
+/* traversal */
+
+static int setup_traversal(struct cache_slice_header *head, unsigned char *map, struct commit *commit, struct commit_list **work)
+{
+ struct index_entry *iep;
+ struct object_entry *oep;
+ struct commit_list *prev, *wp, **wpp;
+ int retval;
+
+ /* printf("adding %s\n", sha1_to_hex(commit->object.sha1)); */
+ iep = search_index(commit->object.sha1);
+ oep = OE_CAST(map + ntohl(iep->pos));
+ oep->include = 1;
+ retval = ntohl(iep->pos);
+
+ /* include any others in the work array */
+ prev = 0;
+ wpp = work;
+ wp = *work;
+ while (wp) {
+ struct object *obj = &wp->item->object;
+ struct commit *co;
+ int t;
+
+ /* is this in our cache slice? */
+ iep = search_index(obj->sha1);
+ if (!iep || hashcmp(idx_caches + iep->cache_index * 20, head->sha1)) {
+ prev = wp;
+ wp = wp->next;
+ wpp = ℘
+ continue;
+ }
+
+ t = ntohl(iep->pos);
+ oep = OE_CAST(map + t);
+
+ oep->include = 1;
+ oep->uninteresting = !!(obj->flags & UNINTERESTING);
+ if (t < retval)
+ retval = t;
+
+ /* remove from work list */
+ co = pop_commit(wpp);
+ wp = *wpp;
+ if (prev)
+ prev->next = wp;
+ }
+
+ return retval;
+}
+
+#define IPATH 0x40
+#define UPATH 0x80
+
+#define GET_COUNT(x) ((x) & 0x3f)
+#define SET_COUNT(x, s) ((x) = ((x) & ~0x3f) | ((s) & 0x3f))
+
+static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char *map,
+ struct rev_info *revs, struct commit *commit,
+ unsigned long *date_so_far, int *slop_so_far,
+ struct commit_list ***queue, struct commit_list **work)
+{
+ struct commit_list *insert_cache = 0;
+ struct commit **last_objects, *co;
+ int i, total_path_nr = head->path_nr, retval = -1;
+ char consume_children = 0;
+ unsigned char *paths;
+
+ paths = xcalloc(total_path_nr, PATH_WIDTH);
+ last_objects = xcalloc(total_path_nr, sizeof(struct commit *));
+
+ i = setup_traversal(head, map, commit, work);
+
+ /* i already set */
+ while (i < head->size) {
+ struct object_entry *entry = OE_CAST(map + i);
+ int path = ntohs(entry->path);
+ struct object *obj;
+ int index = i;
+
+ i += ACTUAL_OBJECT_ENTRY_SIZE(entry);
+
+ /* add extra objects if necessary */
+ if (entry->type != OBJ_COMMIT)
+ continue;
+ else
+ consume_children = 0;
+
+ if (path >= total_path_nr)
+ goto end;
+
+ /* in one of our branches?
+ * uninteresting trumps interesting */
+ if (entry->include)
+ paths[path] |= entry->uninteresting ? UPATH : IPATH;
+ else if (!paths[path])
+ continue;
+
+ /* date stuff */
+ if (revs->max_age != -1 && ntohl(entry->date) < revs->max_age)
+ paths[path] |= UPATH;
+
+ /* lookup object */
+ co = lookup_commit(entry->sha1);
+ obj = &co->object;
+
+ if (obj->flags & UNINTERESTING)
+ paths[path] |= UPATH;
+
+ if ((paths[path] & IPATH) && (paths[path] & UPATH)) {
+ paths[path] = UPATH;
+
+ /* mark edge */
+ if (last_objects[path]) {
+ parse_commit(last_objects[path]);
+
+ last_objects[path]->object.flags &= ~FACE_VALUE;
+ last_objects[path] = 0;
+ }
+ }
+
+ /* now we gotta re-assess the whole interesting thing... */
+ entry->uninteresting = !!(paths[path] & UPATH);
+
+ /* first close paths */
+ if (entry->split_nr) {
+ int j, off = index + OE_SIZE + PATH_SIZE(entry->merge_nr);
+
+ for (j = 0; j < entry->split_nr; j++) {
+ unsigned short p = ntohs(*(unsigned short *)(map + off + PATH_SIZE(j)));
+
+ if (p >= total_path_nr)
+ goto end;
+
+ /* boundary commit? */
+ if ((paths[p] & IPATH) && entry->uninteresting) {
+ if (last_objects[p]) {
+ parse_commit(last_objects[p]);
+
+ last_objects[p]->object.flags &= ~FACE_VALUE;
+ last_objects[p] = 0;
+ }
+ } else if (last_objects[p] && !last_objects[p]->object.parsed)
+ commit_list_insert(co, &last_objects[p]->parents);
+
+ /* can't close a merge path until all are parents have been encountered */
+ if (GET_COUNT(paths[p])) {
+ SET_COUNT(paths[p], GET_COUNT(paths[p]) - 1);
+
+ if (GET_COUNT(paths[p]))
+ continue;
+ }
+
+ paths[p] = 0;
+ last_objects[p] = 0;
+ }
+ }
+
+ /* make topo relations */
+ if (last_objects[path] && !last_objects[path]->object.parsed)
+ commit_list_insert(co, &last_objects[path]->parents);
+
+ /* initialize commit */
+ if (!entry->is_end) {
+ co->date = ntohl(entry->date);
+ obj->flags |= ADDED | FACE_VALUE;
+ } else
+ parse_commit(co);
+
+ obj->flags |= SEEN;
+
+ if (entry->uninteresting)
+ obj->flags |= UNINTERESTING;
+
+ /* we need to know what the edges are */
+ last_objects[path] = co;
+
+ /* add to list */
+ if (!(obj->flags & UNINTERESTING) || revs->show_all) {
+ if (entry->is_end)
+ insert_by_date_cached(co, work, insert_cache, &insert_cache);
+ else
+ *queue = &commit_list_insert(co, *queue)->next;
+
+ /* add children to list as well */
+ if (obj->flags & UNINTERESTING)
+ consume_children = 0;
+ else
+ consume_children = 1;
+ }
+
+ /* open parents */
+ if (entry->merge_nr) {
+ int j, off = index + OE_SIZE;
+ char flag = entry->uninteresting ? UPATH : IPATH;
+
+ for (j = 0; j < entry->merge_nr; j++) {
+ unsigned short p = ntohs(*(unsigned short *)(map + off + PATH_SIZE(j)));
+
+ if (p >= total_path_nr)
+ goto end;
+
+ if (paths[p] & flag)
+ continue;
+
+ paths[p] |= flag;
+ }
+
+ /* make sure we don't use this path before all our parents have had their say */
+ SET_COUNT(paths[path], entry->merge_nr);
+ }
+
+ }
+
+ retval = 0;
+
+end:
+ free(paths);
+ free(last_objects);
+
+ return retval;
+}
+
+static int get_cache_slice_header(unsigned char *cache_sha1, unsigned char *map, int len, struct cache_slice_header *head)
+{
+ int t;
+
+ memcpy(head, map, sizeof(struct cache_slice_header));
+ head->ofs_objects = ntohl(head->ofs_objects);
+ head->object_nr = ntohl(head->object_nr);
+ head->size = ntohl(head->size);
+ head->path_nr = ntohs(head->path_nr);
+
+ if (memcmp(head->signature, "REVCACHE", 8))
+ return -1;
+ if (head->version > SUPPORTED_REVCACHE_VERSION)
+ return -2;
+ if (hashcmp(head->sha1, cache_sha1))
+ return -3;
+ t = sizeof(struct cache_slice_header);
+ if (t != head->ofs_objects || t >= len)
+ return -4;
+
+ head->size = len;
+
+ return 0;
+}
+
+int traverse_cache_slice(struct rev_info *revs,
+ unsigned char *cache_sha1, struct commit *commit,
+ unsigned long *date_so_far, int *slop_so_far,
+ struct commit_list ***queue, struct commit_list **work)
+{
+ int fd = -1, retval = -3;
+ struct stat fi;
+ struct cache_slice_header head;
+ struct rev_cache_info *rci;
+ unsigned char *map = MAP_FAILED;
+
+ /* the index should've been loaded already to find cache_sha1, but it's good
+ * to be absolutely sure... */
+ if (!idx_map)
+ init_index();
+ if (!idx_map)
+ return -1;
+
+ /* load options */
+ rci = &revs->rev_cache_info;
+
+ memset(&head, 0, sizeof(struct cache_slice_header));
+
+ fd = open(git_path("rev-cache/%s", sha1_to_hex(cache_sha1)), O_RDONLY);
+ if (fd == -1)
+ goto end;
+ if (fstat(fd, &fi) || fi.st_size < sizeof(struct cache_slice_header))
+ goto end;
+
+ map = xmmap(0, fi.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
+ if (map == MAP_FAILED)
+ goto end;
+ if (get_cache_slice_header(cache_sha1, map, fi.st_size, &head))
+ goto end;
+
+ retval = traverse_cache_slice_1(&head, map, revs, commit, date_so_far, slop_so_far, queue, work);
+
+end:
+ if (map != MAP_FAILED)
+ munmap(map, fi.st_size);
+ if (fd != -1)
+ close(fd);
+
+ return retval;
+}
+
+
+
+/* generation */
+
+struct path_track {
+ struct commit *commit;
+ int path; /* for decrementing paths */
+ struct strbuf path_str; /* for closing children */
+
+ struct path_track *next, *prev;
+};
+
+static unsigned char *paths;
+static int path_nr = 1, path_sz;
+
+static struct path_track *children_to_close, *paths_to_dec;
+static struct path_track *path_track_alloc;
+
+#define PATH_IN_USE 0x80 /* biggest bit we can get as a char */
+
+static int get_new_path(void)
+{
+ int i;
+
+ for (i = 1; i < path_nr; i++)
+ if (!paths[i])
+ break;
+
+ if (i == path_nr) {
+ if (path_nr >= path_sz) {
+ path_sz += 50;
+ paths = xrealloc(paths, path_sz);
+ memset(paths + path_sz - 50, 0, 50);
+ }
+ path_nr++;
+ }
+
+ paths[i] = PATH_IN_USE;
+ return i;
+}
+
+static void remove_path_track(struct path_track **ppt, char total_free)
+{
+ struct path_track *t = *ppt;
+
+ if (t->next)
+ t->next->prev = t->prev;
+ if (t->prev)
+ t->prev->next = t->next;
+
+ t = t->next;
+
+ if (total_free)
+ free(*ppt);
+ else {
+ (*ppt)->next = path_track_alloc;
+ path_track_alloc = *ppt;
+ }
+
+ *ppt = t;
+}
+
+static struct path_track *find_path_track(struct path_track *head, struct commit *commit)
+{
+ while (head) {
+ if (head->commit == commit)
+ return head;
+ head = head->next;
+ }
+
+ return 0;
+}
+
+static struct path_track *make_path_track(struct path_track **head, struct commit *commit)
+{
+ struct path_track *pt;
+
+ if (path_track_alloc) {
+ pt = path_track_alloc;
+ path_track_alloc = pt->next;
+ } else
+ pt = xmalloc(sizeof(struct path_track));
+
+ memset(pt, 0, sizeof(struct path_track));
+ pt->commit = commit;
+
+ pt->next = *head;
+ if (*head)
+ (*head)->prev = pt;
+ *head = pt;
+
+ return pt;
+}
+
+static void add_child_to_close(struct commit *commit, int path)
+{
+ struct path_track *pt = find_path_track(children_to_close, commit);
+ unsigned short write_path;
+
+ if (!pt) {
+ pt = make_path_track(&children_to_close, commit);
+ strbuf_init(&pt->path_str, 0);
+ }
+
+ write_path = htons((unsigned short)path);
+ strbuf_add(&pt->path_str, &write_path, PATH_WIDTH);
+}
+
+static void add_path_to_dec(struct commit *commit, int path)
+{
+ make_path_track(&paths_to_dec, commit);
+ paths_to_dec->path = path;
+}
+
+static void handle_paths(struct commit *commit, struct object_entry *object, struct strbuf *merge_str, struct strbuf *split_str)
+{
+ int parent_nr, open_parent_nr, this_path;
+ struct commit_list *list;
+ struct commit *first_parent;
+ struct path_track **ppt, *pt;
+
+ /* we can only re-use a closed path once all it's children have been encountered,
+ * as we need to keep track of commit boundaries */
+ ppt = &paths_to_dec;
+ pt = *ppt;
+ while (pt) {
+ if (pt->commit == commit) {
+ if (paths[pt->path] != PATH_IN_USE)
+ paths[pt->path]--;
+ remove_path_track(ppt, 0);
+ pt = *ppt;
+ } else {
+ pt = pt->next;
+ ppt = &pt;
+ }
+ }
+
+ /* the commit struct has no way of keeping track of children -- necessary for closing
+ * unused paths and tracking path boundaries -- so we have to do it here */
+ ppt = &children_to_close;
+ pt = *ppt;
+ while (pt) {
+ if (pt->commit != commit) {
+ pt = pt->next;
+ ppt = &pt;
+ continue;
+ }
+
+ object->split_nr = pt->path_str.len / PATH_WIDTH;
+ strbuf_add(split_str, pt->path_str.buf, pt->path_str.len);
+
+ strbuf_release(&pt->path_str);
+ remove_path_track(ppt, 0);
+ break;
+ }
+
+ /* initialize our self! */
+ if (!commit->indegree) {
+ commit->indegree = get_new_path();
+ object->is_start = 1;
+ }
+
+ this_path = commit->indegree;
+ paths[this_path] = PATH_IN_USE;
+ object->path = htons(this_path);
+
+ /* count interesting parents */
+ parent_nr = open_parent_nr = 0;
+ first_parent = 0;
+ for (list = commit->parents; list; list = list->next) {
+ if (list->item->object.flags & UNINTERESTING) {
+ object->is_end = 1;
+ continue;
+ }
+
+ parent_nr++;
+ if (!list->item->indegree)
+ open_parent_nr++;
+ if (!first_parent)
+ first_parent = list->item;
+ }
+
+ if (!parent_nr)
+ return;
+
+ if (parent_nr == 1 && open_parent_nr == 1) {
+ first_parent->indegree = this_path;
+ return;
+ }
+
+ /* make merge list */
+ object->merge_nr = parent_nr;
+ paths[this_path] = parent_nr;
+
+ for (list = commit->parents; list; list = list->next) {
+ struct commit *p = list->item;
+ unsigned short write_path;
+
+ if (p->object.flags & UNINTERESTING)
+ continue;
+
+ /* unfortunately due to boundary tracking we can't re-use merge paths
+ * (unable to guarantee last parent path = this -> last won't always be able to
+ * set this as a boundary object */
+ if (!p->indegree)
+ p->indegree = get_new_path();
+
+ write_path = htons((unsigned short)p->indegree);
+ strbuf_add(merge_str, &write_path, PATH_WIDTH);
+
+ /* make sure path is properly ended */
+ add_child_to_close(p, this_path);
+ add_path_to_dec(p, this_path);
+ }
+
+}
+
+
+static void add_object_entry(const unsigned char *sha1, int type, struct object_entry *nothisone,
+ struct strbuf *merge_str, struct strbuf *split_str)
+{
+ struct object_entry object;
+
+ if (!nothisone) {
+ memset(&object, 0, sizeof(object));
+ hashcpy(object.sha1, sha1);
+ object.type = type;
+
+ if (merge_str)
+ object.merge_nr = merge_str->len / PATH_WIDTH;
+ if (split_str)
+ object.split_nr = split_str->len / PATH_WIDTH;
+
+ nothisone = &object;
+ }
+
+ strbuf_add(g_buffer, nothisone, sizeof(object));
+
+ if (merge_str && merge_str->len)
+ strbuf_add(g_buffer, merge_str->buf, merge_str->len);
+ if (split_str && split_str->len)
+ strbuf_add(g_buffer, split_str->buf, split_str->len);
+
+}
+
+static void init_revcache_directory(void)
+{
+ struct stat fi;
+
+ if (stat(git_path("rev-cache"), &fi) || !S_ISDIR(fi.st_mode))
+ if (mkdir(git_path("rev-cache"), 0666))
+ die("can't make rev-cache directory");
+
+}
+
+void init_rci(struct rev_cache_info *rci)
+{
+ rci->objects = 1;
+ rci->legs = 0;
+ rci->make_index = 1;
+
+ rci->save_unique = 0;
+ rci->add_to_pending = 1;
+
+ rci->ignore_size = 0;
+}
+
+int make_cache_slice(struct rev_cache_info *rci,
+ struct rev_info *revs, struct commit_list **starts, struct commit_list **ends,
+ unsigned char *cache_sha1)
+{
+ struct commit_list *list;
+ struct rev_info therevs;
+ struct strbuf buffer, startlist, endlist;
+ struct cache_slice_header head;
+ struct commit *commit;
+ unsigned char sha1[20];
+ struct strbuf merge_paths, split_paths;
+ int object_nr, total_sz, fd;
+ char file[PATH_MAX], *newfile;
+ struct rev_cache_info *trci, def_rci;
+ git_SHA_CTX ctx;
+
+ if (!rci) {
+ rci = &def_rci;
+ init_rci(rci);
+ }
+
+ init_revcache_directory();
+ strcpy(file, git_path("rev-cache/XXXXXX"));
+ fd = xmkstemp(file);
+
+ strbuf_init(&buffer, 0);
+ strbuf_init(&startlist, 0);
+ strbuf_init(&endlist, 0);
+ strbuf_init(&merge_paths, 0);
+ strbuf_init(&split_paths, 0);
+ g_buffer = &buffer;
+
+ if (!revs) {
+ revs = &therevs;
+ init_revisions(revs, 0);
+
+ /* we're gonna assume no one else has already traversed this... */
+ for (list = *starts; list; list = list->next)
+ add_pending_object(revs, &list->item->object, 0);
+
+ for (list = *ends; list; list = list->next) {
+ list->item->object.flags |= UNINTERESTING;
+ add_pending_object(revs, &list->item->object, 0);
+ }
+ }
+
+ /* write head placeholder */
+ memset(&head, 0, sizeof(head));
+ head.ofs_objects = htonl(sizeof(head));
+ xwrite(fd, &head, sizeof(head));
+
+ /* init revisions! */
+ revs->tree_objects = 1;
+ revs->blob_objects = 1;
+ revs->topo_order = 1;
+ revs->lifo = 1;
+
+ /* re-use info from other caches if possible */
+ trci = &revs->rev_cache_info;
+ init_rci(trci);
+ trci->save_unique = 1;
+ trci->add_to_pending = 0;
+
+ setup_revisions(0, 0, revs, 0);
+ if (prepare_revision_walk(revs))
+ die("died preparing revision walk");
+
+ object_nr = total_sz = 0;
+ while ((commit = get_revision(revs)) != 0) {
+ struct object_entry object;
+
+ strbuf_setlen(&merge_paths, 0);
+ strbuf_setlen(&split_paths, 0);
+
+ memset(&object, 0, sizeof(object));
+ object.type = OBJ_COMMIT;
+ object.date = htonl(commit->date);
+ hashcpy(object.sha1, commit->object.sha1);
+
+ handle_paths(commit, &object, &merge_paths, &split_paths);
+
+ if (object.is_end)
+ strbuf_add(&endlist, object.sha1, 20);
+ if (object.is_start)
+ strbuf_add(&startlist, object.sha1, 20);
+
+ commit->indegree = 0;
+
+ add_object_entry(0, 0, &object, &merge_paths, &split_paths);
+ object_nr++;
+
+ /* print every ~1MB or so */
+ if (buffer.len > 1000000) {
+ write_in_full(fd, buffer.buf, buffer.len);
+ total_sz += buffer.len;
+
+ strbuf_setlen(&buffer, 0);
+ }
+ }
+
+ if (buffer.len) {
+ write_in_full(fd, buffer.buf, buffer.len);
+ total_sz += buffer.len;
+ }
+
+ /* go ahead a free some stuff... */
+ strbuf_release(&buffer);
+ strbuf_release(&merge_paths);
+ strbuf_release(&split_paths);
+ if (path_sz)
+ free(paths);
+ while (path_track_alloc)
+ remove_path_track(&path_track_alloc, 1);
+
+ /* the meaning of the hash name is more or less irrelevant, it's the uniqueness that matters */
+ strbuf_add(&endlist, startlist.buf, startlist.len);
+ git_SHA1_Init(&ctx);
+ git_SHA1_Update(&ctx, endlist.buf, endlist.len);
+ git_SHA1_Final(sha1, &ctx);
+
+ /* now actually initialize header */
+ strcpy(head.signature, "REVCACHE");
+ head.version = SUPPORTED_REVCACHE_VERSION;
+
+ head.object_nr = htonl(object_nr);
+ head.size = htonl(ntohl(head.ofs_objects) + total_sz);
+ head.path_nr = htons(path_nr);
+ hashcpy(head.sha1, sha1);
+
+ /* some info! */
+ fprintf(stderr, "objects: %d\n", object_nr);
+ fprintf(stderr, "paths: %d\n", path_nr);
+
+ lseek(fd, 0, SEEK_SET);
+ xwrite(fd, &head, sizeof(head));
+
+ if (rci->make_index && make_cache_index(rci, sha1, fd, ntohl(head.size)) < 0)
+ die("can't update index");
+
+ close(fd);
+
+ newfile = git_path("rev-cache/%s", sha1_to_hex(sha1));
+ if (rename(file, newfile))
+ die("can't move temp file");
+
+ /* let our caller know what we've just made */
+ if (cache_sha1)
+ hashcpy(cache_sha1, sha1);
+
+ strbuf_release(&endlist);
+ strbuf_release(&startlist);
+
+ return 0;
+}
+
+
+static int index_sort_hash(const void *a, const void *b)
+{
+ return hashcmp(IE_CAST(a)->sha1, IE_CAST(b)->sha1);
+}
+
+static int write_cache_index(struct strbuf *body)
+{
+ struct index_header whead;
+ struct lock_file *lk;
+ int fd, i;
+
+ /* clear index map if loaded */
+ if (idx_map) {
+ munmap(idx_map, idx_size);
+ idx_map = 0;
+ }
+
+ lk = xcalloc(sizeof(struct lock_file), 1);
+ fd = hold_lock_file_for_update(lk, git_path("rev-cache/index"), 0);
+ if (fd < 0) {
+ free(lk);
+ return -1;
+ }
+
+ /* endianness yay! */
+ memset(&whead, 0, sizeof(whead));
+ memcpy(whead.signature, "REVINDEX", 8);
+ whead.version = idx_head.version;
+ whead.ofs_objects = htonl(idx_head.ofs_objects);
+ whead.object_nr = htonl(idx_head.object_nr);
+ whead.cache_nr = idx_head.cache_nr;
+ whead.max_date = htonl(idx_head.max_date);
+
+ write(fd, &whead, sizeof(struct index_header));
+ write_in_full(fd, idx_caches, idx_head.cache_nr * 20);
+
+ for (i = 0; i <= 0xff; i++)
+ fanout[i] = htonl(fanout[i]);
+ write_in_full(fd, fanout, 0x100 * sizeof(uint32_t));
+
+ write_in_full(fd, body->buf, body->len);
+
+ if (commit_lock_file(lk) < 0)
+ return -2;
+
+ /* lk freed by lockfile.c */
+
+ return 0;
+}
+
+int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
+ int fd, unsigned int size)
+{
+ struct strbuf buffer;
+ int i, cache_index, cur;
+ unsigned char *map;
+ unsigned long max_date;
+
+ if (!idx_map)
+ init_index();
+
+ lseek(fd, 0, SEEK_SET);
+ map = xmmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
+ if (map == MAP_FAILED)
+ return -1;
+
+ strbuf_init(&buffer, 0);
+ if (idx_map) {
+ strbuf_add(&buffer, idx_map + fanout[0], fanout[0x100] - fanout[0]);
+ } else {
+ /* not an update */
+ memset(&idx_head, 0, sizeof(struct index_header));
+ idx_caches = 0;
+
+ strcpy(idx_head.signature, "REVINDEX");
+ idx_head.version = SUPPORTED_REVINDEX_VERSION;
+ idx_head.ofs_objects = sizeof(struct index_header) + 0x100 * sizeof(uint32_t);
+ }
+
+ /* are we remaking a slice? */
+ for (i = 0; i < idx_head.cache_nr; i++)
+ if (!hashcmp(idx_caches + i * 20, cache_sha1))
+ break;
+
+ if (i == idx_head.cache_nr) {
+ cache_index = idx_head.cache_nr++;
+ idx_head.ofs_objects += 20;
+
+ idx_caches = xrealloc(idx_caches, idx_head.cache_nr * 20);
+ hashcpy(idx_caches + cache_index * 20, cache_sha1);
+ } else
+ cache_index = i;
+
+ i = sizeof(struct cache_slice_header); /* offset */
+ max_date = idx_head.max_date;
+ while (i < size) {
+ struct index_entry index_entry, *entry;
+ struct object_entry *object_entry = OE_CAST(map + i);
+ unsigned long date;
+ int pos = i;
+
+ i += ACTUAL_OBJECT_ENTRY_SIZE(object_entry);
+
+ if (object_entry->type != OBJ_COMMIT)
+ continue;
+
+ /* don't include ends; otherwise we'll find ourselves in loops */
+ if (object_entry->is_end)
+ continue;
+
+ /* handle index duplication
+ * -> keep old copy unless new one is an end -- based on expected usage, older ones will be more
+ * likely to lead to greater slice traversals than new ones
+ * should we allow more intelligent overriding? */
+ date = ntohl(object_entry->date);
+ if (date > idx_head.max_date) {
+ entry = 0;
+ if (date > max_date)
+ max_date = date;
+ } else
+ entry = search_index(object_entry->sha1);
+
+ if (entry && !object_entry->is_start)
+ continue;
+ else if (entry) /* mmm, pointer arithmetic... tasty */ /* (entry-idx_map = offset, so cast is valid) */
+ entry = IE_CAST(buffer.buf + (unsigned int)((unsigned char *)entry - idx_map) - fanout[0]);
+ else
+ entry = &index_entry;
+
+ memset(entry, 0, sizeof(index_entry));
+ hashcpy(entry->sha1, object_entry->sha1);
+ entry->is_start = object_entry->is_start;
+ entry->cache_index = cache_index;
+ entry->pos = htonl(pos);
+
+ if (entry == &index_entry) {
+ strbuf_add(&buffer, entry, sizeof(index_entry));
+ idx_head.object_nr++;
+ }
+
+ }
+
+ idx_head.max_date = max_date;
+ qsort(buffer.buf, buffer.len / IE_SIZE, IE_SIZE, index_sort_hash);
+
+ /* generate fanout */
+ cur = 0x00;
+ for (i = 0; i < buffer.len; i += IE_SIZE) {
+ struct index_entry *entry = IE_CAST(buffer.buf + i);
+
+ while (cur <= entry->sha1[0])
+ fanout[cur++] = i + idx_head.ofs_objects;
+ }
+
+ while (cur <= 0xff)
+ fanout[cur++] = idx_head.ofs_objects + buffer.len;
+
+ /* BOOM! */
+ if (write_cache_index(&buffer))
+ return -1;
+
+ munmap(map, size);
+ strbuf_release(&buffer);
+
+ /* idx_map is unloaded without cleanup_cache_slices(), so regardless of previous index existence
+ * we can still free this up */
+ free(idx_caches);
+
+ return 0;
+}
+
+
+/* add start-commits from each cache slice (uninterestingness will be propogated) */
+void starts_from_slices(struct rev_info *revs, unsigned int flags)
+{
+ struct commit *commit;
+ int i;
+
+ if (!idx_map)
+ init_index();
+ if (!idx_map)
+ return;
+
+ /* haven't implemented which yet; no need really... */
+ for (i = idx_head.ofs_objects; i < idx_size; i += IE_SIZE) {
+ struct index_entry *entry = IE_CAST(idx_map + i);
+
+ if (!entry->is_start)
+ continue;
+
+ commit = lookup_commit(entry->sha1);
+ if (!commit)
+ continue;
+
+ commit->object.flags |= flags;
+ add_pending_object(revs, &commit->object, 0);
+ }
+
+}
diff --git a/revision.c b/revision.c
index 9f5dac5..485bf72 100644
--- a/revision.c
+++ b/revision.c
@@ -432,7 +432,7 @@ static void try_to_simplify_commit(struct rev_info *revs, struct commit *commit)
commit->object.flags |= TREESAME;
}
-static void insert_by_date_cached(struct commit *p, struct commit_list **head,
+void insert_by_date_cached(struct commit *p, struct commit_list **head,
struct commit_list *cached_base, struct commit_list **cache)
{
struct commit_list *new_entry;
diff --git a/revision.h b/revision.h
index fb74492..200042a 100644
--- a/revision.h
+++ b/revision.h
@@ -13,11 +13,26 @@
#define CHILD_SHOWN (1u<<6)
#define ADDED (1u<<7) /* Parents already parsed and added? */
#define SYMMETRIC_LEFT (1u<<8)
-#define ALL_REV_FLAGS ((1u<<9)-1)
+#define FACE_VALUE (1u<<9)
+#define ALL_REV_FLAGS ((1u<<10)-1)
struct rev_info;
struct log_info;
+struct rev_cache_info {
+ /* generation flags */
+ unsigned objects : 1,
+ legs : 1,
+ make_index : 1;
+
+ /* traversal flags */
+ unsigned save_unique : 1,
+ add_to_pending : 1;
+
+ /* fuse options */
+ unsigned int ignore_size;
+};
+
struct rev_info {
/* Starting list */
struct commit_list *commits;
@@ -73,6 +88,10 @@ struct rev_info {
dense_combined_merges:1,
always_show_header:1;
+ /* rev-cache flags */
+ unsigned int for_pack:1,
+ dont_cache_me:1;
+
/* Format info */
unsigned int shown_one:1,
show_merge:1,
@@ -116,6 +135,9 @@ struct rev_info {
struct reflog_walk_info *reflog_info;
struct decoration children;
struct decoration merge_simplification;
+
+ /* caching info, used ONLY by traverse_cache_slice */
+ struct rev_cache_info rev_cache_info;
};
#define REV_TREE_SAME 0
@@ -167,4 +189,24 @@ enum commit_action {
extern enum commit_action simplify_commit(struct rev_info *revs, struct commit *commit);
+extern void insert_by_date_cached(struct commit *p, struct commit_list **head,
+ struct commit_list *cached_base, struct commit_list **cache);
+
+/* rev-cache.c */
+
+extern unsigned char *get_cache_slice(struct commit *commit);
+extern int traverse_cache_slice(struct rev_info *revs,
+ unsigned char *cache_sha1, struct commit *commit,
+ unsigned long *date_so_far, int *slop_so_far,
+ struct commit_list ***queue, struct commit_list **work);
+
+extern void init_rci(struct rev_cache_info *rci);
+extern int make_cache_slice(struct rev_cache_info *rci,
+ struct rev_info *revs, struct commit_list **tops, struct commit_list **bottoms,
+ unsigned char *cache_sha1);
+extern int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
+ int fd, unsigned int size);
+
+extern void starts_from_slices(struct rev_info *revs, unsigned int flags);
+
#endif
diff --git a/t/t6015-rev-cache-list.sh b/t/t6015-rev-cache-list.sh
new file mode 100755
index 0000000..ccac4de
--- /dev/null
+++ b/t/t6015-rev-cache-list.sh
@@ -0,0 +1,104 @@
+#!/bin/sh
+
+test_description='git rev-cache tests'
+. ./test-lib.sh
+
+test_cmp_sorted() {
+ grep -io "[a-f0-9]*" $1 | sort >.tmpfile1 &&
+ grep -io "[a-f0-9]*" $2 | sort >.tmpfile2 &&
+ test_cmp .tmpfile1 .tmpfile2
+}
+
+# we want a totally wacked out branch structure...
+# we need branching and merging of sizes up through 3, tree
+# addition/deletion, and enough branching to exercise path
+# reuse
+test_expect_success 'init repo' '
+ echo bla >file &&
+ git add . &&
+ git commit -m "bla" &&
+
+ git branch b1 &&
+ git checkout b1 &&
+ echo blu >file2 &&
+ mkdir d1 &&
+ echo bang >d1/filed1 &&
+ git add . &&
+ git commit -m "blu" &&
+
+ git checkout master &&
+ git branch b2 &&
+ git checkout b2 &&
+ echo kaplaa >>file &&
+ git commit -a -m "kaplaa" &&
+
+ git checkout master &&
+ mkdir smoke &&
+ echo omg >smoke/bong &&
+ git add . &&
+ git commit -m "omg" &&
+
+ git branch b4 &&
+ git checkout b4 &&
+ echo shazam >file8 &&
+ git add . &&
+ git commit -m "shazam" &&
+ git merge -m "merge b2" b2 &&
+
+ echo bam >smoke/pipe &&
+ git add .
+ git commit -m "bam" &&
+
+ git checkout master &&
+ echo pow >file7 &&
+ git add . &&
+ git commit -m "pow" &&
+ git merge -m "merge b4" b4 &&
+
+ git checkout b1 &&
+ echo stuff >d1/filed1 &&
+ git commit -a -m "stuff" &&
+
+ git branch b11 &&
+ git checkout b11 &&
+ echo wazzup >file3 &&
+ git add file3 &&
+ git commit -m "wazzup" &&
+
+ git checkout b1 &&
+ mkdir d1/d2 &&
+ echo lol >d1/d2/filed2 &&
+ git add . &&
+ git commit -m "lol" &&
+
+ git checkout master &&
+ git merge -m "triple merge" b1 b11 &&
+ git rm -r d1 &&
+ git commit -a -m "oh noes"
+'
+
+git-rev-list HEAD --not HEAD~3 >proper_commit_list_limited
+git-rev-list HEAD >proper_commit_list
+
+test_expect_success 'make cache slice' '
+ git-rev-cache add HEAD 2>output.err &&
+ grep "final return value: 0" output.err
+'
+
+test_expect_success 'remake cache slice' '
+ git-rev-cache add --sizes HEAD 2>output.err &&
+ grep "final return value: 0" output.err
+'
+
+#check core mechanics and rev-list hook for commits
+test_expect_success 'test rev-caches walker directly (limited)' '
+ git-rev-cache walk HEAD --not HEAD~3 >list &&
+ test_cmp_sorted list proper_commit_list_limited
+'
+
+test_expect_success 'test rev-caches walker directly (unlimited)' '
+ git-rev-cache walk HEAD >list &&
+ test_cmp_sorted list proper_commit_list
+'
+
+test_done
--
tg: (e658002..) t/revcache/basic (depends on: master)
^ permalink raw reply related
* [PATCH 3/6 (v2)] non-commit object support for rev-cache
From: Nick Edelen @ 2009-08-08 7:31 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
From: Nick Edelen <sirnot@gmail.com>
Subject: [PATCH] t/revcache/objects
Summarized, this third patch contains:
- support for non-commit object caching
- expansion of porcelain to accomodate non-commit objects
- appropriate tests
Objects are stored relative to the commit in which they were introduced --
commits are 'diffed' against their parents. This will eliminate the need for
tree recursion in cached commits (significantly reducing I/O), and potentially
be useful to external applications.
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
No major changes in this patch. Correlates with 3/5 in previous.
rev-cache.c | 206 ++++++++++++++++++++++++++++++++++++++++++++-
t/t6015-rev-cache-list.sh | 8 ++
2 files changed, 211 insertions(+), 3 deletions(-)
diff --git a/rev-cache.c b/rev-cache.c
index 5d9e150..cd618cf 100755
--- a/rev-cache.c
+++ b/rev-cache.c
@@ -229,6 +229,32 @@ unsigned char *get_cache_slice(struct commit *commit)
/* traversal */
+static void handle_noncommit(struct rev_info *revs, struct object_entry *entry)
+{
+ struct object *obj = 0;
+
+ switch (entry->type) {
+ case OBJ_TREE :
+ if (revs->tree_objects)
+ obj = (struct object *)lookup_tree(entry->sha1);
+ break;
+ case OBJ_BLOB :
+ if (revs->blob_objects)
+ obj = (struct object *)lookup_blob(entry->sha1);
+ break;
+ case OBJ_TAG :
+ if (revs->tag_objects)
+ obj = (struct object *)lookup_tag(entry->sha1);
+ break;
+ }
+
+ if (!obj)
+ return;
+
+ obj->flags |= FACE_VALUE;
+ add_pending_object(revs, obj, "");
+}
+
static int setup_traversal(struct cache_slice_header *head, unsigned char *map, struct commit *commit, struct commit_list **work)
{
struct index_entry *iep;
@@ -310,9 +336,12 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char
i += ACTUAL_OBJECT_ENTRY_SIZE(entry);
/* add extra objects if necessary */
- if (entry->type != OBJ_COMMIT)
+ if (entry->type != OBJ_COMMIT) {
+ if (consume_children)
+ handle_noncommit(revs, entry);
+
continue;
- else
+ } else
consume_children = 0;
if (path >= total_path_nr)
@@ -766,6 +795,168 @@ static void add_object_entry(const unsigned char *sha1, int type, struct object_
}
+/* returns non-zero to continue parsing, 0 to skip */
+typedef int (*dump_tree_fn)(const unsigned char *, const char *, unsigned int); /* sha1, path, mode */
+
+/* we need to walk the trees by hash, so unfortunately we can't use traverse_trees in tree-walk.c */
+static int dump_tree(struct tree *tree, dump_tree_fn fn)
+{
+ struct tree_desc desc;
+ struct name_entry entry;
+ struct tree *subtree;
+ int r;
+
+ if (parse_tree(tree))
+ return -1;
+
+ init_tree_desc(&desc, tree->buffer, tree->size);
+ while (tree_entry(&desc, &entry)) {
+ switch (fn(entry.sha1, entry.path, entry.mode)) {
+ case 0 :
+ goto continue_loop;
+ default :
+ break;
+ }
+
+ if (S_ISDIR(entry.mode)) {
+ subtree = lookup_tree(entry.sha1);
+ if (!subtree)
+ return -2;
+
+ if ((r = dump_tree(subtree, fn)) < 0)
+ return r;
+ }
+
+continue_loop:
+ continue;
+ }
+
+ return 0;
+}
+
+static int dump_tree_callback(const unsigned char *sha1, const char *path, unsigned int mode)
+{
+ unsigned char data[21];
+
+ hashcpy(data, sha1);
+ data[20] = !!S_ISDIR(mode);
+
+ strbuf_add(g_buffer, data, 21);
+
+ return 1;
+}
+
+static void tree_addremove(struct diff_options *options,
+ int whatnow, unsigned mode,
+ const unsigned char *sha1,
+ const char *concatpath)
+{
+ unsigned char data[21];
+
+ if (whatnow != '+')
+ return;
+
+ hashcpy(data, sha1);
+ data[20] = !!S_ISDIR(mode);
+
+ strbuf_add(g_buffer, data, 21);
+}
+
+static void tree_change(struct diff_options *options,
+ unsigned old_mode, unsigned new_mode,
+ const unsigned char *old_sha1,
+ const unsigned char *new_sha1,
+ const char *concatpath)
+{
+ unsigned char data[21];
+
+ if (!hashcmp(old_sha1, new_sha1))
+ return;
+
+ hashcpy(data, new_sha1);
+ data[20] = !!S_ISDIR(new_mode);
+
+ strbuf_add(g_buffer, data, 21);
+}
+
+static int sort_type_hash(const void *a, const void *b)
+{
+ const unsigned char *sa = (const unsigned char *)a,
+ *sb = (const unsigned char *)b;
+
+ if (sa[20] == sb[20])
+ return hashcmp(sa, sb);
+
+ return sa[20] > sb[20] ? -1 : 1;
+}
+
+static int add_unique_objects(struct commit *commit)
+{
+ struct commit_list *list;
+ struct strbuf os, ost, *orig_buf;
+ struct diff_options opts;
+ int i, j, next;
+ char is_first = 1;
+
+ strbuf_init(&os, 0);
+ strbuf_init(&ost, 0);
+ orig_buf = g_buffer;
+
+ diff_setup(&opts);
+ DIFF_OPT_SET(&opts, RECURSIVE);
+ DIFF_OPT_SET(&opts, TREE_IN_RECURSIVE);
+ opts.change = tree_change;
+ opts.add_remove = tree_addremove;
+
+ /* this is only called for non-ends (ie. all parents interesting) */
+ for (list = commit->parents; list; list = list->next) {
+ if (is_first)
+ g_buffer = &os;
+ else
+ g_buffer = &ost;
+
+ strbuf_setlen(g_buffer, 0);
+ diff_tree_sha1(list->item->tree->object.sha1, commit->tree->object.sha1, "", &opts);
+ qsort(g_buffer->buf, g_buffer->len / 21, 21, (int (*)(const void *, const void *))hashcmp);
+
+ /* take intersection */
+ if (!is_first) {
+ for (next = i = j = 0; i < os.len; i += 21) {
+ while (j < ost.len && hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)) < 0)
+ j += 21;
+
+ if (j >= ost.len || hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)))
+ continue;
+
+ if (next != i)
+ memcpy(os.buf + next, os.buf + i, 21);
+ next += 21;
+ }
+
+ if (next != i)
+ strbuf_setlen(&os, next);
+ } else
+ is_first = 0;
+ }
+
+ if (is_first) {
+ g_buffer = &os;
+ dump_tree(commit->tree, dump_tree_callback);
+ }
+
+ if (os.len)
+ qsort(os.buf, os.len / 21, 21, sort_type_hash);
+
+ g_buffer = orig_buf;
+ for (i = 0; i < os.len; i += 21)
+ add_object_entry((unsigned char *)(os.buf + i), os.buf[i + 20] ? OBJ_TREE : OBJ_BLOB, 0, 0, 0);
+
+ strbuf_release(&ost);
+ strbuf_release(&os);
+
+ return 0;
+}
+
static void init_revcache_directory(void)
{
struct stat fi;
@@ -879,6 +1070,15 @@ int make_cache_slice(struct rev_cache_info *rci,
add_object_entry(0, 0, &object, &merge_paths, &split_paths);
object_nr++;
+ if (!(commit->object.flags & TREESAME)) {
+ /* add all unique children for this commit */
+ add_object_entry(commit->tree->object.sha1, OBJ_TREE, 0, 0, 0);
+ object_nr++;
+
+ if (!object.is_end)
+ object_nr += add_unique_objects(commit);
+ }
+
/* print every ~1MB or so */
if (buffer.len > 1000000) {
write_in_full(fd, buffer.buf, buffer.len);
@@ -1055,7 +1255,7 @@ int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
continue;
/* handle index duplication
- * -> keep old copy unless new one is an end -- based on expected usage, older ones will be more
+ * -> keep old copy unless new one is a start -- based on expected usage, older ones will be more
* likely to lead to greater slice traversals than new ones
* should we allow more intelligent overriding? */
date = ntohl(object_entry->date);
diff --git a/t/t6015-rev-cache-list.sh b/t/t6015-rev-cache-list.sh
index ccac4de..8aea2e3 100755
--- a/t/t6015-rev-cache-list.sh
+++ b/t/t6015-rev-cache-list.sh
@@ -79,6 +79,7 @@ test_expect_success 'init repo' '
git-rev-list HEAD --not HEAD~3 >proper_commit_list_limited
git-rev-list HEAD >proper_commit_list
+git-rev-list HEAD --objects >proper_object_list
test_expect_success 'make cache slice' '
git-rev-cache add HEAD 2>output.err &&
@@ -101,4 +102,11 @@ test_expect_success 'test rev-caches walker directly (unlimited)' '
test_cmp_sorted list proper_commit_list
'
+#do the same for objects
+test_expect_success 'test rev-caches walker with objects' '
+ git-rev-cache walk --objects HEAD >list &&
+ test_cmp_sorted list proper_object_list
+'
+
test_done
+
--
tg: (3d98a1d..) t/revcache/objects (depends on: t/revcache/basic)
^ permalink raw reply related
* [PATCH 4/6 (v2)] administrative functions for rev-cache, and start of integration into git
From: Nick Edelen @ 2009-08-08 7:31 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
This patch contains miscellaneous (maintenance) features:
- support for cache slice fusion, index regeneration and object size caching
- extra 'size' field added to commit, tag and blob objects, initialized in
parse_* functions and cache traversal
- non-commit object generation refactored to take advantage of 'size' field
- porcelain updated to support feature additions
The beginnings of integration into git are present in this patch, mainly
centered on caching object size; the object generation is refactored to more
elegantly exploit this. Fusion allows smaller (incremental) slices to be
coagulated into a larger slice, reducing overhead, while index regeneration
enables repair or cleaning of the cache index.
Note that tests for these features are included in the following patch, as they
take advantage of the rev-cache's integration into the revision walker.
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
4/5 previously. Small adjustment in parsing of --ignore-size field in porcelain.
blob.c | 1 +
blob.h | 1 +
builtin-rev-cache.c | 73 +++++++++-
commit.c | 1 +
commit.h | 1 +
rev-cache.c | 417 +++++++++++++++++++++++++++++++++++++++++---------
revision.h | 4 +-
7 files changed, 421 insertions(+), 77 deletions(-)
diff --git a/blob.c b/blob.c
index bd7d078..e1bab3f 100644
--- a/blob.c
+++ b/blob.c
@@ -21,6 +21,7 @@ struct blob *lookup_blob(const unsigned char *sha1)
int parse_blob_buffer(struct blob *item, void *buffer, unsigned long size)
{
item->object.parsed = 1;
+ item->size = size;
return 0;
}
diff --git a/blob.h b/blob.h
index ea5d9e9..d8db96a 100644
--- a/blob.h
+++ b/blob.h
@@ -7,6 +7,7 @@ extern const char *blob_type;
struct blob {
struct object object;
+ unsigned long size;
};
struct blob *lookup_blob(const unsigned char *sha1);
diff --git a/builtin-rev-cache.c b/builtin-rev-cache.c
index 2338871..532ec9f 100755
--- a/builtin-rev-cache.c
+++ b/builtin-rev-cache.c
@@ -4,6 +4,8 @@
#include "diff.h"
#include "revision.h"
+#define DEFAULT_IGNORE_SLICE_SIZE "30mb"
+
/* porcelain for rev-cache.c */
static int handle_add(int argc, const char *argv[]) /* args beyond this command */
{
@@ -21,7 +23,7 @@ static int handle_add(int argc, const char *argv[]) /* args beyond this command
if (!strcmp(argv[i], "--stdin"))
dostdin = 1;
else if (!strcmp(argv[i], "--fresh"))
- starts_from_slices(&revs, UNINTERESTING);
+ starts_from_slices(&revs, UNINTERESTING, 0, 0);
else if (!strcmp(argv[i], "--not"))
flags ^= UNINTERESTING;
else if (!strcmp(argv[i], "--legs"))
@@ -141,6 +143,71 @@ static int handle_walk(int argc, const char *argv[])
return 0;
}
+static unsigned int parse_size(const char *name)
+{
+ unsigned int size;
+ char *p;
+
+ size = strtol(name, &p, 10);
+ switch (*p) {
+ case 'k' :
+ case 'K' :
+ size *= 0x400;
+ break;
+ case 'm' :
+ case 'M' :
+ size *= 0x100000;
+ break;
+ default :
+ break;
+ }
+
+ return size;
+}
+
+static int handle_fuse(int argc, const char *argv[])
+{
+ struct rev_info revs;
+ struct rev_cache_info rci;
+ const char *args[5];
+ int i, argn = 0;
+ char add_all = 0;
+
+ init_revisions(&revs, 0);
+ init_rci(&rci);
+ args[argn++] = "rev-list";
+
+ for (i = 0; i < argc; i++) {
+ if (!strcmp(argv[i], "--all")) {
+ args[argn++] = "--all";
+ setup_revisions(argn, args, &revs, 0);
+ add_all = 1;
+ } else if (!strcmp(argv[i], "--noobjects"))
+ rci.objects = 0;
+ else if (!strncmp(argv[i], "--ignore-size", 13)) {
+ unsigned int sz;
+
+ if (argv[i][13] == '=')
+ sz = parse_size(argv[i] + 14);
+ else
+ sz = parse_size(DEFAULT_IGNORE_SLICE_SIZE);
+
+ rci.ignore_size = sz;
+ } else
+ continue;
+ }
+
+ if (!add_all)
+ starts_from_slices(&revs, 0, 0, 0);
+
+ return coagulate_cache_slices(&rci, &revs);
+}
+
+static int handle_index(int argc, const char *argv[])
+{
+ return regenerate_cache_index(0);
+}
+
static int handle_help(void)
{
char *usage = "\
@@ -186,8 +253,12 @@ int cmd_rev_cache(int argc, const char *argv[], const char *prefix)
argv += 2;
if (!strcmp(arg, "add"))
r = handle_add(argc, argv);
+ else if (!strcmp(arg, "fuse"))
+ r = handle_fuse(argc, argv);
else if (!strcmp(arg, "walk"))
r = handle_walk(argc, argv);
+ else if (!strcmp(arg, "index"))
+ r = handle_index(argc, argv);
else
return handle_help();
diff --git a/commit.c b/commit.c
index 65d223e..afe504f 100644
--- a/commit.c
+++ b/commit.c
@@ -243,6 +243,7 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
if (item->object.parsed)
return 0;
item->object.parsed = 1;
+ item->size = size;
tail += size;
if (tail <= bufptr + 46 || memcmp(bufptr, "tree ", 5) || bufptr[45] != '\n')
return error("bogus commit object %s", sha1_to_hex(item->object.sha1));
diff --git a/commit.h b/commit.h
index ba9f638..0529fcf 100644
--- a/commit.h
+++ b/commit.h
@@ -19,6 +19,7 @@ struct commit {
struct commit_list *parents;
struct tree *tree;
char *buffer;
+ unsigned long size;
};
extern int save_commit_buffer;
diff --git a/rev-cache.c b/rev-cache.c
index cd618cf..1d29f8e 100755
--- a/rev-cache.c
+++ b/rev-cache.c
@@ -8,6 +8,7 @@
#include "diff.h"
#include "revision.h"
#include "run-command.h"
+#include "string-list.h"
/* single index maps objects to cache files */
@@ -88,6 +89,7 @@ static struct strbuf *g_buffer;
#define IE_CAST(p) ((struct index_entry *)(p))
#define ACTUAL_OBJECT_ENTRY_SIZE(e) (OE_SIZE + PATH_SIZE((e)->merge_nr + (e)->split_nr) + (e)->size_size)
+#define ENTRY_SIZE_OFFSET(e) (ACTUAL_OBJECT_ENTRY_SIZE(e) - (e)->size_size)
#define SLOP 5
@@ -219,7 +221,6 @@ unsigned char *get_cache_slice(struct commit *commit)
return 0;
ie = search_index(commit->object.sha1);
-
if (ie && ie->cache_index < idx_head.cache_nr)
return idx_caches + ie->cache_index * 20;
@@ -229,27 +230,46 @@ unsigned char *get_cache_slice(struct commit *commit)
/* traversal */
-static void handle_noncommit(struct rev_info *revs, struct object_entry *entry)
+static unsigned long decode_size(unsigned char *str, int len);
+
+static void handle_noncommit(struct rev_info *revs, struct commit *commit, struct object_entry *entry)
{
- struct object *obj = 0;
+ struct blob *blob;
+ struct tree *tree;
+ struct object *obj;
+ unsigned long size;
+ size = decode_size((unsigned char *)entry + ENTRY_SIZE_OFFSET(entry), entry->size_size);
switch (entry->type) {
case OBJ_TREE :
- if (revs->tree_objects)
- obj = (struct object *)lookup_tree(entry->sha1);
+ if (!revs->tree_objects)
+ return;
+
+ tree = lookup_tree(entry->sha1);
+ if (!tree)
+ return;
+
+ tree->size = size;
+ commit->tree = tree;
+ obj = (struct object *)tree;
break;
+
case OBJ_BLOB :
- if (revs->blob_objects)
- obj = (struct object *)lookup_blob(entry->sha1);
- break;
- case OBJ_TAG :
- if (revs->tag_objects)
- obj = (struct object *)lookup_tag(entry->sha1);
+ if (!revs->blob_objects)
+ return;
+
+ blob = lookup_blob(entry->sha1);
+ if (!blob)
+ return;
+
+ blob->size = size;
+ obj = (struct object *)blob;
break;
- }
-
- if (!obj)
+
+ default :
+ /* tag objects aren't really supposed to be here */
return;
+ }
obj->flags |= FACE_VALUE;
add_pending_object(revs, obj, "");
@@ -262,7 +282,6 @@ static int setup_traversal(struct cache_slice_header *head, unsigned char *map,
struct commit_list *prev, *wp, **wpp;
int retval;
- /* printf("adding %s\n", sha1_to_hex(commit->object.sha1)); */
iep = search_index(commit->object.sha1);
oep = OE_CAST(map + ntohl(iep->pos));
oep->include = 1;
@@ -338,7 +357,7 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char
/* add extra objects if necessary */
if (entry->type != OBJ_COMMIT) {
if (consume_children)
- handle_noncommit(revs, entry);
+ handle_noncommit(revs, co, entry);
continue;
} else
@@ -372,6 +391,8 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char
if (last_objects[path]) {
parse_commit(last_objects[path]);
+ /* we needn't worry about the unique field; that will be valid as
+ * long as we're not a end entry */
last_objects[path]->object.flags &= ~FACE_VALUE;
last_objects[path] = 0;
}
@@ -554,6 +575,68 @@ end:
/* generation */
+static int is_endpoint(struct commit *commit)
+{
+ struct commit_list *list = commit->parents;
+
+ while (list) {
+ if (!(list->item->object.flags & UNINTERESTING))
+ return 0;
+
+ list = list->next;
+ }
+
+ return 1;
+}
+
+/* ensures branch is self-contained: parents are either all interesting or all uninteresting */
+static void make_legs(struct rev_info *revs)
+{
+ struct commit_list *list, **plist;
+ int total = 0;
+
+ /* attach plist to end of commits list */
+ list = revs->commits;
+ while (list && list->next)
+ list = list->next;
+
+ if (list)
+ plist = &list->next;
+ else
+ return;
+
+ /* duplicates don't matter, as get_revision() ignores them */
+ for (list = revs->commits; list; list = list->next) {
+ struct commit *item = list->item;
+ struct commit_list *parents = item->parents;
+
+ if (item->object.flags & UNINTERESTING)
+ continue;
+ if (is_endpoint(item))
+ continue;
+
+ while (parents) {
+ struct commit *p = parents->item;
+ parents = parents->next;
+
+ if (!(p->object.flags & UNINTERESTING))
+ continue;
+
+ p->object.flags &= ~UNINTERESTING;
+ parse_commit(p);
+ plist = &commit_list_insert(p, plist)->next;
+
+ if (!(p->object.flags & SEEN))
+ total++;
+ }
+ }
+
+ if (total)
+ sort_in_topological_order(&revs->commits, 1);
+
+}
+
+
struct path_track {
struct commit *commit;
int path; /* for decrementing paths */
@@ -768,31 +851,97 @@ static void handle_paths(struct commit *commit, struct object_entry *object, str
}
-static void add_object_entry(const unsigned char *sha1, int type, struct object_entry *nothisone,
+static int encode_size(unsigned long size, unsigned char *out)
+{
+ int len = 0;
+
+ while (size) {
+ *out++ = (unsigned char)(size & 0xff);
+ size >>= 8;
+ len++;
+ }
+
+ return len;
+}
+
+static unsigned long decode_size(unsigned char *str, int len)
+{
+ unsigned long size = 0;
+ int shift = 0;
+
+ while (len--) {
+ size |= (unsigned long)*str << shift;
+ shift += 8;
+ str++;
+ }
+
+ return size;
+}
+
+static void add_object_entry(const unsigned char *sha1, struct object_entry *entryp,
struct strbuf *merge_str, struct strbuf *split_str)
{
- struct object_entry object;
+ struct object_entry entry;
+ struct object *obj;
+ unsigned char size_str[7];
+ unsigned long size;
+ enum object_type type;
+
+ if (entryp)
+ sha1 = entryp->sha1;
+
+ obj = lookup_object(sha1);
+ if (obj) {
+ /* it'd be smoother to have the size in the object... */
+ switch (obj->type) {
+ case OBJ_COMMIT :
+ size = ((struct commit *)obj)->size;
+ break;
+ case OBJ_TREE :
+ size = ((struct tree *)obj)->size;
+ break;
+ case OBJ_BLOB :
+ size = ((struct blob *)obj)->size;
+ break;
+ default :
+ /* tags are potentially dynamic metadata; they don't really belong here */
+ return;
+ }
+
+ type = obj->type;
+ }
- if (!nothisone) {
- memset(&object, 0, sizeof(object));
- hashcpy(object.sha1, sha1);
- object.type = type;
+ if (!obj || !size) {
+ void *data = read_sha1_file(sha1, &type, &size);
+
+ if (data)
+ free(data);
+ }
+
+ if (!entryp) {
+ memset(&entry, 0, sizeof(entry));
+ hashcpy(entry.sha1, sha1);
+ entry.type = type;
if (merge_str)
- object.merge_nr = merge_str->len / PATH_WIDTH;
+ entry.merge_nr = merge_str->len / PATH_WIDTH;
if (split_str)
- object.split_nr = split_str->len / PATH_WIDTH;
+ entry.split_nr = split_str->len / PATH_WIDTH;
- nothisone = &object;
+ entryp = &entry;
}
- strbuf_add(g_buffer, nothisone, sizeof(object));
+ entryp->size_size = encode_size(size, size_str);
+
+ /* write the muvabitch */
+ strbuf_add(g_buffer, entryp, sizeof(entry));
- if (merge_str && merge_str->len)
+ if (merge_str)
strbuf_add(g_buffer, merge_str->buf, merge_str->len);
- if (split_str && split_str->len)
+ if (split_str)
strbuf_add(g_buffer, split_str->buf, split_str->len);
+ strbuf_add(g_buffer, size_str, entryp->size_size);
}
/* returns non-zero to continue parsing, 0 to skip */
@@ -836,12 +985,7 @@ continue_loop:
static int dump_tree_callback(const unsigned char *sha1, const char *path, unsigned int mode)
{
- unsigned char data[21];
-
- hashcpy(data, sha1);
- data[20] = !!S_ISDIR(mode);
-
- strbuf_add(g_buffer, data, 21);
+ strbuf_add(g_buffer, sha1, 20);
return 1;
}
@@ -851,15 +995,10 @@ static void tree_addremove(struct diff_options *options,
const unsigned char *sha1,
const char *concatpath)
{
- unsigned char data[21];
-
if (whatnow != '+')
return;
- hashcpy(data, sha1);
- data[20] = !!S_ISDIR(mode);
-
- strbuf_add(g_buffer, data, 21);
+ strbuf_add(g_buffer, sha1, 20);
}
static void tree_change(struct diff_options *options,
@@ -868,26 +1007,10 @@ static void tree_change(struct diff_options *options,
const unsigned char *new_sha1,
const char *concatpath)
{
- unsigned char data[21];
-
if (!hashcmp(old_sha1, new_sha1))
return;
- hashcpy(data, new_sha1);
- data[20] = !!S_ISDIR(new_mode);
-
- strbuf_add(g_buffer, data, 21);
-}
-
-static int sort_type_hash(const void *a, const void *b)
-{
- const unsigned char *sa = (const unsigned char *)a,
- *sb = (const unsigned char *)b;
-
- if (sa[20] == sb[20])
- return hashcmp(sa, sb);
-
- return sa[20] > sb[20] ? -1 : 1;
+ strbuf_add(g_buffer, new_sha1, 20);
}
static int add_unique_objects(struct commit *commit)
@@ -898,6 +1021,7 @@ static int add_unique_objects(struct commit *commit)
int i, j, next;
char is_first = 1;
+ /* ...no, calculate unique objects */
strbuf_init(&os, 0);
strbuf_init(&ost, 0);
orig_buf = g_buffer;
@@ -917,20 +1041,20 @@ static int add_unique_objects(struct commit *commit)
strbuf_setlen(g_buffer, 0);
diff_tree_sha1(list->item->tree->object.sha1, commit->tree->object.sha1, "", &opts);
- qsort(g_buffer->buf, g_buffer->len / 21, 21, (int (*)(const void *, const void *))hashcmp);
+ qsort(g_buffer->buf, g_buffer->len / 20, 20, (int (*)(const void *, const void *))hashcmp);
/* take intersection */
if (!is_first) {
- for (next = i = j = 0; i < os.len; i += 21) {
+ for (next = i = j = 0; i < os.len; i += 20) {
while (j < ost.len && hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)) < 0)
- j += 21;
+ j += 20;
if (j >= ost.len || hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)))
continue;
if (next != i)
- memcpy(os.buf + next, os.buf + i, 21);
- next += 21;
+ memcpy(os.buf + next, os.buf + i, 20);
+ next += 20;
}
if (next != i)
@@ -939,22 +1063,21 @@ static int add_unique_objects(struct commit *commit)
is_first = 0;
}
+ /* no parents (!) */
if (is_first) {
g_buffer = &os;
dump_tree(commit->tree, dump_tree_callback);
}
- if (os.len)
- qsort(os.buf, os.len / 21, 21, sort_type_hash);
-
+ /* the ordering of non-commit objects dosn't really matter, so we're not gonna bother */
g_buffer = orig_buf;
- for (i = 0; i < os.len; i += 21)
- add_object_entry((unsigned char *)(os.buf + i), os.buf[i + 20] ? OBJ_TREE : OBJ_BLOB, 0, 0, 0);
+ for (i = 0; i < os.len; i += 20)
+ add_object_entry((unsigned char *)(os.buf + i), 0, 0, 0);
strbuf_release(&ost);
strbuf_release(&os);
- return 0;
+ return i / 20;
}
static void init_revcache_directory(void)
@@ -1046,6 +1169,9 @@ int make_cache_slice(struct rev_cache_info *rci,
if (prepare_revision_walk(revs))
die("died preparing revision walk");
+ if (rci->legs)
+ make_legs(revs);
+
object_nr = total_sz = 0;
while ((commit = get_revision(revs)) != 0) {
struct object_entry object;
@@ -1067,12 +1193,12 @@ int make_cache_slice(struct rev_cache_info *rci,
commit->indegree = 0;
- add_object_entry(0, 0, &object, &merge_paths, &split_paths);
+ add_object_entry(0, &object, &merge_paths, &split_paths);
object_nr++;
- if (!(commit->object.flags & TREESAME)) {
+ if (rci->objects && !(commit->object.flags & TREESAME)) {
/* add all unique children for this commit */
- add_object_entry(commit->tree->object.sha1, OBJ_TREE, 0, 0, 0);
+ add_object_entry(commit->tree->object.sha1, 0, 0, 0);
object_nr++;
if (!object.is_end)
@@ -1260,7 +1386,7 @@ int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
* should we allow more intelligent overriding? */
date = ntohl(object_entry->date);
if (date > idx_head.max_date) {
- entry = 0;
+ entry = 0;
if (date > max_date)
max_date = date;
} else
@@ -1316,8 +1442,7 @@ int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
}
-/* add start-commits from each cache slice (uninterestingness will be propogated) */
-void starts_from_slices(struct rev_info *revs, unsigned int flags)
+void starts_from_slices(struct rev_info *revs, unsigned int flags, unsigned char *which, int n)
{
struct commit *commit;
int i;
@@ -1327,13 +1452,24 @@ void starts_from_slices(struct rev_info *revs, unsigned int flags)
if (!idx_map)
return;
- /* haven't implemented which yet; no need really... */
for (i = idx_head.ofs_objects; i < idx_size; i += IE_SIZE) {
struct index_entry *entry = IE_CAST(idx_map + i);
if (!entry->is_start)
continue;
+ /* only include entries in 'which' slices */
+ if (n) {
+ int j;
+
+ for (j = 0; j < n; j++)
+ if (!hashcmp(idx_caches + entry->cache_index * 20, which + j * 20))
+ break;
+
+ if (j == n)
+ continue;
+ }
+
commit = lookup_commit(entry->sha1);
if (!commit)
continue;
@@ -1343,3 +1479,134 @@ void starts_from_slices(struct rev_info *revs, unsigned int flags)
}
}
+
+
+/* the most work-intensive attributes in the cache are the unique objects and size, both
+ * of which can be re-used. although path structures will be isomorphic, path generation is
+ * not particularly expensive, and at any rate we need to re-sort the commits */
+int coagulate_cache_slices(struct rev_cache_info *rci, struct rev_info *revs)
+{
+ unsigned char cache_sha1[20];
+ char base[PATH_MAX];
+ int fd, baselen, i;
+ struct stat fi;
+ struct string_list files = {0, 0, 0, 1}; /* dup */
+ struct strbuf ignore;
+ DIR *dirh;
+
+ strbuf_init(&ignore, 0);
+ strncpy(base, git_path("rev-cache"), sizeof(base));
+ baselen = strlen(base);
+
+ /* enumerate files */
+ dirh = opendir(base);
+ if (dirh) {
+ struct dirent *de;
+
+ while ((de = readdir(dirh))) {
+ if (de->d_name[0] == '.')
+ continue;
+
+ base[baselen] = '/';
+ strncpy(base + baselen + 1, de->d_name, sizeof(base) - baselen - 1);
+
+ /* _theoretically_ it is possible a slice < ignore_size to map objects not covered by, yet reachable from,
+ * a slice >= ignore_size, meaning that we could potentially delete an 'unfused' slice; but if that
+ * ever *did* happen their cache structure'd be so fucked up they might as well refuse the entire thing.
+ * and at any rate the worst it'd do is make rev-list revert to standard walking in that (small) bit.
+ */
+ if (rci->ignore_size) {
+ unsigned char sha1[20];
+
+ if (stat(base, &fi))
+ warning("can't query file %s\n", base);
+ else if (fi.st_size >= rci->ignore_size && !get_sha1_hex(de->d_name, sha1)) {
+ strbuf_add(&ignore, sha1, 20);
+ continue;
+ }
+ }
+
+ string_list_insert(base, &files);
+ }
+
+ closedir(dirh);
+ }
+
+ if (ignore.len) {
+ starts_from_slices(revs, UNINTERESTING, (unsigned char *)ignore.buf, ignore.len / 20);
+ strbuf_release(&ignore);
+ }
+
+ rci->make_index = 0;
+ if (make_cache_slice(rci, revs, 0, 0, cache_sha1) < 0)
+ die("can't make cache slice");
+
+ /* clean up time! */
+ cleanup_cache_slices();
+
+ for (i = 0; i < files.nr; i++) {
+ char *name = files.items[i].string;
+
+ fprintf(stderr, "removing %s\n", name);
+ unlink_or_warn(name);
+ }
+
+ string_list_clear(&files, 0);
+
+ fd = open(git_path("rev-cache/%s", sha1_to_hex(cache_sha1)), O_RDWR);
+ if (fd < 0 || fstat(fd, &fi))
+ die("what? I can't open/query the cache I just generated");
+
+ if (make_cache_index(rci, cache_sha1, fd, fi.st_size) < 0)
+ die("can't make new index");
+
+ close(fd);
+
+ return 0;
+}
+
+int regenerate_cache_index(struct rev_cache_info *rci)
+{
+ DIR *dirh;
+ char base[PATH_MAX];
+ int baselen;
+
+ /* first remove old index if it exists */
+ unlink_or_warn(git_path("rev-cache/index"));
+
+ strncpy(base, git_path("rev-cache"), sizeof(base));
+ baselen = strlen(base);
+
+ dirh = opendir(base);
+ if (dirh) {
+ struct dirent *de;
+ struct stat fi;
+ int fd;
+ unsigned char sha1[20];
+
+ while ((de = readdir(dirh))) {
+ if (de->d_name[0] == '.')
+ continue;
+
+ if (get_sha1_hex(de->d_name, sha1))
+ continue;
+
+ base[baselen] = '/';
+ strncpy(base + baselen + 1, de->d_name, sizeof(base) - baselen - 1);
+
+ /* open with RDWR because of mmap call in make_cache_index() */
+ fd = open(base, O_RDWR);
+ if (fd < 0 || fstat(fd, &fi))
+ warning("bad cache found [%s]; fuse recommended", de->d_name);
+
+ if (make_cache_index(rci, sha1, fd, fi.st_size) < 0)
+ die("error writing cache");
+
+ close(fd);
+ }
+
+ closedir(dirh);
+ }
+
+ return 0;
+}
\ No newline at end of file
diff --git a/revision.h b/revision.h
index 200042a..d97ac8c 100644
--- a/revision.h
+++ b/revision.h
@@ -207,6 +207,8 @@ extern int make_cache_slice(struct rev_cache_info *rci,
extern int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,
int fd, unsigned int size);
-extern void starts_from_slices(struct rev_info *revs, unsigned int flags);
+extern void starts_from_slices(struct rev_info *revs, unsigned int flags, unsigned char *which, int n);
+extern int coagulate_cache_slices(struct rev_cache_info *rci, struct rev_info *revs);
+extern int regenerate_cache_index(struct rev_cache_info *rci);
#endif
--
tg: (9861015..) t/revcache/misc (depends on: t/revcache/objects)
^ permalink raw reply related
* [PATCH 5/6 (v2)] full integration of rev-cache into git's revision walker, completed test suite
From: Nick Edelen @ 2009-08-08 7:31 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
This patch provides a working integration of rev-cache into the revision
walker, along with some touch-ups:
- integration into revision walker and list-objects
- addition of 'unique' field to commit objects, optionally initialized in
rev-cache with the objects introduced in that commit
- tweak of object generation to take advantage of the 'unique' field
- more fluid handling of damaged cache slices
- numerous tests for both features from the previous patch, and the
integration's integrity
'Integration' is rather broad -- a more detailed description follows for each
aspect:
- rev-cache
the traversal mechanism is updated to handle many of the non-prune options
rev-list does (date limiting, slop-handling, etc.), and is adjusted to allow
for non-fatal cache-traversal failures.
- revision walker
both limited and unlimited traversal attempt to use the cache when possible,
smoothly falling back if it's not.
- list-objects
object listing does not recurse into cached trees, and has been adjusted to
guarantee commit-tag-tree-blob ordering.
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
5/5 in previous. I decided to not split this up, as Sam suggested, largely
because the changesets, while seemingly numerous, are actually rather small,
and very connected with each other. Tests for --ignore-size are also added.
builtin-rev-cache.c | 46 +++++++-
commit.c | 2 +-
commit.h | 1 +
list-objects.c | 49 ++++++-
rev-cache.c | 303 ++++++++++++++++++++++++++++++++++++++------
revision.c | 87 +++++++++++---
t/t6015-rev-cache-list.sh | 152 +++++++++++++++++++++--
7 files changed, 562 insertions(+), 78 deletions(-)
diff --git a/builtin-rev-cache.c b/builtin-rev-cache.c
index 532ec9f..4a02af6 100755
--- a/builtin-rev-cache.c
+++ b/builtin-rev-cache.c
@@ -3,6 +3,7 @@
#include "commit.h"
#include "diff.h"
#include "revision.h"
+#include "list-objects.h"
#define DEFAULT_IGNORE_SLICE_SIZE "30mb"
@@ -69,6 +70,43 @@ static int handle_add(int argc, const char *argv[]) /* args beyond this command
return 0;
}
+static void show_commit(struct commit *commit, void *data)
+{
+ printf("%s\n", sha1_to_hex(commit->object.sha1));
+}
+
+static void show_object(struct object *obj, const struct name_path *path, const char *last)
+{
+ printf("%s\n", sha1_to_hex(obj->sha1));
+}
+
+static int test_rev_list(int argc, const char *argv[])
+{
+ struct rev_info revs;
+ unsigned int flags = 0;
+ int i;
+
+ init_revisions(&revs, 0);
+
+ for (i = 0; i < argc; i++) {
+ if (!strcmp(argv[i], "--not"))
+ flags ^= UNINTERESTING;
+ else if (!strcmp(argv[i], "--objects"))
+ revs.tree_objects = revs.blob_objects = 1;
+ else
+ handle_revision_arg(argv[i], &revs, flags, 1);
+ }
+
+ setup_revisions(0, 0, &revs, 0);
+ revs.topo_order = 1;
+ revs.lifo = 1;
+ prepare_revision_walk(&revs);
+
+ traverse_commit_list(&revs, show_commit, show_object, 0);
+
+ return 0;
+}
+
static int handle_walk(int argc, const char *argv[])
{
struct commit *commit;
@@ -117,17 +155,17 @@ static int handle_walk(int argc, const char *argv[])
if (retval < 0)
return retval;
- printf("queue:\n");
+ fprintf(stderr, "queue:\n");
while ((commit = pop_commit(&queue)) != 0) {
printf("%s\n", sha1_to_hex(commit->object.sha1));
}
- printf("work:\n");
+ fprintf(stderr, "work:\n");
while ((commit = pop_commit(&work)) != 0) {
printf("%s\n", sha1_to_hex(commit->object.sha1));
}
- printf("pending:\n");
+ fprintf(stderr, "pending:\n");
for (i = 0; i < revs.pending.nr; i++) {
struct object *obj = revs.pending.objects[i].item;
@@ -259,6 +297,8 @@ int cmd_rev_cache(int argc, const char *argv[], const char *prefix)
r = handle_walk(argc, argv);
else if (!strcmp(arg, "index"))
r = handle_index(argc, argv);
+ else if (!strcmp(arg, "test"))
+ r = test_rev_list(argc, argv);
else
return handle_help();
diff --git a/commit.c b/commit.c
index afe504f..1b710c2 100644
--- a/commit.c
+++ b/commit.c
@@ -252,7 +252,7 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
sha1_to_hex(item->object.sha1));
item->tree = lookup_tree(parent);
bufptr += 46; /* "tree " + "hex sha1" + "\n" */
- pptr = &item->parents;
+ pptr = &item->parents;
while (pop_commit(pptr))
; /* clear anything from cache */
diff --git a/commit.h b/commit.h
index 0529fcf..7c50a7f 100644
--- a/commit.h
+++ b/commit.h
@@ -20,6 +20,7 @@ struct commit {
struct tree *tree;
char *buffer;
unsigned long size;
+ struct object_list *unique;
};
extern int save_commit_buffer;
diff --git a/list-objects.c b/list-objects.c
index 8953548..958c0f8 100644
--- a/list-objects.c
+++ b/list-objects.c
@@ -74,22 +74,33 @@ static void process_tree(struct rev_info *revs,
die("bad tree object");
if (obj->flags & (UNINTERESTING | SEEN))
return;
- if (parse_tree(tree) < 0)
- die("bad tree object %s", sha1_to_hex(obj->sha1));
+
obj->flags |= SEEN;
show(obj, path, name);
+ if (obj->flags & FACE_VALUE)
+ return;
+
+ /* traverse_commit_list is only used for enumeration purposes,
+ * ie. nothing relies on trees being parsed in this routine */
+ if (parse_tree(tree) < 0)
+ die("bad tree object %s", sha1_to_hex(obj->sha1));
+
me.up = path;
me.elem = name;
me.elem_len = strlen(name);
-
init_tree_desc(&desc, tree->buffer, tree->size);
while (tree_entry(&desc, &entry)) {
- if (S_ISDIR(entry.mode))
+ if (S_ISDIR(entry.mode)) {
+ struct tree *subtree = lookup_tree(entry.sha1);
+ if (!subtree)
+ continue;
+
+ subtree->object.flags &= ~FACE_VALUE;
process_tree(revs,
- lookup_tree(entry.sha1),
+ subtree,
show, &me, entry.path);
- else if (S_ISGITLINK(entry.mode))
+ } else if (S_ISGITLINK(entry.mode))
process_gitlink(revs, entry.sha1,
show, &me, entry.path);
else
@@ -136,6 +147,7 @@ void mark_edges_uninteresting(struct commit_list *list,
static void add_pending_tree(struct rev_info *revs, struct tree *tree)
{
+ tree->object.flags &= ~FACE_VALUE;
add_pending_object(revs, &tree->object, "");
}
@@ -146,17 +158,27 @@ void traverse_commit_list(struct rev_info *revs,
{
int i;
struct commit *commit;
+ enum object_type what = OBJ_TAG;
+ char face_value = 0;
while ((commit = get_revision(revs)) != NULL) {
- add_pending_tree(revs, commit->tree);
+ if (!(commit->object.flags & FACE_VALUE))
+ add_pending_tree(revs, commit->tree);
+ else
+ face_value = 1;
show_commit(commit, data);
}
+
+loop_objects:
for (i = 0; i < revs->pending.nr; i++) {
struct object_array_entry *pending = revs->pending.objects + i;
struct object *obj = pending->item;
const char *name = pending->name;
if (obj->flags & (UNINTERESTING | SEEN))
continue;
+ if (obj->type != what && face_value)
+ continue;
+
if (obj->type == OBJ_TAG) {
obj->flags |= SEEN;
show_object(obj, NULL, name);
@@ -175,6 +197,19 @@ void traverse_commit_list(struct rev_info *revs,
die("unknown pending object %s (%s)",
sha1_to_hex(obj->sha1), name);
}
+ if (face_value) {
+ switch (what) {
+ case OBJ_TAG :
+ what = OBJ_TREE;
+ goto loop_objects;
+ case OBJ_TREE :
+ what = OBJ_BLOB;
+ goto loop_objects;
+ default :
+ break;
+ }
+ }
+
if (revs->pending.nr) {
free(revs->pending.objects);
revs->pending.nr = 0;
diff --git a/rev-cache.c b/rev-cache.c
index 4cda858..e6808e7 100755
--- a/rev-cache.c
+++ b/rev-cache.c
@@ -65,14 +65,20 @@ struct object_entry {
/* size */
};
+struct bad_slice {
+ unsigned char sha1[20];
+ struct bad_slice *next;
+};
+
/* list resembles pack index format */
static uint32_t fanout[0xff + 2];
static unsigned char *idx_map;
static int idx_size;
static struct index_header idx_head;
+static char no_idx, save_unique, add_to_pending;
+static struct bad_slice *bad_slices;
static unsigned char *idx_caches;
-static char no_idx;
static struct strbuf *g_buffer;
@@ -91,10 +97,36 @@ static struct strbuf *g_buffer;
#define ACTUAL_OBJECT_ENTRY_SIZE(e) (OE_SIZE + PATH_SIZE((e)->merge_nr + (e)->split_nr) + (e)->size_size)
#define ENTRY_SIZE_OFFSET(e) (ACTUAL_OBJECT_ENTRY_SIZE(e) - (e)->size_size)
-#define SLOP 5
+#define SLOP 5
+
+#define HAS_UNIQUES FACE_VALUE
/* initialization */
+static void mark_bad_slice(unsigned char *sha1)
+{
+ struct bad_slice *bad;
+
+ bad = xcalloc(sizeof(struct bad_slice), 1);
+ hashcpy(bad->sha1, sha1);
+
+ bad->next = bad_slices;
+ bad_slices = bad;
+}
+
+static int is_bad_slice(unsigned char *sha1)
+{
+ struct bad_slice *bad = bad_slices;
+
+ while (bad) {
+ if (!hashcmp(bad->sha1, sha1))
+ return 1;
+ bad = bad->next;
+ }
+
+ return 0;
+}
+
static int get_index_head(unsigned char *map, int len, struct index_header *head, uint32_t *fanout, unsigned char **caches)
{
struct index_header whead;
@@ -210,6 +242,7 @@ static struct index_entry *search_index(unsigned char *sha1)
unsigned char *get_cache_slice(struct commit *commit)
{
struct index_entry *ie;
+ unsigned char *sha1;
if (!idx_map) {
if (no_idx)
@@ -221,8 +254,13 @@ unsigned char *get_cache_slice(struct commit *commit)
return 0;
ie = search_index(commit->object.sha1);
- if (ie && ie->cache_index < idx_head.cache_nr)
- return idx_caches + ie->cache_index * 20;
+ if (ie && ie->cache_index < idx_head.cache_nr) {
+ sha1 = idx_caches + ie->cache_index * 20;
+
+ if (is_bad_slice(sha1))
+ return 0;
+ return sha1;
+ }
return 0;
}
@@ -232,8 +270,29 @@ unsigned char *get_cache_slice(struct commit *commit)
static unsigned long decode_size(unsigned char *str, int len);
+/* on failure */
+static void restore_commit(struct commit *commit)
+{
+ if (commit->unique) {
+ free(commit->unique);
+ commit->unique = 0;
+ }
+
+ commit->object.flags &= ~(ADDED | SEEN | FACE_VALUE);
+
+ if (!commit->object.parsed) {
+ while (pop_commit(&commit->parents))
+ ;
+
+ parse_commit(commit);
+ }
+
+}
+
static void handle_noncommit(struct rev_info *revs, struct commit *commit, struct object_entry *entry)
{
+ static struct commit *last_commit = 0;
+ static struct object_list **last_unique = 0;
struct blob *blob;
struct tree *tree;
struct object *obj;
@@ -271,11 +330,27 @@ static void handle_noncommit(struct rev_info *revs, struct commit *commit, struc
return;
}
+ /* add to unique list if we're not an end */
+ if (save_unique && (commit->object.flags & FACE_VALUE)) {
+ if (last_commit != commit) {
+ last_commit = commit;
+ last_unique = 0;
+ }
+
+ if (!last_unique)
+ last_unique = &commit->unique;
+
+ object_list_append(obj, last_unique);
+ last_unique = &(*last_unique)->next;
+ }
+
obj->flags |= FACE_VALUE;
- add_pending_object(revs, obj, "");
+ if (add_to_pending)
+ add_pending_object(revs, obj, "");
}
-static int setup_traversal(struct cache_slice_header *head, unsigned char *map, struct commit *commit, struct commit_list **work)
+static int setup_traversal(struct cache_slice_header *head, unsigned char *map, struct commit *commit, struct commit_list **work,
+ struct commit_list **unwork, int *ipath_nr, int *upath_nr, char *ioutside)
{
struct index_entry *iep;
struct object_entry *oep;
@@ -284,6 +359,11 @@ static int setup_traversal(struct cache_slice_header *head, unsigned char *map,
iep = search_index(commit->object.sha1);
oep = OE_CAST(map + ntohl(iep->pos));
+ if (commit->object.flags & UNINTERESTING) {
+ ++*upath_nr;
+ oep->uninteresting = 1;
+ } else
+ ++*ipath_nr;
oep->include = 1;
retval = ntohl(iep->pos);
@@ -299,10 +379,14 @@ static int setup_traversal(struct cache_slice_header *head, unsigned char *map,
/* is this in our cache slice? */
iep = search_index(obj->sha1);
if (!iep || hashcmp(idx_caches + iep->cache_index * 20, head->sha1)) {
+ /* there are interesing objects outside the slice */
+ if (!(obj->flags & UNINTERESTING))
+ *ioutside = 1;
+
prev = wp;
wp = wp->next;
wpp = ℘
- continue;
+ continue;
}
t = ntohl(iep->pos);
@@ -313,11 +397,20 @@ static int setup_traversal(struct cache_slice_header *head, unsigned char *map,
if (t < retval)
retval = t;
+ /* count even if not in slice so we can stop enumerating if possible */
+ if (obj->flags & UNINTERESTING)
+ ++*upath_nr;
+ else
+ ++*ipath_nr;
+
/* remove from work list */
co = pop_commit(wpp);
wp = *wpp;
if (prev)
prev->next = wp;
+
+ /* ...and store in temp list so we can restore work on failure */
+ commit_list_insert(co, unwork);
}
return retval;
@@ -334,16 +427,21 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char
unsigned long *date_so_far, int *slop_so_far,
struct commit_list ***queue, struct commit_list **work)
{
- struct commit_list *insert_cache = 0;
+ struct commit_list *insert_cache = 0, *myq = 0, **myqp = &myq, *mywork = 0, **myworkp = &mywork, *unwork = 0;
struct commit **last_objects, *co;
- int i, total_path_nr = head->path_nr, retval = -1;
- char consume_children = 0;
+ unsigned long date = date_so_far ? *date_so_far : ~0ul;
+ int i, ipath_nr = 0, upath_nr = 0, orig_obj_nr = 0,
+ total_path_nr = head->path_nr, retval = -1, slop = slop_so_far ? *slop_so_far : SLOP;
+ char consume_children = 0, ioutside = 0;
unsigned char *paths;
+ /* take note in case we need to regress */
+ orig_obj_nr = revs->pending.nr;
+
paths = xcalloc(total_path_nr, PATH_WIDTH);
last_objects = xcalloc(total_path_nr, sizeof(struct commit *));
- i = setup_traversal(head, map, commit, work);
+ i = setup_traversal(head, map, commit, work, &unwork, &ipath_nr, &upath_nr, &ioutside);
/* i already set */
while (i < head->size) {
@@ -386,6 +484,7 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char
if ((paths[path] & IPATH) && (paths[path] & UPATH)) {
paths[path] = UPATH;
+ ipath_nr--;
/* mark edge */
if (last_objects[path]) {
@@ -396,6 +495,7 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char
last_objects[path]->object.flags &= ~FACE_VALUE;
last_objects[path] = 0;
}
+ obj->flags |= BOUNDARY;
}
/* now we gotta re-assess the whole interesting thing... */
@@ -419,8 +519,10 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char
last_objects[p]->object.flags &= ~FACE_VALUE;
last_objects[p] = 0;
}
- } else if (last_objects[p] && !last_objects[p]->object.parsed)
+ obj->flags |= BOUNDARY;
+ } else if (last_objects[p] && !last_objects[p]->object.parsed) {
commit_list_insert(co, &last_objects[p]->parents);
+ }
/* can't close a merge path until all are parents have been encountered */
if (GET_COUNT(paths[p])) {
@@ -430,44 +532,90 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char
continue;
}
+ if (paths[p] & IPATH)
+ ipath_nr--;
+ else
+ upath_nr--;
+
paths[p] = 0;
last_objects[p] = 0;
}
}
/* make topo relations */
- if (last_objects[path] && !last_objects[path]->object.parsed)
+ if (last_objects[path] && !last_objects[path]->object.parsed) {
commit_list_insert(co, &last_objects[path]->parents);
+ }
+
+ /* we've been here already */
+ if (obj->flags & ADDED) {
+ if (entry->uninteresting && !(obj->flags & UNINTERESTING)) {
+ obj->flags |= UNINTERESTING;
+ mark_parents_uninteresting(co);
+ upath_nr--;
+ } else if (!entry->uninteresting)
+ ipath_nr--;
+
+ paths[path] = 0;
+ continue;
+ }
/* initialize commit */
if (!entry->is_end) {
co->date = ntohl(entry->date);
- obj->flags |= ADDED | FACE_VALUE;
+ obj->flags |= ADDED | FACE_VALUE;
} else
parse_commit(co);
obj->flags |= SEEN;
-
- if (entry->uninteresting)
- obj->flags |= UNINTERESTING;
+
+ if (entry->uninteresting)
+ obj->flags |= UNINTERESTING;
+ else if (co->date < date)
+ date = co->date;
/* we need to know what the edges are */
last_objects[path] = co;
/* add to list */
- if (!(obj->flags & UNINTERESTING) || revs->show_all) {
- if (entry->is_end)
- insert_by_date_cached(co, work, insert_cache, &insert_cache);
- else
- *queue = &commit_list_insert(co, *queue)->next;
+ if (slop && !(revs->min_age != -1 && co->date > revs->min_age)) {
+
+ if (!(obj->flags & UNINTERESTING) || revs->show_all) {
+ if (entry->is_end)
+ myworkp = &commit_list_insert(co, myworkp)->next;
+ else
+ myqp = &commit_list_insert(co, myqp)->next;
+
+ /* add children to list as well */
+ if (obj->flags & UNINTERESTING)
+ consume_children = 0;
+ else
+ consume_children = 1;
+ }
- /* add children to list as well */
- if (obj->flags & UNINTERESTING)
- consume_children = 0;
- else
- consume_children = 1;
}
+ /* should we continue? */
+ if (!slop) {
+ if (!upath_nr) {
+ break;
+ } else if (ioutside || revs->show_all) {
+ /* pass it back to rev-list
+ * we purposely ignore everything outside this cache, so we don't needlessly traverse the whole
+ * thing on uninteresting, but that does mean that we may need to bounce back
+ * and forth a few times with rev-list */
+ myworkp = &commit_list_insert(co, myworkp)->next;
+
+ paths[path] = 0;
+ upath_nr--;
+ } else {
+ break;
+ }
+ } else if (!ipath_nr && co->date <= date)
+ slop--;
+ else
+ slop = SLOP;
+
/* open parents */
if (entry->merge_nr) {
int j, off = index + OE_SIZE;
@@ -482,6 +630,11 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char
if (paths[p] & flag)
continue;
+ if (flag == IPATH)
+ ipath_nr++;
+ else
+ upath_nr++;
+
paths[p] |= flag;
}
@@ -491,12 +644,55 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char
}
+ if (date_so_far)
+ *date_so_far = date;
+ if (slop_so_far)
+ *slop_so_far = slop;
retval = 0;
+ /* success: attach to given lists */
+ if (myqp != &myq) {
+ **queue = myq;
+ *queue = myqp;
+ }
+
+ while ((co = pop_commit(&mywork)) != 0) {
+ insert_by_date_cached(co, work, insert_cache, &insert_cache);
+ }
+
+ /* free backup */
+ while (pop_commit(&unwork))
+ ;
+
end:
free(paths);
free(last_objects);
+ /* failure: restore work to previous condition
+ * (cache corruption should *not* be fatal) */
+ if (retval) {
+ while ((co = pop_commit(&unwork)) != 0) {
+ restore_commit(co);
+ co->object.flags |= SEEN;
+ insert_by_date(co, work);
+ }
+
+ /* free lists */
+ while ((co = pop_commit(&myq)) != 0)
+ restore_commit(co);
+
+ while ((co = pop_commit(&mywork)) != 0)
+ restore_commit(co);
+
+ /* truncate object array */
+ for (i = orig_obj_nr; i < revs->pending.nr; i++) {
+ struct object *obj = revs->pending.objects[i].item;
+
+ obj->flags &= ~FACE_VALUE;
+ }
+ revs->pending.nr = orig_obj_nr;
+ }
+
return retval;
}
@@ -545,6 +741,8 @@ int traverse_cache_slice(struct rev_info *revs,
/* load options */
rci = &revs->rev_cache_info;
+ save_unique = rci->save_unique;
+ add_to_pending = rci->add_to_pending;
memset(&head, 0, sizeof(struct cache_slice_header));
@@ -568,6 +766,10 @@ end:
if (fd != -1)
close(fd);
+ /* remember this! */
+ if (retval)
+ mark_bad_slice(cache_sha1);
+
return retval;
}
@@ -762,6 +964,7 @@ static void handle_paths(struct commit *commit, struct object_entry *object, str
if (pt->commit == commit) {
if (paths[pt->path] != PATH_IN_USE)
paths[pt->path]--;
+
remove_path_track(ppt, 0);
pt = *ppt;
} else {
@@ -1021,6 +1224,21 @@ static int add_unique_objects(struct commit *commit)
int i, j, next;
char is_first = 1;
+ /* but wait! is this itself from a slice? */
+ if (commit->unique) {
+ struct object_list *olist;
+
+ olist = commit->unique;
+ i = 0;
+ while (olist) {
+ add_object_entry(olist->item->sha1, 0, 0, 0);
+ i++;
+ olist = olist->next;
+ }
+
+ return i;
+ }
+
/* ...no, calculate unique objects */
strbuf_init(&os, 0);
strbuf_init(&ost, 0);
@@ -1074,10 +1292,13 @@ static int add_unique_objects(struct commit *commit)
for (i = 0; i < os.len; i += 20)
add_object_entry((unsigned char *)(os.buf + i), 0, 0, 0);
+ /* last but not least, the main tree */
+ add_object_entry(commit->tree->object.sha1, 0, 0, 0);
+
strbuf_release(&ost);
strbuf_release(&os);
- return i / 20;
+ return i / 20 + 1;
}
static void init_revcache_directory(void)
@@ -1196,14 +1417,8 @@ int make_cache_slice(struct rev_cache_info *rci,
add_object_entry(0, &object, &merge_paths, &split_paths);
object_nr++;
- if (rci->objects && !(commit->object.flags & TREESAME)) {
- /* add all unique children for this commit */
- add_object_entry(commit->tree->object.sha1, 0, 0, 0);
- object_nr++;
-
- if (!object.is_end)
- object_nr += add_unique_objects(commit);
- }
+ if (rci->objects && !(commit->object.flags & TREESAME) && !object.is_end)
+ object_nr += add_unique_objects(commit);
/* print every ~1MB or so */
if (buffer.len > 1000000) {
@@ -1481,7 +1696,6 @@ void starts_from_slices(struct rev_info *revs, unsigned int flags, unsigned char
}
-
/* the most work-intensive attributes in the cache are the unique objects and size, both
* of which can be re-used. although path structures will be isomorphic, path generation is
* not particularly expensive, and at any rate we need to re-sort the commits */
@@ -1527,7 +1741,7 @@ int coagulate_cache_slices(struct rev_cache_info *rci, struct rev_info *revs)
}
}
- string_list_insert(base, &files);
+ string_list_insert(de->d_name, &files);
}
closedir(dirh);
@@ -1548,15 +1762,20 @@ int coagulate_cache_slices(struct rev_cache_info *rci, struct rev_info *revs)
for (i = 0; i < files.nr; i++) {
char *name = files.items[i].string;
- fprintf(stderr, "removing %s\n", name);
- unlink_or_warn(name);
+ /* in the odd case of only having one cache slice we effectively just remaking the index... */
+ if (strlen(name) >= 40 && !strncmp(name, sha1_to_hex(cache_sha1), 40))
+ continue;
+
+ strncpy(base + baselen + 1, name, sizeof(base) - baselen - 1);
+ fprintf(stderr, "removing %s\n", base);
+ unlink_or_warn(base);
}
string_list_clear(&files, 0);
fd = open(git_path("rev-cache/%s", sha1_to_hex(cache_sha1)), O_RDWR);
if (fd < 0 || fstat(fd, &fi))
- die("what? I can't open/query the cache I just generated");
+ die("what? I can't open/query the cache I just generated\n (sha1: %s)", sha1_to_hex(cache_sha1));
if (make_cache_index(rci, cache_sha1, fd, fi.st_size) < 0)
die("can't make new index");
@@ -1610,4 +1829,4 @@ int regenerate_cache_index(struct rev_cache_info *rci)
}
return 0;
-}
\ No newline at end of file
+}
diff --git a/revision.c b/revision.c
index 485bf72..e76fd1d 100644
--- a/revision.c
+++ b/revision.c
@@ -638,6 +638,8 @@ static int limit_list(struct rev_info *revs)
struct commit_list *list = revs->commits;
struct commit_list *newlist = NULL;
struct commit_list **p = &newlist;
+ unsigned char *cache_sha1;
+ char used_cache;
while (list) {
struct commit_list *entry = list;
@@ -650,24 +652,39 @@ static int limit_list(struct rev_info *revs)
if (revs->max_age != -1 && (commit->date < revs->max_age))
obj->flags |= UNINTERESTING;
- if (add_parents_to_list(revs, commit, &list, NULL) < 0)
- return -1;
- if (obj->flags & UNINTERESTING) {
- mark_parents_uninteresting(commit);
- if (revs->show_all)
- p = &commit_list_insert(commit, p)->next;
- slop = still_interesting(list, date, slop);
- if (slop)
+
+ /* rev-cache to the rescue!!! */
+ used_cache = 0;
+ if (!revs->dont_cache_me && !(obj->flags & ADDED)) {
+ cache_sha1 = get_cache_slice(commit);
+ if (cache_sha1) {
+ if (traverse_cache_slice(revs, cache_sha1, commit, &date, &slop, &p, &list) < 0)
+ used_cache = 0;
+ else
+ used_cache = 1;
+ }
+ }
+
+ if (!used_cache) {
+ if (add_parents_to_list(revs, commit, &list, NULL) < 0)
+ return -1;
+ if (obj->flags & UNINTERESTING) {
+ mark_parents_uninteresting(commit); /* ME: why? */
+ if (revs->show_all)
+ p = &commit_list_insert(commit, p)->next;
+ slop = still_interesting(list, date, slop);
+ if (slop > 0)
+ continue;
+ /* If showing all, add the whole pending list to the end */
+ if (revs->show_all)
+ *p = list;
+ break;
+ }
+ if (revs->min_age != -1 && (commit->date > revs->min_age))
continue;
- /* If showing all, add the whole pending list to the end */
- if (revs->show_all)
- *p = list;
- break;
+ date = commit->date;
+ p = &commit_list_insert(commit, p)->next;
}
- if (revs->min_age != -1 && (commit->date > revs->min_age))
- continue;
- date = commit->date;
- p = &commit_list_insert(commit, p)->next;
show = show_early_output;
if (!show)
@@ -813,6 +830,8 @@ void init_revisions(struct rev_info *revs, const char *prefix)
revs->diffopt.prefix = prefix;
revs->diffopt.prefix_length = strlen(prefix);
}
+
+ init_rci(&revs->rev_cache_info);
}
static void add_pending_commit_list(struct rev_info *revs,
@@ -1372,6 +1391,11 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, const ch
if (revs->reflog_info && revs->graph)
die("cannot combine --walk-reflogs with --graph");
+ /* limits on caching
+ * todo: implement this functionality */
+ if (revs->prune || revs->diff)
+ revs->dont_cache_me = 1;
+
return left;
}
@@ -1654,6 +1678,8 @@ static int commit_match(struct commit *commit, struct rev_info *opt)
{
if (!opt->grep_filter.pattern_list)
return 1;
+ if (!commit->object.parsed)
+ parse_commit(commit);
return grep_buffer(&opt->grep_filter,
NULL, /* we say nothing, not even filename */
commit->buffer, strlen(commit->buffer));
@@ -1706,6 +1732,7 @@ static struct commit *get_revision_1(struct rev_info *revs)
do {
struct commit_list *entry = revs->commits;
struct commit *commit = entry->item;
+ struct object *obj = &commit->object;
revs->commits = entry->next;
free(entry);
@@ -1722,11 +1749,39 @@ static struct commit *get_revision_1(struct rev_info *revs)
if (revs->max_age != -1 &&
(commit->date < revs->max_age))
continue;
+
+ if (!revs->dont_cache_me) {
+ struct commit_list *queue = 0, **queuep = &queue;;
+ unsigned char *cache_sha1;
+
+ if (obj->flags & ADDED)
+ goto skip_parenting;
+
+ cache_sha1 = get_cache_slice(commit);
+ if (cache_sha1) {
+ if (!traverse_cache_slice(revs, cache_sha1, commit, 0, 0, &queuep, &revs->commits)) {
+ struct commit_list *work = revs->commits;
+
+ /* attach queue to end of ->commits */
+ while (work && work->next)
+ work = work->next;
+
+ if (work)
+ work->next = queue;
+ else
+ revs->commits = queue;
+
+ goto skip_parenting;
+ }
+ }
+ }
+
if (add_parents_to_list(revs, commit, &revs->commits, NULL) < 0)
die("Failed to traverse parents of commit %s",
sha1_to_hex(commit->object.sha1));
}
+skip_parenting:
switch (simplify_commit(revs, commit)) {
case commit_ignore:
continue;
diff --git a/t/t6015-rev-cache-list.sh b/t/t6015-rev-cache-list.sh
index 8aea2e3..561810f 100755
--- a/t/t6015-rev-cache-list.sh
+++ b/t/t6015-rev-cache-list.sh
@@ -37,7 +37,8 @@ test_expect_success 'init repo' '
echo omg >smoke/bong &&
git add . &&
git commit -m "omg" &&
-
+
+ sleep 2 &&
git branch b4 &&
git checkout b4 &&
echo shazam >file8 &&
@@ -46,9 +47,9 @@ test_expect_success 'init repo' '
git merge -m "merge b2" b2 &&
echo bam >smoke/pipe &&
- git add .
+ git add . &&
git commit -m "bam" &&
-
+
git checkout master &&
echo pow >file7 &&
git add . &&
@@ -71,18 +72,26 @@ test_expect_success 'init repo' '
git add . &&
git commit -m "lol" &&
+ sleep 2 &&
git checkout master &&
git merge -m "triple merge" b1 b11 &&
git rm -r d1 &&
+ sleep 2 &&
git commit -a -m "oh noes"
'
-git-rev-list HEAD --not HEAD~3 >proper_commit_list_limited
-git-rev-list HEAD >proper_commit_list
-git-rev-list HEAD --objects >proper_object_list
+max_date=`git-rev-list --timestamp HEAD~1 --max-count=1 | grep -e "^[0-9]*" -o`
+min_date=`git-rev-list --timestamp b4 --max-count=1 | grep -e "^[0-9]*" -o`
+
+git-rev-list --topo-order HEAD --not HEAD~3 >proper_commit_list_limited
+git-rev-list --topo-order HEAD --not HEAD~2 >proper_commit_list_limited2
+git-rev-list --topo-order HEAD >proper_commit_list
+git-rev-list --objects HEAD >proper_object_list
+git-rev-list HEAD --max-age=$min_date --min-age=$max_date >proper_list_date_limited
+
+cache_sha1=`git-rev-cache add HEAD 2>output.err`
test_expect_success 'make cache slice' '
- git-rev-cache add HEAD 2>output.err &&
grep "final return value: 0" output.err
'
@@ -98,15 +107,140 @@ test_expect_success 'test rev-caches walker directly (limited)' '
'
test_expect_success 'test rev-caches walker directly (unlimited)' '
- git-rev-cache walk HEAD >list &&
+ git-rev-cache walk HEAD >list &&
test_cmp_sorted list proper_commit_list
'
+test_expect_success 'test rev-list traversal (limited)' '
+ git-rev-list HEAD --not HEAD~3 >list &&
+ test_cmp list proper_commit_list_limited
+'
+
+test_expect_success 'test rev-list traversal (unlimited)' '
+ git-rev-list HEAD >list &&
+ test_cmp list proper_commit_list
+'
+
#do the same for objects
test_expect_success 'test rev-caches walker with objects' '
git-rev-cache walk --objects HEAD >list &&
test_cmp_sorted list proper_object_list
'
-test_done
+test_expect_success 'test rev-list with objects (topo order)' '
+ git-rev-list --topo-order --objects HEAD >list &&
+ test_cmp_sorted list proper_object_list
+'
+
+test_expect_success 'test rev-list with objects (no order)' '
+ git-rev-list --objects HEAD >list &&
+ test_cmp_sorted list proper_object_list
+'
+
+#verify age limiting
+test_expect_success 'test rev-list date limiting (topo order)' '
+ git-rev-list --topo-order --max-age=$min_date --min-age=$max_date HEAD >list &&
+ test_cmp_sorted list proper_list_date_limited
+'
+
+test_expect_success 'test rev-list date limiting (no order)' '
+ git-rev-list --max-age=$min_date --min-age=$max_date HEAD >list &&
+ test_cmp_sorted list proper_list_date_limited
+'
+
+#check partial cache slice
+test_expect_success 'saving old cache and generating partial slice' '
+ cp ".git/rev-cache/$cache_sha1" .git/rev-cache/.old &&
+ rm ".git/rev-cache/$cache_sha1" .git/rev-cache/index &&
+
+ git-rev-cache add HEAD~2 2>output.err &&
+ grep "final return value: 0" output.err
+'
+
+test_expect_success 'rev-list with wholly interesting partial slice' '
+ git-rev-list --topo-order HEAD >list &&
+ test_cmp list proper_commit_list
+'
+
+test_expect_success 'rev-list with partly uninteresting partial slice' '
+ git-rev-list --topo-order HEAD --not HEAD~3 >list &&
+ test_cmp list proper_commit_list_limited
+'
+
+test_expect_success 'rev-list with wholly uninteresting partial slice' '
+ git-rev-list --topo-order HEAD --not HEAD~2 >list &&
+ test_cmp list proper_commit_list_limited2
+'
+#try out index generation and fuse (note that --all == HEAD in this case)
+#probably should make a test for that too...
+test_expect_success 'make fresh slice' '
+ git-rev-cache add --all --fresh 2>output.err &&
+ grep "final return value: 0" output.err
+'
+
+test_expect_success 'check dual slices' '
+ git-rev-list --topo-order HEAD~2 HEAD >list &&
+ test_cmp list proper_commit_list
+'
+
+test_expect_success 'regenerate index' '
+ rm .git/rev-cache/index &&
+ git-rev-cache index 2>output.err &&
+ grep "final return value: 0" output.err
+'
+
+test_expect_success 'fuse slices' '
+ test -e .git/rev-cache/.old &&
+ git-rev-cache fuse 2>output.err &&
+ grep "final return value: 0" output.err &&
+ test_cmp .git/rev-cache/$cache_sha1 .git/rev-cache/.old
+'
+
+#make sure we can smoothly handle corrupted caches
+test_expect_success 'corrupt slice' '
+ echo bla >.git/rev-cache/$cache_sha1
+'
+
+test_expect_success 'test rev-list traversal (limited) (corrupt slice)' '
+ git-rev-list --topo-order HEAD --not HEAD~3 >list &&
+ test_cmp list proper_commit_list_limited
+'
+
+test_expect_success 'test rev-list traversal (unlimited) (corrupt slice)' '
+ git-rev-list HEAD >list &&
+ test_cmp_sorted list proper_commit_list
+'
+
+test_expect_success 'corrupt index' '
+ echo blu >.git/rev-cache/index
+'
+
+test_expect_success 'test rev-list traversal (limited) (corrupt index)' '
+ git-rev-list --topo-order HEAD --not HEAD~3 >list &&
+ test_cmp list proper_commit_list_limited
+'
+
+test_expect_success 'test rev-list traversal (unlimited) (corrupt index)' '
+ git-rev-list HEAD >list &&
+ test_cmp_sorted list proper_commit_list
+'
+
+#test --ignore-size in fuse
+rm .git/rev-cache/*
+cache_sha1=`git-rev-cache add HEAD --not HEAD~3 2>output.err`
+
+test_expect_success 'make fragmented slices' '
+ git-rev-cache add HEAD~2 --not HEAD~3 2>>output.err &&
+ git-rev-cache add HEAD --fresh 2>>output.err &&
+ test `grep "final return value: 0" output.err | wc -l` -eq 3
+'
+
+cache_size=`wc -c .git/rev-cache/$cache_sha1 | grep -o "[0-9]*"`
+test_expect_success 'test --ignore-size function in fuse' '
+ git-rev-cache fuse --ignore-size=$cache_size 2>output.err &&
+ grep "final return value: 0" output.err &&
+ test -e .git/rev-cache/$cache_sha1
+'
+
+test_done
--
tg: (c7a46b0..) t/revcache/integration (depends on: t/revcache/misc)
^ permalink raw reply related
* [PATCH 6/6 (v2)] support for path name caching of blobs/trees in rev-cache
From: Nick Edelen @ 2009-08-08 7:32 UTC (permalink / raw)
To: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber
An update to caching mechanism, allowing path names to be cached for blob and
tree objects. A list of names appearing in each cache slice is appended to the
end of the slice, which is referenced by variable-sized indexes per entry.
This allows pack-objects to more intelligently schedule unpacked/poorly packed
object, and enables proper duplication of rev-list's behaivor.
The mechanism for this involves adding a 'name' field to blob and tree objects,
mainly to facilitate reuse of caches during maintenence (like the 'unique'
field).
Signed-off-by: Nick Edelen <sirnot@gmail.com>
---
This is a totally new patch to the series. I had somehow missed name caching
earlier.
blob.h | 1 +
builtin-rev-cache.c | 3 +-
rev-cache.c | 293 ++++++++++++++++++++++++++++++++++++++-------
revision.h | 3 +-
t/t6015-rev-cache-list.sh | 4 +-
tree.h | 1 +
6 files changed, 259 insertions(+), 46 deletions(-)
diff --git a/blob.h b/blob.h
index d8db96a..e6a46f3 100644
--- a/blob.h
+++ b/blob.h
@@ -8,6 +8,7 @@ extern const char *blob_type;
struct blob {
struct object object;
unsigned long size;
+ char *name;
};
struct blob *lookup_blob(const unsigned char *sha1);
diff --git a/builtin-rev-cache.c b/builtin-rev-cache.c
index 4a02af6..5f88214 100755
--- a/builtin-rev-cache.c
+++ b/builtin-rev-cache.c
@@ -168,13 +168,14 @@ static int handle_walk(int argc, const char *argv[])
fprintf(stderr, "pending:\n");
for (i = 0; i < revs.pending.nr; i++) {
struct object *obj = revs.pending.objects[i].item;
+ char *name = revs.pending.objects[i].name;
/* unfortunately, despite our careful generation, object duplication *is* a possibility...
* (eg. same object introduced into two different branches) */
if (obj->flags & SEEN)
continue;
- printf("%s\n", sha1_to_hex(revs.pending.objects[i].item->sha1));
+ printf("%s %s\n", sha1_to_hex(revs.pending.objects[i].item->sha1), name);
obj->flags |= SEEN;
}
diff --git a/rev-cache.c b/rev-cache.c
index e6808e7..347e005 100755
--- a/rev-cache.c
+++ b/rev-cache.c
@@ -42,6 +42,8 @@ struct cache_slice_header {
uint32_t size;
unsigned char sha1[20];
+
+ uint32_t name_size;
};
struct object_entry {
@@ -56,6 +58,8 @@ struct object_entry {
unsigned merge_nr : 6;
unsigned split_nr : 7;
unsigned size_size : 3;
+ unsigned name_size : 3;
+ unsigned what_to_do : 5; /* unused */
uint32_t date;
uint16_t path;
@@ -63,6 +67,7 @@ struct object_entry {
/* merge paths */
/* split paths */
/* size */
+ /* name id */
};
struct bad_slice {
@@ -70,16 +75,27 @@ struct bad_slice {
struct bad_slice *next;
};
+struct name_list {
+ unsigned char sha1[20];
+ unsigned int len;
+ struct name_list *next;
+
+ char buf[FLEX_ARRAY];
+};
+
/* list resembles pack index format */
static uint32_t fanout[0xff + 2];
static unsigned char *idx_map;
static int idx_size;
static struct index_header idx_head;
-static char no_idx, save_unique, add_to_pending;
-static struct bad_slice *bad_slices;
+static char no_idx, save_unique, add_to_pending, add_names;
static unsigned char *idx_caches;
+static struct bad_slice *bad_slices;
+static struct name_list *name_lists, *cur_name_list;
+
+static struct strbuf *g_name_buffer;
static struct strbuf *g_buffer;
#define SUPPORTED_REVCACHE_VERSION 1
@@ -94,8 +110,9 @@ static struct strbuf *g_buffer;
#define OE_CAST(p) ((struct object_entry *)(p))
#define IE_CAST(p) ((struct index_entry *)(p))
-#define ACTUAL_OBJECT_ENTRY_SIZE(e) (OE_SIZE + PATH_SIZE((e)->merge_nr + (e)->split_nr) + (e)->size_size)
-#define ENTRY_SIZE_OFFSET(e) (ACTUAL_OBJECT_ENTRY_SIZE(e) - (e)->size_size)
+#define ACTUAL_OBJECT_ENTRY_SIZE(e) (OE_SIZE + PATH_SIZE((e)->merge_nr + (e)->split_nr) + (e)->size_size + (e)->name_size)
+#define ENTRY_NAME_OFFSET(e) (ACTUAL_OBJECT_ENTRY_SIZE(e) - (e)->name_size)
+#define ENTRY_SIZE_OFFSET(e) (ENTRY_NAME_OFFSET(e) - (e)->size_size)
#define SLOP 5
@@ -167,6 +184,12 @@ static void cleanup_cache_slices(void)
idx_map = 0;
}
+ while (name_lists) {
+ struct name_list *nl = name_lists->next;
+ free(name_lists);
+ name_lists = nl;
+ }
+
}
static int init_index(void)
@@ -296,7 +319,8 @@ static void handle_noncommit(struct rev_info *revs, struct commit *commit, struc
struct blob *blob;
struct tree *tree;
struct object *obj;
- unsigned long size;
+ unsigned long size, name_index;
+ char **namep = 0;
size = decode_size((unsigned char *)entry + ENTRY_SIZE_OFFSET(entry), entry->size_size);
switch (entry->type) {
@@ -309,6 +333,7 @@ static void handle_noncommit(struct rev_info *revs, struct commit *commit, struc
return;
tree->size = size;
+ namep = &tree->name;
commit->tree = tree;
obj = (struct object *)tree;
break;
@@ -322,6 +347,7 @@ static void handle_noncommit(struct rev_info *revs, struct commit *commit, struc
return;
blob->size = size;
+ namep = &blob->name;
obj = (struct object *)blob;
break;
@@ -330,6 +356,13 @@ static void handle_noncommit(struct rev_info *revs, struct commit *commit, struc
return;
}
+ if (add_names && cur_name_list) {
+ name_index = decode_size((unsigned char *)entry + ENTRY_NAME_OFFSET(entry), entry->name_size);
+
+ if (name_index >= cur_name_list->len)
+ name_index = 0;
+ } else name_index = 0;
+
/* add to unique list if we're not an end */
if (save_unique && (commit->object.flags & FACE_VALUE)) {
if (last_commit != commit) {
@@ -344,9 +377,20 @@ static void handle_noncommit(struct rev_info *revs, struct commit *commit, struc
last_unique = &(*last_unique)->next;
}
+ /* add cached name */
+ if (name_index && namep) {
+ *namep = cur_name_list->buf + name_index;
+ }
+
obj->flags |= FACE_VALUE;
- if (add_to_pending)
- add_pending_object(revs, obj, "");
+ if (add_to_pending) {
+ char *name = "";
+
+ if (name_index)
+ name = cur_name_list->buf + name_index;
+
+ add_pending_object(revs, obj, name);
+ }
}
static int setup_traversal(struct cache_slice_header *head, unsigned char *map, struct commit *commit, struct commit_list **work,
@@ -696,15 +740,44 @@ end:
return retval;
}
-static int get_cache_slice_header(unsigned char *cache_sha1, unsigned char *map, int len, struct cache_slice_header *head)
+static struct name_list *get_cache_slice_name_list(struct cache_slice_header *head, int fd)
+{
+ struct name_list *nl = name_lists;
+
+ while (nl) {
+ if (!hashcmp(nl->sha1, head->sha1))
+ break;
+ nl = nl->next;
+ }
+
+ if (nl)
+ return nl;
+
+ nl = xcalloc(1, sizeof(struct name_list) + head->name_size);
+ nl->len = head->name_size;
+ hashcpy(nl->sha1, head->sha1);
+
+ lseek(fd, head->size, SEEK_SET);
+ read_in_full(fd, nl->buf, head->name_size);
+
+ nl->next = name_lists;
+ name_lists = nl;
+
+ return nl;
+}
+
+static int get_cache_slice_header(int fd, unsigned char *cache_sha1, int len, struct cache_slice_header *head)
{
int t;
- memcpy(head, map, sizeof(struct cache_slice_header));
+ if (xread(fd, head, sizeof(struct cache_slice_header)) != sizeof(struct cache_slice_header))
+ return -1;
+
head->ofs_objects = ntohl(head->ofs_objects);
head->object_nr = ntohl(head->object_nr);
head->size = ntohl(head->size);
head->path_nr = ntohs(head->path_nr);
+ head->name_size = ntohl(head->name_size);
if (memcmp(head->signature, "REVCACHE", 8))
return -1;
@@ -713,10 +786,10 @@ static int get_cache_slice_header(unsigned char *cache_sha1, unsigned char *map,
if (hashcmp(head->sha1, cache_sha1))
return -3;
t = sizeof(struct cache_slice_header);
- if (t != head->ofs_objects || t >= len)
+ if (t != head->ofs_objects)
return -4;
-
- head->size = len;
+ if (head->size + head->name_size != len)
+ return -5;
return 0;
}
@@ -726,7 +799,7 @@ int traverse_cache_slice(struct rev_info *revs,
unsigned long *date_so_far, int *slop_so_far,
struct commit_list ***queue, struct commit_list **work)
{
- int fd = -1, retval = -3;
+ int fd = -1, t, retval;
struct stat fi;
struct cache_slice_header head;
struct rev_cache_info *rci;
@@ -743,26 +816,31 @@ int traverse_cache_slice(struct rev_info *revs,
rci = &revs->rev_cache_info;
save_unique = rci->save_unique;
add_to_pending = rci->add_to_pending;
+ add_names = rci->add_names;
memset(&head, 0, sizeof(struct cache_slice_header));
+# define ERROR(x) do { retval = (x); goto end; } while (0);
fd = open(git_path("rev-cache/%s", sha1_to_hex(cache_sha1)), O_RDONLY);
if (fd == -1)
- goto end;
+ ERROR(-1);
if (fstat(fd, &fi) || fi.st_size < sizeof(struct cache_slice_header))
- goto end;
+ ERROR(-2);
+
+ if ((t = get_cache_slice_header(fd, cache_sha1, fi.st_size, &head)) < 0)
+ ERROR(-t);
+ if (add_names)
+ cur_name_list = get_cache_slice_name_list(&head, fd);
- map = xmmap(0, fi.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
+ map = xmmap(0, head.size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
if (map == MAP_FAILED)
- goto end;
- if (get_cache_slice_header(cache_sha1, map, fi.st_size, &head))
- goto end;
+ ERROR(-3);
retval = traverse_cache_slice_1(&head, map, revs, commit, date_so_far, slop_so_far, queue, work);
end:
if (map != MAP_FAILED)
- munmap(map, fi.st_size);
+ munmap(map, head.size);
if (fd != -1)
close(fd);
@@ -770,6 +848,7 @@ end:
if (retval)
mark_bad_slice(cache_sha1);
+# undef ERROR
return retval;
}
@@ -1081,12 +1160,100 @@ static unsigned long decode_size(unsigned char *str, int len)
return size;
}
+#define NL_HASH_TABLE_SIZE (0xffff + 1)
+#define NL_HASH_NUMBER (NL_HASH_TABLE_SIZE >> 3)
+
+struct name_list_hash {
+ char *ptr;
+ int ind;
+ struct name_list_hash *next;
+};
+
+static struct name_list_hash **nl_hash_table;
+static unsigned char *nl_hashes;
+
+/* FNV-1a hash */
+static unsigned int hash_name(char *name)
+{
+ unsigned int hash = 2166136261ul;
+ char *p = name;
+
+ while (*p) {
+ hash ^= *p++;
+ hash *= 16777619ul;
+ }
+
+ return hash & 0xffff;
+}
+
+static int name_in_list(char *name)
+{
+ unsigned int h = hash_name(name);
+ struct name_list_hash *entry = nl_hash_table[h];
+
+ while (entry && strcmp(entry->ptr, name))
+ entry = entry->next;
+
+ if (entry)
+ return entry->ind;
+
+ /* add name to buffer and create hash reference */
+ entry = xcalloc(1, sizeof(struct name_list_hash));
+ entry->ind = g_name_buffer->len;
+
+ strbuf_add(g_name_buffer, name, strlen(name) + 1);
+ entry->ptr = g_name_buffer->buf + entry->ind;
+
+ entry->next = nl_hash_table[h];
+ nl_hash_table[h] = entry;
+
+ nl_hashes[h / 8] |= h % 8;
+
+ return entry->ind;
+}
+
+static void init_name_list_hash(void)
+{
+ nl_hash_table = xcalloc(NL_HASH_TABLE_SIZE, sizeof(struct name_list_hash));
+ nl_hashes = xcalloc(NL_HASH_NUMBER, 1);
+}
+
+static void cleanup_name_list_hash(void)
+{
+ int i;
+
+ for (i = 0; i < NL_HASH_NUMBER; i++) {
+ int j, ind = nl_hashes[i];
+
+ if (!ind)
+ continue;
+
+ for (j = 0; j < 8; j++) {
+ struct name_list_hash **entryp;
+
+ if (!(ind & 1 << j))
+ continue;
+
+ entryp = &nl_hash_table[i * 8 + j];
+ while (*entryp) {
+ struct name_list_hash *t = (*entryp)->next;
+
+ free(*entryp);
+ *entryp = t;
+ }
+ }
+ } /* code overhang! */
+
+ free(nl_hashes);
+ free(nl_hash_table);
+}
+
static void add_object_entry(const unsigned char *sha1, struct object_entry *entryp,
- struct strbuf *merge_str, struct strbuf *split_str)
+ struct strbuf *merge_str, struct strbuf *split_str, char *name)
{
struct object_entry entry;
struct object *obj;
- unsigned char size_str[7];
+ unsigned char size_str[7], name_str[7];
unsigned long size;
enum object_type type;
@@ -1100,12 +1267,18 @@ static void add_object_entry(const unsigned char *sha1, struct object_entry *ent
case OBJ_COMMIT :
size = ((struct commit *)obj)->size;
break;
- case OBJ_TREE :
- size = ((struct tree *)obj)->size;
- break;
- case OBJ_BLOB :
- size = ((struct blob *)obj)->size;
- break;
+ case OBJ_TREE : {
+ struct tree *tree = (struct tree *)obj;
+ size = tree->size;
+ if (!name)
+ name = tree->name;
+ } break;
+ case OBJ_BLOB : {
+ struct blob *blob = (struct blob *)obj;
+ size = blob->size;
+ if (!name)
+ name = blob->name;
+ } break;
default :
/* tags are potentially dynamic metadata; they don't really belong here */
return;
@@ -1136,6 +1309,9 @@ static void add_object_entry(const unsigned char *sha1, struct object_entry *ent
entryp->size_size = encode_size(size, size_str);
+ if (name)
+ entryp->name_size = encode_size(name_in_list(name), name_str);
+
/* write the muvabitch */
strbuf_add(g_buffer, entryp, sizeof(entry));
@@ -1145,6 +1321,9 @@ static void add_object_entry(const unsigned char *sha1, struct object_entry *ent
strbuf_add(g_buffer, split_str->buf, split_str->len);
strbuf_add(g_buffer, size_str, entryp->size_size);
+
+ if (name)
+ strbuf_add(g_buffer, name_str, entryp->name_size);
}
/* returns non-zero to continue parsing, 0 to skip */
@@ -1189,6 +1368,9 @@ continue_loop:
static int dump_tree_callback(const unsigned char *sha1, const char *path, unsigned int mode)
{
strbuf_add(g_buffer, sha1, 20);
+ strbuf_add(g_buffer, (char *)&g_name_buffer->len, sizeof(size_t));
+
+ strbuf_add(g_name_buffer, path, strlen(path) + 1);
return 1;
}
@@ -1202,6 +1384,9 @@ static void tree_addremove(struct diff_options *options,
return;
strbuf_add(g_buffer, sha1, 20);
+ strbuf_add(g_buffer, (char *)&g_name_buffer->len, sizeof(size_t));
+
+ strbuf_add(g_name_buffer, concatpath, strlen(concatpath) + 1);
}
static void tree_change(struct diff_options *options,
@@ -1214,12 +1399,15 @@ static void tree_change(struct diff_options *options,
return;
strbuf_add(g_buffer, new_sha1, 20);
+ strbuf_add(g_buffer, (char *)&g_name_buffer->len, sizeof(size_t));
+
+ strbuf_add(g_name_buffer, concatpath, strlen(concatpath) + 1);
}
static int add_unique_objects(struct commit *commit)
{
struct commit_list *list;
- struct strbuf os, ost, *orig_buf;
+ struct strbuf os, ost, names, *orig_name_buf, *orig_buf;
struct diff_options opts;
int i, j, next;
char is_first = 1;
@@ -1231,7 +1419,7 @@ static int add_unique_objects(struct commit *commit)
olist = commit->unique;
i = 0;
while (olist) {
- add_object_entry(olist->item->sha1, 0, 0, 0);
+ add_object_entry(olist->item->sha1, 0, 0, 0, 0 /* retrieved with size in function */);
i++;
olist = olist->next;
}
@@ -1242,13 +1430,17 @@ static int add_unique_objects(struct commit *commit)
/* ...no, calculate unique objects */
strbuf_init(&os, 0);
strbuf_init(&ost, 0);
+ strbuf_init(&names, 0);
orig_buf = g_buffer;
+ orig_name_buf = g_name_buffer;
+ g_name_buffer = &names;
diff_setup(&opts);
DIFF_OPT_SET(&opts, RECURSIVE);
DIFF_OPT_SET(&opts, TREE_IN_RECURSIVE);
opts.change = tree_change;
opts.add_remove = tree_addremove;
+# define ENTRY_SIZE (20 + sizeof(size_t))
/* this is only called for non-ends (ie. all parents interesting) */
for (list = commit->parents; list; list = list->next) {
@@ -1259,20 +1451,20 @@ static int add_unique_objects(struct commit *commit)
strbuf_setlen(g_buffer, 0);
diff_tree_sha1(list->item->tree->object.sha1, commit->tree->object.sha1, "", &opts);
- qsort(g_buffer->buf, g_buffer->len / 20, 20, (int (*)(const void *, const void *))hashcmp);
+ qsort(g_buffer->buf, g_buffer->len / ENTRY_SIZE, ENTRY_SIZE, (int (*)(const void *, const void *))hashcmp);
/* take intersection */
if (!is_first) {
- for (next = i = j = 0; i < os.len; i += 20) {
+ for (next = i = j = 0; i < os.len; i += ENTRY_SIZE) {
while (j < ost.len && hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)) < 0)
- j += 20;
+ j += ENTRY_SIZE;
if (j >= ost.len || hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)))
continue;
if (next != i)
- memcpy(os.buf + next, os.buf + i, 20);
- next += 20;
+ memcpy(os.buf + next, os.buf + i, ENTRY_SIZE);
+ next += ENTRY_SIZE;
}
if (next != i)
@@ -1289,16 +1481,19 @@ static int add_unique_objects(struct commit *commit)
/* the ordering of non-commit objects dosn't really matter, so we're not gonna bother */
g_buffer = orig_buf;
- for (i = 0; i < os.len; i += 20)
- add_object_entry((unsigned char *)(os.buf + i), 0, 0, 0);
+ g_name_buffer = orig_name_buf;
+ for (i = 0; i < os.len; i += ENTRY_SIZE)
+ add_object_entry((unsigned char *)(os.buf + i), 0, 0, 0, names.buf + *(size_t *)(os.buf + i + 20));
/* last but not least, the main tree */
- add_object_entry(commit->tree->object.sha1, 0, 0, 0);
+ add_object_entry(commit->tree->object.sha1, 0, 0, 0, 0);
strbuf_release(&ost);
strbuf_release(&os);
+ strbuf_release(&names);
- return i / 20 + 1;
+ return i / ENTRY_SIZE + 1;
+# undef ENTRY_SIZE
}
static void init_revcache_directory(void)
@@ -1319,6 +1514,7 @@ void init_rci(struct rev_cache_info *rci)
rci->save_unique = 0;
rci->add_to_pending = 1;
+ rci->add_names = 1;
rci->ignore_size = 0;
}
@@ -1333,9 +1529,9 @@ int make_cache_slice(struct rev_cache_info *rci,
struct cache_slice_header head;
struct commit *commit;
unsigned char sha1[20];
- struct strbuf merge_paths, split_paths;
+ struct strbuf merge_paths, split_paths, namelist;
int object_nr, total_sz, fd;
- char file[PATH_MAX], *newfile;
+ char file[PATH_MAX], *newfile, null;
struct rev_cache_info *trci, def_rci;
git_SHA_CTX ctx;
@@ -1353,7 +1549,13 @@ int make_cache_slice(struct rev_cache_info *rci,
strbuf_init(&endlist, 0);
strbuf_init(&merge_paths, 0);
strbuf_init(&split_paths, 0);
+ strbuf_init(&namelist, 0);
g_buffer = &buffer;
+ g_name_buffer = &namelist;
+
+ null = 0;
+ strbuf_add(&namelist, &null, 1);
+ init_name_list_hash();
if (!revs) {
revs = &therevs;
@@ -1384,6 +1586,7 @@ int make_cache_slice(struct rev_cache_info *rci,
trci = &revs->rev_cache_info;
init_rci(trci);
trci->save_unique = 1;
+ trci->add_names = 1;
trci->add_to_pending = 0;
setup_revisions(0, 0, revs, 0);
@@ -1414,7 +1617,7 @@ int make_cache_slice(struct rev_cache_info *rci,
commit->indegree = 0;
- add_object_entry(0, &object, &merge_paths, &split_paths);
+ add_object_entry(0, &object, &merge_paths, &split_paths, 0);
object_nr++;
if (rci->objects && !(commit->object.flags & TREESAME) && !object.is_end)
@@ -1434,10 +1637,16 @@ int make_cache_slice(struct rev_cache_info *rci,
total_sz += buffer.len;
}
+ /* write path name lookup list */
+ head.name_size = htonl(namelist.len);
+ write_in_full(fd, namelist.buf, namelist.len);
+
/* go ahead a free some stuff... */
strbuf_release(&buffer);
strbuf_release(&merge_paths);
strbuf_release(&split_paths);
+ strbuf_release(&namelist);
+ cleanup_name_list_hash();
if (path_sz)
free(paths);
while (path_track_alloc)
diff --git a/revision.h b/revision.h
index d97ac8c..5114208 100644
--- a/revision.h
+++ b/revision.h
@@ -27,7 +27,8 @@ struct rev_cache_info {
/* traversal flags */
unsigned save_unique : 1,
- add_to_pending : 1;
+ add_to_pending : 1,
+ add_names : 1;
/* fuse options */
unsigned int ignore_size;
diff --git a/t/t6015-rev-cache-list.sh b/t/t6015-rev-cache-list.sh
index 561810f..3836596 100755
--- a/t/t6015-rev-cache-list.sh
+++ b/t/t6015-rev-cache-list.sh
@@ -4,8 +4,8 @@ test_description='git rev-cache tests'
. ./test-lib.sh
test_cmp_sorted() {
- grep -io "[a-f0-9]*" $1 | sort >.tmpfile1 &&
- grep -io "[a-f0-9]*" $2 | sort >.tmpfile2 &&
+ sort $1 >.tmpfile1 &&
+ sort $2 >.tmpfile2 &&
test_cmp .tmpfile1 .tmpfile2
}
diff --git a/tree.h b/tree.h
index 2ff01a4..6eb0cd0 100644
--- a/tree.h
+++ b/tree.h
@@ -9,6 +9,7 @@ struct tree {
struct object object;
void *buffer;
unsigned long size;
+ char *name;
};
struct tree *lookup_tree(const unsigned char *sha1);
--
tg: (c47593c..) t/revcache/names (depends on: t/revcache/docs)
^ permalink raw reply related
* Re: [PATCH 0/5] Suggested for PU: revision caching system to significantly speed up packing/walking
From: Nick Edelen @ 2009-08-08 7:40 UTC (permalink / raw)
To: Jeff King
Cc: Junio C Hamano, Nicolas Pitre, Johannes Schindelin, Sam Vilain,
Michael J Gruber, Shawn O. Pearce, Andreas Ericsson,
Christian Couder, git@vger.kernel.org
In-Reply-To: <20090808073042.GA20951@coredump.intra.peff.net>
> That might be useful, but I was thinking more of an "alternates"-like
> mechanism between repos. So that the data is stored only once on disk
> and in the disk cache, which is helpful for sites like kernel.org which
> serve many similar repositories.
Oh, right. Yes, that seems like it could work. We'd have to be
careful that a shared cache slice wouldn' change (like in a fuse or
something), but other than that we could have something as simple as a
link.
^ permalink raw reply
* Re: [PATCH 2/2 (v2)] reset: make the output more user-friendly.
From: Matthieu Moy @ 2009-08-08 7:44 UTC (permalink / raw)
To: Junio C Hamano; +Cc: git
In-Reply-To: <7viqgztj76.fsf@alter.siamese.dyndns.org>
Junio C Hamano <gitster@pobox.com> writes:
> Matthieu Moy <Matthieu.Moy@imag.fr> writes:
>
>> cat > expect << EOF
>> -file2: locally modified
>> +Unstaged changes after reset:
>> +M file2
>
> It simply feels backwards when plumbing output says something in human
> language (e.g. "needs update") while Porcelain output spits out a cryptic
> M or U. If the goal is human-readability and user-friendliness,
The goal here is just consistency.
And I do consider 'git diff --name-status' as porcelain.
> shouldn't we rather say:
>
> Path with local modifications:
> file2
>
> or something?
Why not, but if we do, we should also remove this "M" from other
places. It was already there in one error message given by 'git
rebase' in a non-clean tree (and you just accepted a patch giving the
same output for another one).
--
Matthieu
^ permalink raw reply
* [PATCH v2] push: point to 'git pull' and 'git push --force' in case of non-fast forward
From: Matthieu Moy @ 2009-08-08 7:51 UTC (permalink / raw)
To: git, gitster; +Cc: Matthieu Moy
In-Reply-To: <1249579933-1782-1-git-send-email-Matthieu.Moy@imag.fr>
'git push' failing because of non-fast forward is a very common situation,
and a beginner does not necessarily understand "fast forward" immediately.
Signed-off-by: Matthieu Moy <Matthieu.Moy@imag.fr>
---
(hmm, I thought I had sent this v2 already, but seems I didn't)
With grammar fixes from Michael J Gruber.
Junio, this comes on top of your patch, since it refers to the
to-be-added section in 'git push --help'.
builtin-push.c | 10 +++++++++-
transport.c | 10 +++++++---
transport.h | 3 ++-
3 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/builtin-push.c b/builtin-push.c
index 1d92e22..5d4df7f 100644
--- a/builtin-push.c
+++ b/builtin-push.c
@@ -140,6 +140,7 @@ static int do_push(const char *repo, int flags)
struct transport *transport =
transport_get(remote, url[i]);
int err;
+ int nonfastforward;
if (receivepack)
transport_set_option(transport,
TRANS_OPT_RECEIVEPACK, receivepack);
@@ -148,13 +149,20 @@ static int do_push(const char *repo, int flags)
if (flags & TRANSPORT_PUSH_VERBOSE)
fprintf(stderr, "Pushing to %s\n", url[i]);
- err = transport_push(transport, refspec_nr, refspec, flags);
+ err = transport_push(transport, refspec_nr, refspec, flags,
+ &nonfastforward);
err |= transport_disconnect(transport);
if (!err)
continue;
error("failed to push some refs to '%s'", url[i]);
+ if (nonfastforward) {
+ printf("Push was rejected because it would not result in a fast forward:\n"
+ "Merge in the remote changes (using git pull) before pushing yours,\n"
+ "or use git push --force to discard the remote changes.\n"
+ "See 'non-fast forward' section of 'git push --help' for details.\n");
+ }
errs++;
}
return !!errs;
diff --git a/transport.c b/transport.c
index de0d587..f231b35 100644
--- a/transport.c
+++ b/transport.c
@@ -820,7 +820,7 @@ static int print_one_push_status(struct ref *ref, const char *dest, int count, i
}
static void print_push_status(const char *dest, struct ref *refs,
- int verbose, int porcelain)
+ int verbose, int porcelain, int * nonfastforward)
{
struct ref *ref;
int n = 0;
@@ -835,11 +835,14 @@ static void print_push_status(const char *dest, struct ref *refs,
if (ref->status == REF_STATUS_OK)
n += print_one_push_status(ref, dest, n, porcelain);
+ *nonfastforward = 0;
for (ref = refs; ref; ref = ref->next) {
if (ref->status != REF_STATUS_NONE &&
ref->status != REF_STATUS_UPTODATE &&
ref->status != REF_STATUS_OK)
n += print_one_push_status(ref, dest, n, porcelain);
+ if (ref->status == REF_STATUS_REJECT_NONFASTFORWARD)
+ *nonfastforward = 1;
}
}
@@ -997,7 +1000,8 @@ int transport_set_option(struct transport *transport,
}
int transport_push(struct transport *transport,
- int refspec_nr, const char **refspec, int flags)
+ int refspec_nr, const char **refspec, int flags,
+ int * nonfastforward)
{
verify_remote_names(refspec_nr, refspec);
@@ -1024,7 +1028,7 @@ int transport_push(struct transport *transport,
ret = transport->push_refs(transport, remote_refs, flags);
- print_push_status(transport->url, remote_refs, verbose | porcelain, porcelain);
+ print_push_status(transport->url, remote_refs, verbose | porcelain, porcelain, nonfastforward);
if (!(flags & TRANSPORT_PUSH_DRY_RUN)) {
struct ref *ref;
diff --git a/transport.h b/transport.h
index 51b5397..639f13d 100644
--- a/transport.h
+++ b/transport.h
@@ -68,7 +68,8 @@ int transport_set_option(struct transport *transport, const char *name,
const char *value);
int transport_push(struct transport *connection,
- int refspec_nr, const char **refspec, int flags);
+ int refspec_nr, const char **refspec, int flags,
+ int * nonfastforward);
const struct ref *transport_get_remote_refs(struct transport *transport);
--
1.6.4.62.g39c83.dirty
^ permalink raw reply related
* Re: [PATCH v2] push: point to 'git pull' and 'git push --force' in case of non-fast forward
From: Teemu Likonen @ 2009-08-08 8:35 UTC (permalink / raw)
To: Matthieu Moy; +Cc: git, gitster
In-Reply-To: <1249717868-10946-1-git-send-email-Matthieu.Moy@imag.fr>
On 2009-08-08 09:51 (+0200), Matthieu Moy wrote:
> 'git push' failing because of non-fast forward is a very common situation,
> and a beginner does not necessarily understand "fast forward" immediately.
> + if (nonfastforward) {
> + printf("Push was rejected because it would not result in a fast forward:\n"
> + "Merge in the remote changes (using git pull) before pushing yours,\n"
> + "or use git push --force to discard the remote changes.\n"
> + "See 'non-fast forward' section of 'git push --help' for details.\n");
> + }
I'd like to add that some projects that use Git in (partially)
centralized manner prefer "git pull --rebase" before "git push".
^ permalink raw reply
* [PATCH 1/9] add long options completion for 'git checkout'
From: Emmanuel Trillaud @ 2009-08-08 8:55 UTC (permalink / raw)
To: Shawn O. Pearce; +Cc: git
add long options completion to git checkout
Signed-off-by: Emmanuel Trillaud <etrillaud@gmail.com>
---
contrib/completion/git-completion.bash | 14 ++++++++++++--
1 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/contrib/completion/git-completion.bash
b/contrib/completion/git-completion.bash
index 745b5fb..bc5427f 100755
--- a/contrib/completion/git-completion.bash
+++ b/contrib/completion/git-completion.bash
@@ -815,8 +815,18 @@ _git_bundle ()
_git_checkout ()
{
__git_has_doubledash && return
-
- __gitcomp "$(__git_refs)"
+ case "${COMP_WORDS[COMP_CWORD]}" in
+ --*)
+ __gitcomp "
+ --ours --theirs --track --no-track --merge
+ --conflict=
+ "
+ ;;
+ *)
+ __gitcomp "$(__git_refs)"
+ ;;
+ esac
+
}
_git_cherry ()
--
1.6.4
^ permalink raw reply related
* [PATCH 2/9] add missing long options to 'git difftool' completion
From: Emmanuel Trillaud @ 2009-08-08 8:55 UTC (permalink / raw)
To: Shawn O. Pearce; +Cc: git
according to its man page, git-difftool has 3 long options :
--tools=, --no-prompt and --prompt. The last 2 options are missing.
Signed-off-by: Emmanuel Trillaud <etrillaud@gmail.com>
---
contrib/completion/git-completion.bash | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/contrib/completion/git-completion.bash
b/contrib/completion/git-completion.bash
index bc5427f..9e86d28 100755
--- a/contrib/completion/git-completion.bash
+++ b/contrib/completion/git-completion.bash
@@ -960,7 +960,7 @@ _git_difftool ()
return
;;
--*)
- __gitcomp "--tool="
+ __gitcomp "--tool= --no-prompt --prompt"
return
;;
esac
--
1.6.4
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox