[WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

[WIP: this does not handle tags yet, and it lacks a test script
 as well as documentation.]

This program dumps (parts of) a git repository in the format that
fast-import understands.

For clarity's sake, it does not use the 'inline' method of specifying
blobs in the commits, but builds the blobs before building the commits.B

---
	I am way too tired now to continue, but maybe someone else wants
	to pick up the ball.

	Oh, and it relies on "int" being castable to void * and vice 
	versa.  Is anybody aware of a platform where this can lead to
	problems?

	And yes, I will add a copyright when I woke up again.

 .gitignore            |    1 +
 Makefile              |    1 +
 builtin-fast-export.c |  202 +++++++++++++++++++++++++++++++++++++++++++++++++
 builtin.h             |    1 +
 git.c                 |    1 +
 5 files changed, 206 insertions(+), 0 deletions(-)
 create mode 100755 builtin-fast-export.c

diff --git a/.gitignore b/.gitignore
index 507e351..ef69e2f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ git-diff-files
 git-diff-index
 git-diff-tree
 git-describe
+git-fast-export
 git-fast-import
 git-fetch
 git-fetch--tool
diff --git a/Makefile b/Makefile
index dd3ba6f..8784eff 100644
--- a/Makefile
+++ b/Makefile
@@ -336,6 +336,7 @@ BUILTIN_OBJS = \
 	builtin-diff-files.o \
 	builtin-diff-index.o \
 	builtin-diff-tree.o \
+	builtin-fast-export.o \
 	builtin-fetch.o \
 	builtin-fetch-pack.o \
 	builtin-fetch--tool.o \
diff --git a/builtin-fast-export.c b/builtin-fast-export.c
new file mode 100755
index 0000000..2e8e3dd
--- /dev/null
+++ b/builtin-fast-export.c
@@ -0,0 +1,202 @@
+#include "builtin.h"
+#include "cache.h"
+#include "commit.h"
+#include "object.h"
+#include "tag.h"
+#include "diff.h"
+#include "diffcore.h"
+#include "log-tree.h"
+#include "revision.h"
+#include "decorate.h"
+#include "path-list.h"
+
+static struct decoration idnums;
+static uint32_t last_idnum;
+
+static const char *fast_export_usage = "git-fast-export [rev-list-opts]";
+
+static int has_unshown_parent(struct commit *commit)
+{
+	struct commit_list *parent;
+
+	for (parent = commit->parents; parent; parent = parent->next)
+		if (!(parent->item->object.flags & SHOWN))
+			return 1;
+	return 0;
+}
+
+static void handle_object(const unsigned char *sha1)
+{
+	unsigned long size;
+	enum object_type type;
+	char *buf;
+	struct object *object;
+
+	if (is_null_sha1(sha1))
+		return;
+
+	object = parse_object(sha1);
+	if (!object)
+		die ("Could not read blob %s", sha1_to_hex(sha1));
+
+	if (object->flags & SHOWN)
+		return;
+
+	buf = read_sha1_file(sha1, &type, &size);
+	if (!buf)
+		die ("Could not read blob %s", sha1_to_hex(sha1));
+
+	last_idnum++;
+	add_decoration(&idnums, object, (void *)last_idnum);
+
+	printf("blob\nmark :%d\ndata %lu\n", last_idnum, size);
+	if (fwrite(buf, size, 1, stdout) != 1)
+		die ("Could not write blob %s", sha1_to_hex(sha1));
+	printf("\n");
+
+	object->flags |= SHOWN;
+	free(buf);
+}
+
+static void show_filemodify(struct diff_queue_struct *q,
+		struct diff_options *options, void *data)
+{
+	int i;
+	for (i = 0; i < q->nr; i++) {
+		struct diff_filespec *spec = q->queue[i]->two;
+		if (is_null_sha1(spec->sha1))
+			printf("D %s\n", spec->path);
+		else {
+			struct object *object = lookup_object(spec->sha1);
+			printf("M 0%06o :%d %s\n", spec->mode,
+					(int)lookup_decoration(&idnums, object),
+					spec->path);
+		}
+	}
+}
+
+static void handle_commit(struct commit *commit, struct rev_info *rev)
+{
+	const char *author, *author_end, *committer, *committer_end, *message;
+	int saved_output_format = rev->diffopt.output_format;
+	int i;
+
+	rev->diffopt.output_format = DIFF_FORMAT_CALLBACK;
+
+	parse_commit(commit);
+	author = strstr(commit->buffer, "\nauthor ");
+	if (!author)
+		die ("Could not find author in commit %s",
+				sha1_to_hex(commit->object.sha1));
+	author++;
+	author_end = strchrnul(author, '\n');
+	committer = strstr(author_end, "\ncommitter ");
+	if (!committer)
+		die ("Could not find committer in commit %s",
+				sha1_to_hex(commit->object.sha1));
+	committer++;
+	committer_end = strchrnul(committer, '\n');
+	message = strstr(committer_end, "\n\n");
+	if (message)
+		message += 2;
+
+	if (commit->parents) {
+		parse_commit(commit->parents->item);
+		diff_tree_sha1(commit->parents->item->tree->object.sha1,
+				commit->tree->object.sha1, "", &rev->diffopt);
+	}
+	else
+		diff_root_tree_sha1(commit->tree->object.sha1,
+				"", &rev->diffopt);
+
+	for (i = 0; i < diff_queued_diff.nr; i++)
+		handle_object(diff_queued_diff.queue[i]->two->sha1);
+
+	last_idnum++;
+	add_decoration(&idnums, &commit->object, (void *)last_idnum);
+	printf("commit %s\nmark :%d\n%.*s\n%.*s\ndata %u\n%s",
+		(const char *)commit->util, last_idnum,
+		author_end - author, author,
+		committer_end - committer, committer,
+		message ? strlen(message) : 0, message ? message : "");
+
+	log_tree_diff_flush(rev);
+	rev->diffopt.output_format = saved_output_format;
+
+	printf("\n");
+}
+
+static void handle_tail(struct object_array *commits, struct rev_info *revs)
+{
+	struct commit *commit;
+	while (commits->nr) {
+		commit = (struct commit *)commits->objects[commits->nr - 1].item;
+		if (has_unshown_parent(commit))
+			return;
+		handle_commit(commit, revs);
+		commits->nr--;
+	}
+}
+
+int cmd_fast_export(int argc, const char **argv, const char *prefix)
+{
+	struct rev_info revs;
+	struct object_array commits = { 0, 0, NULL };
+	struct path_list extra_refs = { NULL, 0, 0, 0 };
+	struct commit *commit;
+	int i;
+
+	init_revisions(&revs, prefix);
+	argc = setup_revisions(argc, argv, &revs, NULL);
+	if (argc > 1)
+		usage (fast_export_usage);
+
+	for (i = 0; i < revs.pending.nr; i++) {
+		struct object_array_entry *e = revs.pending.objects + i;
+		unsigned char sha1[20];
+		char *full_name;
+
+		/* TODO: handle tags */
+		commit = (struct commit *)e->item;
+		if (dwim_ref(e->name, strlen(e->name), sha1, &full_name) != 1)
+			die ("Could not get a unique ref name for '%s'",
+				e->name);
+		if (commit->util)
+			/* more than one name for the same object */
+			path_list_insert(full_name, &extra_refs)->util = commit;
+		else
+			commit->util = full_name;
+	}
+
+	prepare_revision_walk(&revs);
+	revs.diffopt.format_callback = show_filemodify;
+	revs.diffopt.recursive = 1;
+	while ((commit = get_revision(&revs))) {
+		if (has_unshown_parent(commit)) {
+			struct commit_list *parent = commit->parents;
+			add_object_array(&commit->object, NULL, &commits);
+			for (; parent; parent = parent->next)
+				if (!parent->item->util)
+					parent->item->util = commit->util;
+		}
+		else {
+			handle_commit(commit, &revs);
+			handle_tail(&commits, &revs);
+		}
+	}
+
+	if (commits.nr)
+		die ("This should not happen: there were %d commits left!",
+			commits.nr);
+
+	for (i = 0; i < extra_refs.nr; i++) {
+		/* create refs pointing to already seen commits */
+		commit = extra_refs.items[i].util;
+		printf("reset %s\nfrom :%d\n\n",
+				extra_refs.items[i].path,
+				(int)lookup_decoration(&idnums,
+					&commit->object));
+	}
+
+	return 0;
+}
diff --git a/builtin.h b/builtin.h
index e76f5da..5723f9e 100644
--- a/builtin.h
+++ b/builtin.h
@@ -31,6 +31,7 @@ extern int cmd_diff_files(int argc, const char **argv, const char *prefix);
 extern int cmd_diff_index(int argc, const char **argv, const char *prefix);
 extern int cmd_diff(int argc, const char **argv, const char *prefix);
 extern int cmd_diff_tree(int argc, const char **argv, const char *prefix);
+extern int cmd_fast_export(int argc, const char **argv, const char *prefix);
 extern int cmd_fetch(int argc, const char **argv, const char *prefix);
 extern int cmd_fetch_pack(int argc, const char **argv, const char *prefix);
 extern int cmd_fetch__tool(int argc, const char **argv, const char *prefix);
diff --git a/git.c b/git.c
index 0b6825e..deda6dd 100644
--- a/git.c
+++ b/git.c
@@ -301,6 +301,7 @@ static void handle_internal_command(int argc, const char **argv)
 		{ "diff-files", cmd_diff_files },
 		{ "diff-index", cmd_diff_index, RUN_SETUP },
 		{ "diff-tree", cmd_diff_tree, RUN_SETUP },
+		{ "fast-export", cmd_fast_export, RUN_SETUP },
 		{ "fetch", cmd_fetch, RUN_SETUP },
 		{ "fetch-pack", cmd_fetch_pack, RUN_SETUP },
 		{ "fetch--tool", cmd_fetch__tool, RUN_SETUP },
-- 
1.5.3.5.1751.gf6fb4

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Sixt]

Johannes Schindelin schrieb:
> 	Oh, and it relies on "int" being castable to void * and vice 
> 	versa.  Is anybody aware of a platform where this can lead to
> 	problems?

Win64?

-- Hannes

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Shawn O. Pearce]

Johannes Sixt <j.sixt@viscovery.net> wrote:
> Johannes Schindelin schrieb:
> >	Oh, and it relies on "int" being castable to void * and vice 
> >	versa.  Is anybody aware of a platform where this can lead to
> >	problems?
> 
> Win64?

Does anyone even use that?  I thought that was dead.  Like Vista.

Seriously though, we include stdint.h.  If you want to cast a void*
to an integer type and back use ptrint_t.  That's what it exists for.

-- 
Shawn.

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Geert Bosch]

On Nov 20, 2007, at 22:40, Johannes Schindelin wrote:
> 	Oh, and it relies on "int" being castable to void * and vice
> 	versa.  Is anybody aware of a platform where this can lead to
> 	problems?

Like any 64-bit system? int is 32 bit and void * is 64.
So, casting a pointer to an int drops half the bits.
When you convert it back it will point to a funny place...

   -Geert

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

On Wed, 21 Nov 2007, Johannes Sixt wrote:

> Johannes Schindelin schrieb:
> > 	Oh, and it relies on "int" being castable to void * and vice versa.
> >     Is anybody aware of a platform where this can lead to problems?
> 
> Win64?

Is this really a problem?  I mean, all I need is that i == (int)(void *)i.

My doubts came from platforms where you can access memory only aligned to 
4-byte chunks, and that it _may_ be possible that some of them do not even 
store the least significant two bits.

Ciao,
Dscho

P.S.: I'll try to read up on ptrint_t, as suggested by Shawn.

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

On Wed, 21 Nov 2007, Geert Bosch wrote:

> On Nov 20, 2007, at 22:40, Johannes Schindelin wrote:
> > 	Oh, and it relies on "int" being castable to void * and vice
> > 	versa.  Is anybody aware of a platform where this can lead to
> > 	problems?
> 
> Like any 64-bit system? int is 32 bit and void * is 64. So, casting a 
> pointer to an int drops half the bits. When you convert it back it will 
> point to a funny place...

Have you even read the patch?  Like I pointed out in my reply to Hannes, I 
only need the guarantee that

	i == (int)(void *)i

for all integers i.

Ciao,
Dscho

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Andreas Ericsson]

Johannes Schindelin wrote:
> 
> P.S.: I'll try to read up on ptrint_t, as suggested by Shawn.

I can't find a reference to ptrint_t anywhere on either my system,
and the 10 first hits on Google finds it on typedef lines, most
commonly like this:

	typedef unsigned long ptrint_t;


I think ptrdiff_t is more portable. Here's the relevant entry
about ptrdiff_t from /usr/include/obstack.h on my system:

---%<---%<---%<---
/* We need the type of a pointer subtraction.  If __PTRDIFF_TYPE__ is
   defined, as with GNU C, use that; that way we don't pollute the
   namespace with <stddef.h>'s symbols.  Otherwise, include <stddef.h>
   and use ptrdiff_t.  */

#ifdef __PTRDIFF_TYPE__
# define PTR_INT_TYPE __PTRDIFF_TYPE__
#else
# include <stddef.h>
# define PTR_INT_TYPE ptrdiff_t
#endif
---%<---%<---%<---

MySQL seems to have botched that one though. It has this instead:

	typedef long          my_ptrdiff_t;

-- 
Andreas Ericsson                   andreas.ericsson@op5.se
OP5 AB                             www.op5.se
Tel: +46 8-230225                  Fax: +46 8-230231

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

On Wed, 21 Nov 2007, Andreas Ericsson wrote:

> Johannes Schindelin wrote:
> > 
> > P.S.: I'll try to read up on ptrint_t, as suggested by Shawn.
> 
> I can't find a reference to ptrint_t anywhere on either my system,
> and the 10 first hits on Google finds it on typedef lines, most
> commonly like this:
> 
> 	typedef unsigned long ptrint_t;

It is really called intptr_t, and it is C99.

Ciao,
Dscho

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Andreas Ericsson]

Johannes Schindelin wrote:
> Hi,
> 
> On Wed, 21 Nov 2007, Andreas Ericsson wrote:
> 
>> Johannes Schindelin wrote:
>>> P.S.: I'll try to read up on ptrint_t, as suggested by Shawn.
>> I can't find a reference to ptrint_t anywhere on either my system,
>> and the 10 first hits on Google finds it on typedef lines, most
>> commonly like this:
>>
>> 	typedef unsigned long ptrint_t;
> 
> It is really called intptr_t, and it is C99.
> 

That'd explain it I guess. Thanks for the clarification.

-- 
Andreas Ericsson                   andreas.ericsson@op5.se
OP5 AB                             www.op5.se
Tel: +46 8-230225                  Fax: +46 8-230231

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Han-Wen Nienhuys]

Johannes Schindelin escreveu:
> [WIP: this does not handle tags yet, and it lacks a test script
>  as well as documentation.]
> 
> This program dumps (parts of) a git repository in the format that
> fast-import understands.
> 
> For clarity's sake, it does not use the 'inline' method of specifying
> blobs in the commits, but builds the blobs before building the commits.B
> 
> ---
> 	I am way too tired now to continue, but maybe someone else wants
> 	to pick up the ball.
> 
> 	Oh, and it relies on "int" being castable to void * and vice 
> 	versa.  Is anybody aware of a platform where this can lead to
> 	problems?
> 
> 	And yes, I will add a copyright when I woke up again.

This one seems to setup a dump of a single branch from the command
line, which then follows the commit structure.  Am I missing
something?

The cool thing about git-fast-import is that it reads from stdin, has
a very easy to use programmatic interface, and does not impose any
order on how you enter the information.

This doesn't seem to be mirrored by this script? 

I am working on a script for [company] which uses git.  Git is a pain
to script for: for every query I need to invoke another git process,
with another command (log, show-ref, cat-file, show, etc.), parse
another output format and/or specify another --pretty=format:%blah
format.

Besides being a nuisance, I actually run git on NFS, and every git
process has to go to NFS a couple times to retrieve the same
information. This has a noticeable performance impact.

It would make my life a lot easier if I could simply open a pipe to a
single process for the duration of the script, and do all my queries
to this one process.  Of course, if the repository is changed by
another process, I would have to restart it, but that's manageable.  I
could even write a nice Python class that runs both fast-import and
fast-export. I could then have an efficient Python interface to a
git-repository, without needing any library wrapping.

--
 Han-Wen Nienhuys - hanwen@xs4all.nl - http://www.xs4all.nl/~hanwen

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

[please Cc me when you reply to my message]

On Thu, 22 Nov 2007, Han-Wen Nienhuys wrote:

> This one seems to setup a dump of a single branch from the command line, 
> which then follows the commit structure.  Am I missing something?

Yes.  It should work with "--all", too.  In fact, with every rev-list 
parameters, even a..b (which would cut off the history up to -- and 
including -- a).

> The cool thing about git-fast-import is that it reads from stdin, has a 
> very easy to use programmatic interface, and does not impose any order 
> on how you enter the information.
> 
> This doesn't seem to be mirrored by this script? 

Umm.  What exactly do you want to reorder?  I mean, this program is meant 
to dump the complete contents to stdout (whether you redirect it to a file 
or edit it with sed does not concern this program).  It does that.

Maybe you want to specify if all blobs should be output first, and then 
the commits?  Or files should be used?  But all of these things seem to be 
useless to me.

So I am puzzled what you ask for.

> I am working on a script for [company] which uses git.  Git is a pain to 
> script for: for every query I need to invoke another git process, with 
> another command (log, show-ref, cat-file, show, etc.), parse another 
> output format and/or specify another --pretty=format:%blah format.

Now I am really puzzled.  Git is one of the most easily scriptable 
programs I ever saw.

It does not even force you to use certain combinations of scripting 
languages, such as Python, Scheme, C++ and PostScript, separated by which 
part of the program you want to script.

> Besides being a nuisance, I actually run git on NFS, and every git 
> process has to go to NFS a couple times to retrieve the same 
> information. This has a noticeable performance impact.

Why don't you just work on a local clone?  If it is really performance 
critical, and I/O is an issue, you are better off working in a tmpfs.

Once you're done, you push back to the NFS repository (which is 
lock-challenged AFAIR, but I guess you know that).

> It would make my life a lot easier if I could simply open a pipe to a 
> single process for the duration of the script, and do all my queries to 
> this one process.  Of course, if the repository is changed by another 
> process, I would have to restart it, but that's manageable.  I could 
> even write a nice Python class that runs both fast-import and 
> fast-export. I could then have an efficient Python interface to a 
> git-repository, without needing any library wrapping.

There is a minimal python wrapper to libgit-thin, which was one of our 
GSoC projects.  You might want to look at this if you are really that 
unhappy with the existing framework.

As to the niceness of Python classes; this lies definitely in the eyes of 
the beholder.  For example, I have given up on understanding any part of 
your GUB framework.

Ciao,
Dscho

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Han-Wen Nienhuys]

2007/11/22, Johannes Schindelin <Johannes.Schindelin@gmx.de>:
> > The cool thing about git-fast-import is that it reads from stdin, has a
> > very easy to use programmatic interface, and does not impose any order
> > on how you enter the information.
> >
> > This doesn't seem to be mirrored by this script?
>
> Umm.  What exactly do you want to reorder?  I mean, this program is meant
> to dump the complete contents to stdout (whether you redirect it to a file
> or edit it with sed does not concern this program).  It does that.

fast-import does not need to take a complete repository as input, but
can also update lots of branches incrementally, all from one process.
Likewise, it would be nice to have a program  that can also dump small
bits of information.

Maybe I'm misunderstanding you, but fast-export just does not seem a
mirror of fast-import; perhaps you can name it 'dump-all' or
something?

> Maybe you want to specify if all blobs should be output first, and then
> the commits?  Or files should be used?  But all of these things seem to be
> useless to me.

No, I want the program to wait for me to tell it what
blobs/commits/trees I want. The commit I want to see secondly may
depend on the output I read in the first request blob. Right now, for
each data dependency I have to start a new git process.

> > Besides being a nuisance, I actually run git on NFS, and every git
> > process has to go to NFS a couple times to retrieve the same
> > information. This has a noticeable performance impact.
>
> Why don't you just work on a local clone?  If it is really performance
> critical, and I/O is an issue, you are better off working in a tmpfs.

In a company setting, NFS is the easiest way to share information with
colleagues without breaking access control and making our security
staff nervous.  It's also snapshotted and backed up automatically.

> > It would make my life a lot easier if I could simply open a pipe to a
> > single process for the duration of the script, and do all my queries to
> > this one process.  Of course, if the repository is changed by another
> > process, I would have to restart it, but that's manageable.  I could
> > even write a nice Python class that runs both fast-import and
> > fast-export. I could then have an efficient Python interface to a
> > git-repository, without needing any library wrapping.
>
> There is a minimal python wrapper to libgit-thin, which was one of our
> GSoC projects.  You might want to look at this if you are really that
> unhappy with the existing framework.

What's the status of this?  I prefer not to diverge from mainline too much.

-- 
Han-Wen Nienhuys - hanwen@xs4all.nl - http://www.xs4all.nl/~hanwen

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

On Thu, 22 Nov 2007, Han-Wen Nienhuys wrote:

> Maybe I'm misunderstanding you, but fast-export just does not seem a 
> mirror of fast-import; perhaps you can name it 'dump-all' or something?

It is a mirror.  It does not necessarily dump all.

> > Maybe you want to specify if all blobs should be output first, and 
> > then the commits?  Or files should be used?  But all of these things 
> > seem to be useless to me.
> 
> No, I want the program to wait for me to tell it what 
> blobs/commits/trees I want. The commit I want to see secondly may depend 
> on the output I read in the first request blob. Right now, for each data 
> dependency I have to start a new git process.

It does not seem like you want a mirror of fast-import, but rather a 
driver.  You might be happy to hear that you can do that already.  Today.
However, you probably want to query different programs about certain 
states of the repository.  This will not change.

> > > Besides being a nuisance, I actually run git on NFS, and every git 
> > > process has to go to NFS a couple times to retrieve the same 
> > > information. This has a noticeable performance impact.
> >
> > Why don't you just work on a local clone?  If it is really performance 
> > critical, and I/O is an issue, you are better off working in a tmpfs.
> 
> In a company setting, NFS is the easiest way to share information with 
> colleagues without breaking access control and making our security staff 
> nervous.  It's also snapshotted and backed up automatically.

So?

How does that prevent you from following my suggestion to do the intensive 
tasks locally, and push when you finished?

Ciao,
Dscho

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Nguyen Thai Ngoc Duy]

On Nov 21, 2007 10:40 AM, Johannes Schindelin
<Johannes.Schindelin@gmx.de> wrote:
>
> [WIP: this does not handle tags yet, and it lacks a test script
>  as well as documentation.]
>
> This program dumps (parts of) a git repository in the format that
> fast-import understands.
>
> For clarity's sake, it does not use the 'inline' method of specifying
> blobs in the commits, but builds the blobs before building the commits.B
>
> ---
>         I am way too tired now to continue, but maybe someone else wants
>         to pick up the ball.

Well, I would better be back on setup_git_directory() than picking up
the ball. I have a suggestion though. git-fast-export and
git-fast-import should support bundle. Bundle is very handy for
transferring a bunch of commits, but it does not (cannot?) hold tags
and branches. With git-fast-{im,ex}port, we will have a perfect
transportation for disconnected development.

>         Oh, and it relies on "int" being castable to void * and vice
>         versa.  Is anybody aware of a platform where this can lead to
>         problems?
>
>         And yes, I will add a copyright when I woke up again.
-- 
Duy

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

On Fri, 23 Nov 2007, Nguyen Thai Ngoc Duy wrote:

> On Nov 21, 2007 10:40 AM, Johannes Schindelin
> <Johannes.Schindelin@gmx.de> wrote:
> >
> > [WIP: this does not handle tags yet, and it lacks a test script
> >  as well as documentation.]
> >
> > This program dumps (parts of) a git repository in the format that
> > fast-import understands.
> >
> > For clarity's sake, it does not use the 'inline' method of specifying
> > blobs in the commits, but builds the blobs before building the commits.B
> >
> > ---
> >         I am way too tired now to continue, but maybe someone else wants
> >         to pick up the ball.
> 
> Well, I would better be back on setup_git_directory() than picking up
> the ball.

Concur.

> I have a suggestion though. git-fast-export and git-fast-import should 
> support bundle.

I think this is not what fast-export and fast-import are about.  They use 
an easy to generate, and easy to edit, format.

Bundles are optimised transport mechanisms for sneaker net.  They are not 
to be meant to be easy to edit, but as small as possible.

> Bundle is very handy for transferring a bunch of commits, but it does 
> not (cannot?) hold tags and branches.

But they can!  Nothing prevents you from calling

	git bundle create a1.bundle refs/tags/v1.0.0 refs/heads/next

(At least this is the idea, haven't tested yet).

Ciao,
Dscho

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Shawn O. Pearce]

Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> On Fri, 23 Nov 2007, Nguyen Thai Ngoc Duy wrote:
> > I have a suggestion though. git-fast-export and git-fast-import should 
> > support bundle.
> 
> I think this is not what fast-export and fast-import are about.  They use 
> an easy to generate, and easy to edit, format.
> 
> Bundles are optimised transport mechanisms for sneaker net.  They are not 
> to be meant to be easy to edit, but as small as possible.

Actually I wonder, what about bundles that are formatted as a
git-fast-import data stream?  Then you can have a human readable
bundle format that can be (reasonably) easily turned back into a
packfile and loaded into the local ODB.

Only we'd probably want to express blobs as diffs if we can, to
save disk space, which means we'd need to have git-apply organized
in a way that we can call it from within fast-import.  :-)

But I agree with Dscho's basic comment; fast-import should not
be reading a bundle.  It doesn't want to.  That's what git-bundle
is for.

-- 
Shawn.

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Shawn O. Pearce]

Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> On Thu, 22 Nov 2007, Han-Wen Nienhuys wrote:
> 
> > > Maybe you want to specify if all blobs should be output first, and 
> > > then the commits?  Or files should be used?  But all of these things 
> > > seem to be useless to me.
> > 
> > No, I want the program to wait for me to tell it what 
> > blobs/commits/trees I want. The commit I want to see secondly may depend 
> > on the output I read in the first request blob. Right now, for each data 
> > dependency I have to start a new git process.
> 
> It does not seem like you want a mirror of fast-import, but rather a 
> driver.  You might be happy to hear that you can do that already.  Today.
> However, you probably want to query different programs about certain 
> states of the repository.  This will not change.
> 
> > > > Besides being a nuisance, I actually run git on NFS, and every git 
> > > > process has to go to NFS a couple times to retrieve the same 
> > > > information. This has a noticeable performance impact.

I have been considering creating a "git-gui daemon" process that
links to libgit.a and can be driven bidirectionally through its
stdin/stdout.  Based on git-fast-export, sorta.  But I haven't
even started it...

But the idea is sort of what Han-Wen wants.  Why should I fork
rev-parse to get a ref value?  Or update-ref to change one?

-- 
Shawn.

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Nguyen Thai Ngoc Duy]

On Nov 23, 2007 9:31 PM, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> Hi,
>
> On Fri, 23 Nov 2007, Nguyen Thai Ngoc Duy wrote:
>
> > On Nov 21, 2007 10:40 AM, Johannes Schindelin
> > <Johannes.Schindelin@gmx.de> wrote:
> > >
> > > [WIP: this does not handle tags yet, and it lacks a test script
> > >  as well as documentation.]
> > >
> > > This program dumps (parts of) a git repository in the format that
> > > fast-import understands.
> > >
> > > For clarity's sake, it does not use the 'inline' method of specifying
> > > blobs in the commits, but builds the blobs before building the commits.B
> > >
> > > ---
> > >         I am way too tired now to continue, but maybe someone else wants
> > >         to pick up the ball.
> >
> > Well, I would better be back on setup_git_directory() than picking up
> > the ball.
>
> Concur.
>
> > I have a suggestion though. git-fast-export and git-fast-import should
> > support bundle.
>
> I think this is not what fast-export and fast-import are about.  They use
> an easy to generate, and easy to edit, format.
>
> Bundles are optimised transport mechanisms for sneaker net.  They are not
> to be meant to be easy to edit, but as small as possible.
>
> > Bundle is very handy for transferring a bunch of commits, but it does
> > not (cannot?) hold tags and branches.
>
> But they can!  Nothing prevents you from calling
>
>         git bundle create a1.bundle refs/tags/v1.0.0 refs/heads/next
>
> (At least this is the idea, haven't tested yet).

It can store commits and heavy tags, but it won't restore tags to
refs/tags or advance branches.

-- 
Duy

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Karl Hasselström]

On 2007-11-23 15:59:58 -0500, Shawn O. Pearce wrote:

> I have been considering creating a "git-gui daemon" process that
> links to libgit.a and can be driven bidirectionally through its
> stdin/stdout. Based on git-fast-export, sorta. But I haven't even
> started it...
>
> But the idea is sort of what Han-Wen wants. Why should I fork
> rev-parse to get a ref value? Or update-ref to change one?

Obviously, something like this would be very valuable for StGit as
well.

-- 
Karl Hasselström, kha@treskal.com
      www.treskal.com/kalle

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

On Fri, 23 Nov 2007, Shawn O. Pearce wrote:

> Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> > On Thu, 22 Nov 2007, Han-Wen Nienhuys wrote:
> > 
> > > > Maybe you want to specify if all blobs should be output first, and 
> > > > then the commits?  Or files should be used?  But all of these things 
> > > > seem to be useless to me.
> > > 
> > > No, I want the program to wait for me to tell it what 
> > > blobs/commits/trees I want. The commit I want to see secondly may depend 
> > > on the output I read in the first request blob. Right now, for each data 
> > > dependency I have to start a new git process.
> > 
> > It does not seem like you want a mirror of fast-import, but rather a 
> > driver.  You might be happy to hear that you can do that already.  Today.
> > However, you probably want to query different programs about certain 
> > states of the repository.  This will not change.
> > 
> > > > > Besides being a nuisance, I actually run git on NFS, and every git 
> > > > > process has to go to NFS a couple times to retrieve the same 
> > > > > information. This has a noticeable performance impact.
> 
> I have been considering creating a "git-gui daemon" process that links 
> to libgit.a and can be driven bidirectionally through its stdin/stdout.  
> Based on git-fast-export, sorta.  But I haven't even started it...
> 
> But the idea is sort of what Han-Wen wants.  Why should I fork rev-parse 
> to get a ref value?  Or update-ref to change one?

I was thinking about this a little bit more.  But I cannot think of a 
really versatile way of enhancing fast-export enough to be of use there.  
Well, if not doing something with SWIG, that is ;-)

Ciao,
Dscho

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

On Sun, 25 Nov 2007, Karl Hasselstr?m wrote:

> On 2007-11-23 15:59:58 -0500, Shawn O. Pearce wrote:
> 
> > I have been considering creating a "git-gui daemon" process that links 
> > to libgit.a and can be driven bidirectionally through its 
> > stdin/stdout. Based on git-fast-export, sorta. But I haven't even 
> > started it...
> >
> > But the idea is sort of what Han-Wen wants. Why should I fork 
> > rev-parse to get a ref value? Or update-ref to change one?
> 
> Obviously, something like this would be very valuable for StGit as well.

Could you be a little more specific _what_ you want to do, and _how_ this 
could be done with fast-export | fast-import?

Ciao,
Dscho

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Karl Hasselström]

On 2007-11-26 16:48:14 +0000, Johannes Schindelin wrote:

> On Sun, 25 Nov 2007, Karl Hasselström wrote:
>
> > On 2007-11-23 15:59:58 -0500, Shawn O. Pearce wrote:
> >
> > > I have been considering creating a "git-gui daemon" process that
> > > links to libgit.a and can be driven bidirectionally through its
> > > stdin/stdout. Based on git-fast-export, sorta. But I haven't
> > > even started it...
> > >
> > > But the idea is sort of what Han-Wen wants. Why should I fork
> > > rev-parse to get a ref value? Or update-ref to change one?
> >
> > Obviously, something like this would be very valuable for StGit as
> > well.
>
> Could you be a little more specific _what_ you want to do, and _how_
> this could be done with fast-export | fast-import?

Currently, a single StGit command can result in quite a few
invocations of git-cat-file, for example, each of which forks off a
new process. If it could start just one daemon such as Shawn proposed,
and feed it simple questions and commands about blobs, trees, commits,
and refs, that would probably be quite a lot faster.

From what I understand, this is not something that would fit a
fast-export | fast-import pipeline. Which is why I didn't take the
time to elaborate on (or indeed find out) exactly which commands StGit
would like such a daemon to support.

By the way: one command one _would_ likely want in the daemon is "list
modified files". Being long-lived (not when driven by StGit, perhaps,
but definitely when driven by git-gui or qgit), the daemon would be
able to use inotify for that.

-- 
Karl Hasselström, kha@treskal.com
      www.treskal.com/kalle

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

On Tue, 27 Nov 2007, Karl Hasselstr?m wrote:

> On 2007-11-26 16:48:14 +0000, Johannes Schindelin wrote:
> 
> > On Sun, 25 Nov 2007, Karl Hasselstr?m wrote:
> >
> > > On 2007-11-23 15:59:58 -0500, Shawn O. Pearce wrote:
> > >
> > > > I have been considering creating a "git-gui daemon" process that 
> > > > links to libgit.a and can be driven bidirectionally through its 
> > > > stdin/stdout. Based on git-fast-export, sorta. But I haven't even 
> > > > started it...
> > > >
> > > > But the idea is sort of what Han-Wen wants. Why should I fork 
> > > > rev-parse to get a ref value? Or update-ref to change one?
> > >
> > > Obviously, something like this would be very valuable for StGit as 
> > > well.
> >
> > Could you be a little more specific _what_ you want to do, and _how_ 
> > this could be done with fast-export | fast-import?
> 
> Currently, a single StGit command can result in quite a few invocations 
> of git-cat-file, for example, each of which forks off a new process. If 
> it could start just one daemon such as Shawn proposed, and feed it 
> simple questions and commands about blobs, trees, commits, and refs, 
> that would probably be quite a lot faster.
> 
> From what I understand, this is not something that would fit a 
> fast-export | fast-import pipeline. Which is why I didn't take the time 
> to elaborate on (or indeed find out) exactly which commands StGit would 
> like such a daemon to support.
> 
> By the way: one command one _would_ likely want in the daemon is "list 
> modified files". Being long-lived (not when driven by StGit, perhaps, 
> but definitely when driven by git-gui or qgit), the daemon would be able 
> to use inotify for that.

Ah, so you would like something like "git --interactive"?  This is indeed 
a completely different scope than the fast-export thingie, which is meant 
as kind of a mysqldump tool.  Indeed, you could use it even as a (kind of 
a) stash of the repo instead of the working tree.

Ciao,
Dscho

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

On Sat, 24 Nov 2007, Nguyen Thai Ngoc Duy wrote:

> On Nov 23, 2007 9:31 PM, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
>
> > On Fri, 23 Nov 2007, Nguyen Thai Ngoc Duy wrote:
> >
> > > Bundle is very handy for transferring a bunch of commits, but it 
> > > does not (cannot?) hold tags and branches.
> >
> > But they can!  Nothing prevents you from calling
> >
> >         git bundle create a1.bundle refs/tags/v1.0.0 refs/heads/next
> >
> > (At least this is the idea, haven't tested yet).
> 
> It can store commits and heavy tags, but it won't restore tags to 
> refs/tags or advance branches.

The idea is that you fetch them from the bundle.  So something like this 
should do what you want:

	git fetch a1.bundle v1.0.0:refs/tags/v1.0.0

Note that the automatic tag handling of git fetch kicks in with bundles 
just like with other fetch URLs (Unassuming Repository Locators).

Ciao,
Dscho

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Nguyen Thai Ngoc Duy]

On Nov 27, 2007 7:16 PM, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> Hi,
>
> On Sat, 24 Nov 2007, Nguyen Thai Ngoc Duy wrote:
>
> > On Nov 23, 2007 9:31 PM, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> >
> > > On Fri, 23 Nov 2007, Nguyen Thai Ngoc Duy wrote:
> > >
> > > > Bundle is very handy for transferring a bunch of commits, but it
> > > > does not (cannot?) hold tags and branches.
> > >
> > > But they can!  Nothing prevents you from calling
> > >
> > >         git bundle create a1.bundle refs/tags/v1.0.0 refs/heads/next
> > >
> > > (At least this is the idea, haven't tested yet).
> >
> > It can store commits and heavy tags, but it won't restore tags to
> > refs/tags or advance branches.
>
> The idea is that you fetch them from the bundle.  So something like this
> should do what you want:
>
>         git fetch a1.bundle v1.0.0:refs/tags/v1.0.0
>
> Note that the automatic tag handling of git fetch kicks in with bundles
> just like with other fetch URLs (Unassuming Repository Locators).

Yes, you are right. Man, if I knew I could pass a bundle to git-fetch,
my life would have been much easier :(
-- 
Duy

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Karl Hasselström]

On 2007-11-27 11:25:47 +0000, Johannes Schindelin wrote:

> Ah, so you would like something like "git --interactive"? This is
> indeed a completely different scope than the fast-export thingie,

Yes. Or rather, I _think_ that's what I want. The only numbers I have
is that StGit makes a number of trivial git calls that right now take
on the order of 10 ms apiece, so the first step in this direction
would be to build a simple prototype just to see what kind of speed-up
one could expect (both in the git calls, and in StGit overall).

-- 
Karl Hasselström, kha@treskal.com
      www.treskal.com/kalle

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

[Johannes Schindelin]

Hi,

On Tue, 27 Nov 2007, Karl Hasselstr?m wrote:

> On 2007-11-27 11:25:47 +0000, Johannes Schindelin wrote:
> 
> > Ah, so you would like something like "git --interactive"? This is 
> > indeed a completely different scope than the fast-export thingie,
> 
> Yes. Or rather, I _think_ that's what I want. The only numbers I have is 
> that StGit makes a number of trivial git calls that right now take on 
> the order of 10 ms apiece, so the first step in this direction would be 
> to build a simple prototype just to see what kind of speed-up one could 
> expect (both in the git calls, and in StGit overall).

I'd rather see you using libgit-thin's Python binding.

Ciao,
Dscho