[PATCH] git builtin "push"

[PATCH] git builtin "push"

[Linus Torvalds]

This adds a builtin "push" command, which is largely just a C'ification of 
the "git-push.sh" script.

Now, the reason I did it as a built-in is partly because it's yet another 
step on relying less on shell, but it's actually mostly because I've 
wanted to be able to push to _multiple_ repositories, and the most obvious 
and simplest interface for that would seem be to just have a "remotes" 
file that has multiple URL entries.

(For "pull", having multiple entries should either just select the first 
one, or you could fall back on the others on failure - your choice).

And quite frankly, it just became too damn messy to do that in shell. 
Besides, we actually have a fair amount of infrastructure in C, so it just 
wasn't that hard to do.

Of course, this is almost totally untested. It probably doesn't work for 
anything but the one trial I threw at it. "Simple" doesn't necessarily 
mean "obviously correct".

Signed-off-by: Linus Torvalds <torvalds@osdl.org>
---

Comments?  I wrote it so that it _should_ be fairly easy to re-use at 
least the branches/remotes helper functions for a built-in "git fetch" as 
well. But I didn't have the multi-URI issue with anything but pushing.

Junio - I currently have a "push-all" script in each of the repos I 
maintain. That has worked, but I'd just rather do something like

	git push all

instead, and have it pick up the list of URI's from .git/remotes/all.

How do you do multi-targeted pushes?

diff --git a/Makefile b/Makefile
index 8ce27a6..9919992 100644
--- a/Makefile
+++ b/Makefile
@@ -214,7 +214,7 @@ LIB_OBJS = \
 	$(DIFF_OBJS)
 
 BUILTIN_OBJS = \
-	builtin-log.o builtin-help.o
+	builtin-log.o builtin-help.o builtin-push.o
 
 GITLIBS = $(LIB_FILE) $(XDIFF_LIB)
 LIBS = $(GITLIBS) -lz
diff --git a/builtin-push.c b/builtin-push.c
new file mode 100644
index 0000000..a0c1caa
--- /dev/null
+++ b/builtin-push.c
@@ -0,0 +1,255 @@
+/*
+ * "git push"
+ */
+#include "cache.h"
+#include "refs.h"
+#include "run-command.h"
+#include "builtin.h"
+
+#define MAX_URI (16)
+
+static const char push_usage[] = "git push [--all] [--tags] [--force] <repository> [<refspec>...]";
+
+static int all = 0, tags = 0, force = 0, thin = 1;
+static const char *execute = NULL;
+
+#define BUF_SIZE (2084)
+static char buffer[BUF_SIZE];
+
+static const char **refspec = NULL;
+static int refspec_nr = 0;
+
+static void add_refspec(const char *ref)
+{
+	int nr = refspec_nr + 1;
+	refspec = xrealloc(refspec, nr * sizeof(char *));
+	refspec[nr-1] = ref;
+	refspec_nr = nr;
+}
+
+static int expand_one_ref(const char *ref, const unsigned char *sha1)
+{
+	/* Ignore the "refs/" at the beginning of the refname */
+	ref += 5;
+
+	if (strncmp(ref, "tags/", 5))
+		return 0;
+
+	add_refspec(strdup(ref));
+	return 0;
+}
+
+static void expand_refspecs(void)
+{
+	if (all) {
+		if (refspec_nr)
+			die("cannot mix '--all' and a refspec");
+
+		/*
+		 * No need to expand "--all" - we'll just use
+		 * the "--all" flag to send-pack
+		 */
+		return;
+	}
+	if (!tags)
+		return;
+	for_each_ref(expand_one_ref);
+}
+
+static void set_refspecs(const char **refs, int nr)
+{
+	if (nr) {
+		size_t bytes = nr * sizeof(char *);
+
+		refspec = xrealloc(refspec, bytes);
+		memcpy(refspec, refs, bytes);
+		refspec_nr = nr;
+	}
+	expand_refspecs();
+}
+
+static int get_remotes_uri(const char *repo, const char *uri[MAX_URI])
+{
+	int n = 0;
+	FILE *f = fopen(git_path("remotes/%s", repo), "r");
+
+	if (!f)
+		return -1;
+	while (fgets(buffer, BUF_SIZE, f)) {
+		char *s, *p;
+
+		if (strncmp("URL: ", buffer, 5))
+			continue;
+		s = buffer + 5;
+
+		/* Remove whitespace at the head.. */
+		while (isspace(*s))
+			s++;
+		if (!*s)
+			continue;
+
+		/* ..and at the end */
+		p = s + strlen(s);
+		while (isspace(p[-1]))
+			*--p = 0;
+
+		uri[n++] = strdup(s);
+		if (n == MAX_URI)
+			break;
+	}
+	fclose(f);
+	if (!n)
+		die("remote '%s' has no URL", repo);
+	return n;
+}
+
+static int get_branches_uri(const char *repo, const char *uri[MAX_URI])
+{
+	const char *slash = strchr(repo, '/');
+	int n = slash ? slash - repo : 1000;
+	FILE *f = fopen(git_path("branches/%.*s", n, repo), "r");
+	char *s, *p;
+	int len;
+
+	if (!f)
+		return 0;
+	s = fgets(buffer, BUF_SIZE, f);
+	fclose(f);
+	if (!s)
+		return 0;
+	while (isspace(*s))
+		s++;
+	if (!*s)
+		return 0;
+	p = s + strlen(s);
+	while (isspace(p[-1]))
+		*--p = 0;
+	len = p - s;
+	if (slash)
+		len += strlen(slash);
+	p = xmalloc(len + 1);
+	strcpy(p, s);
+	if (slash)
+		strcat(p, slash);
+	uri[0] = p;
+	return 1;
+}
+
+static int get_uri(const char *repo, const char *uri[MAX_URI])
+{
+	int n;
+
+	if (*repo != '/') {
+		n = get_remotes_uri(repo, uri);
+		if (n > 0)
+			return n;
+
+		n = get_branches_uri(repo, uri);
+		if (n > 0)
+			return n;
+	}
+
+	uri[0] = repo;
+	return 1;
+}
+
+static int do_push(const char *repo)
+{
+	const char *uri[MAX_URI];
+	int i, n = get_uri(repo, uri);
+	int remote;
+	const char **argv;
+	int argc;
+
+	n = get_uri(repo, uri);
+	if (n <= 0)
+		die("bad repository '%s'", repo);
+
+	argv = xmalloc((refspec_nr + 10) * sizeof(char *));
+	argv[0] = "dummy-send-pack";
+	argc = 1;
+	if (all)
+		argv[argc++] = "--all";
+	if (force)
+		argv[argc++] = "--force";
+	if (execute)
+		argv[argc++] = execute;
+	if (thin)
+		argv[argc++] = "--thin";
+	remote = argc;
+	argv[argc++] = "dummy-remote";
+	while (refspec_nr--)
+		argv[argc++] = *refspec++;
+	argv[argc] = NULL;
+
+	for (i = 0; i < n; i++) {
+		int error;
+		const char *dest = uri[i];
+		const char *sender = "git-send-pack";
+		if (!strncmp(dest, "http://", 7) ||
+		    !strncmp(dest, "https://", 8))
+			sender = "git-http-push";
+		argv[0] = sender;
+		argv[remote] = dest;
+		error = run_command_v(argc, argv);
+		if (!error)
+			continue;
+		switch (error) {
+		case -ERR_RUN_COMMAND_FORK:
+			die("unable to fork for %s", sender);
+		case -ERR_RUN_COMMAND_EXEC:
+			die("unable to exec %s", sender);
+		case -ERR_RUN_COMMAND_WAITPID:
+		case -ERR_RUN_COMMAND_WAITPID_WRONG_PID:
+		case -ERR_RUN_COMMAND_WAITPID_SIGNAL:
+		case -ERR_RUN_COMMAND_WAITPID_NOEXIT:
+			die("%s died with strange error", sender);
+		default:
+			return -error;
+		}
+	}
+	return 0;
+}
+
+int cmd_push(int argc, const char **argv, char **envp)
+{
+	int i;
+	const char *repo = "origin";	// default repository
+
+	for (i = 1; i < argc; i++) {
+		const char *arg = argv[i];
+
+		if (arg[0] != '-') {
+			repo = arg;
+			i++;
+			break;
+		}
+		if (!strcmp(arg, "--all")) {
+			all = 1;
+			continue;
+		}
+		if (!strcmp(arg, "--tags")) {
+			tags = 1;
+			continue;
+		}
+		if (!strcmp(arg, "--force")) {
+			force = 1;
+			continue;
+		}
+		if (!strcmp(arg, "--thin")) {
+			thin = 1;
+			continue;
+		}
+		if (!strcmp(arg, "--no-thin")) {
+			thin = 0;
+			continue;
+		}
+		if (!strncmp(arg, "--exec=", 7)) {
+			execute = arg;
+			continue;
+		}
+		usage(push_usage);
+	}
+	set_refspecs(argv + i, argc - i);
+	return do_push(repo);
+}
diff --git a/builtin.h b/builtin.h
index 47408a0..94fa9b5 100644
--- a/builtin.h
+++ b/builtin.h
@@ -20,4 +20,6 @@ extern int cmd_whatchanged(int argc, con
 extern int cmd_show(int argc, const char **argv, char **envp);
 extern int cmd_log(int argc, const char **argv, char **envp);
 
+extern int cmd_push(int argc, const char **argv, char **envp);
+
 #endif
diff --git a/git.c b/git.c
index 01b7e28..fd479e9 100644
--- a/git.c
+++ b/git.c
@@ -46,6 +46,7 @@ static void handle_internal_command(int 
 		{ "log", cmd_log },
 		{ "whatchanged", cmd_whatchanged },
 		{ "show", cmd_show },
+		{ "push", cmd_push },
 	};
 	int i;
 

Re: [PATCH] git builtin "push"

[Junio C Hamano]

Linus Torvalds <torvalds@osdl.org> writes:

> Junio - I currently have a "push-all" script in each of the repos I 
> maintain. That has worked, but I'd just rather do something like
>
> 	git push all
>
> instead, and have it pick up the list of URI's from .git/remotes/all.

Makes sense.  I am still somewhat drunk (cachaça -- agua de
beber -- pretty good but strong stuff), so I will look at the
patch tomorrow, but...

> How do you do multi-targeted pushes?

I do not.  At least not that often.  I merge things up, test all
of "master", "maint", "next", and "pu" locally (the Meta/Doit
script, available after checking out "todo" branch in Meta/
subdirectory), then "git push ko-private" to push into ~/git of
the kernel.org machine and do "all branches" test again
(Meta/DoKernelOrg script).  Only after all that passes, I do a
"git push ko" from my local machine to push into the public
area, /pub/scm/git/git.git, of the kernel.org machine.

For Solaris and Cygwin testing, only occasionally, I do not push
but pull from the machines at work.  This is because I want to
catch breakage of pull side myself if there is one.

The only time I do multi-target push is just before cutting a
release.  In addition to "git push ko-private" to push into the
primary Opteron machine, I push to "git push ko-old-private" to
push into the i386 machine at kernel.org, "Meta/DoKernelOrg
master" (or "maint") to prepare RPMs for i386 there, and another
"Meta/DoKernelOrg master" (or "maint") on the Opteron to
prepare RPMs for x86-64.  So personally I never felt the need
for a multi-target push.

Re: [PATCH] git builtin "push"

[sean]

On Sat, 29 Apr 2006 21:22:49 -0700 (PDT)
Linus Torvalds <torvalds@osdl.org> wrote:

> Comments?  I wrote it so that it _should_ be fairly easy to re-use at 
> least the branches/remotes helper functions for a built-in "git fetch" as 
> well. But I didn't have the multi-URI issue with anything but pushing.


> +	if (strncmp(ref, "tags/", 5))
> +		return 0;
[...]
> +	for_each_ref(expand_one_ref);

How about a for_each_tag() function?


> +	int i, n = get_uri(repo, uri);
[...]
> +	n = get_uri(repo, uri);
> +	if (n <= 0)
> +		die("bad repository '%s'", repo);

get_uri() called twice.


The patch looks good and is easy to read, but wouldn't it be better
to require new c code to be thread safe and not leak memory?  Assuming
run-once-and-exit is making it difficult to push into a library.

Sean

builtin-push.c |   11 +++--------
refs.c         |    5 +++++
refs.h         |    1 +
3 files changed, 9 insertions(+), 8 deletions(-)

0e0e3cff71d65ac4cdc560ae143aded03acb4ceb
diff --git a/builtin-push.c b/builtin-push.c
index a0c1caa..0d74ed1 100644
--- a/builtin-push.c
+++ b/builtin-push.c
@@ -31,10 +31,6 @@ static int expand_one_ref(const char *re
 {
 	/* Ignore the "refs/" at the beginning of the refname */
 	ref += 5;
-
-	if (strncmp(ref, "tags/", 5))
-		return 0;
-
 	add_refspec(strdup(ref));
 	return 0;
 }
@@ -51,9 +47,8 @@ static void expand_refspecs(void)
 		 */
 		return;
 	}
-	if (!tags)
-		return;
-	for_each_ref(expand_one_ref);
+	if (tags)
+		for_each_tag(expand_one_ref);
 }
 
 static void set_refspecs(const char **refs, int nr)
@@ -156,7 +151,7 @@ static int get_uri(const char *repo, con
 static int do_push(const char *repo)
 {
 	const char *uri[MAX_URI];
-	int i, n = get_uri(repo, uri);
+	int i, n;
 	int remote;
 	const char **argv;
 	int argc;
diff --git a/refs.c b/refs.c
index 03398cc..c5a2dd0 100644
--- a/refs.c
+++ b/refs.c
@@ -178,6 +178,11 @@ int head_ref(int (*fn)(const char *path,
 	return 0;
 }
 
+int for_each_tag(int (*fn)(const char *path, const unsigned char *sha1))
+{
+	return do_for_each_ref("refs/tags", fn);
+}
+
 int for_each_ref(int (*fn)(const char *path, const unsigned char *sha1))
 {
 	return do_for_each_ref("refs", fn);
diff --git a/refs.h b/refs.h
index 2625596..b74cd4d 100644
--- a/refs.h
+++ b/refs.h
@@ -7,6 +7,7 @@ #define REFS_H
  */
 extern int head_ref(int (*fn)(const char *path, const unsigned char *sha1));
 extern int for_each_ref(int (*fn)(const char *path, const unsigned char *sha1));
+extern int for_each_tag(int (*fn)(const char *path, const unsigned char *sha1));
 
 /** Reads the refs file specified into sha1 **/
 extern int get_ref_sha1(const char *ref, unsigned char *sha1);
-- 
1.3.1.g9c203

Re: [PATCH] git builtin "push"

[Jeff Garzik]

Linus Torvalds wrote:
> Now, the reason I did it as a built-in is partly because it's yet another 
> step on relying less on shell, but it's actually mostly because I've 
> wanted to be able to push to _multiple_ repositories, and the most obvious 
> and simplest interface for that would seem be to just have a "remotes" 
> file that has multiple URL entries.

Thanks!  This is why I still use rsync, even though everybody and their 
mother tells me "Linus says rsync is deprecated."

Fact is, rsync is still the best to push a _bunch_ of branches, in the 
same tree, all in one command.  In an attempt to get away from rsync, I 
even tried putting multiple git:// entries into .git/remotes/origin, 
then doing "git push" with no args, without success.

Each time I have a non-trivial libata update, I've updated twelve (12) 
branches, and think its highly silly to have to run

	url=git://git.kernel.org/...jgarzik/...
	git push $url $branch:$branch

twelve times.

	Jeff

Re: [PATCH] git builtin "push"

[Linus Torvalds]

On Sun, 30 Apr 2006, Jeff Garzik wrote:

> Linus Torvalds wrote:
> > Now, the reason I did it as a built-in is partly because it's yet another
> > step on relying less on shell, but it's actually mostly because I've wanted
> > to be able to push to _multiple_ repositories, and the most obvious and
> > simplest interface for that would seem be to just have a "remotes" file that
> > has multiple URL entries.
> 
> Thanks!  This is why I still use rsync, even though everybody and their mother
> tells me "Linus says rsync is deprecated."

No. You're using rsync because you're actively doing something _wrong_.

> Fact is, rsync is still the best to push a _bunch_ of branches, in the same
> tree, all in one command.

Not so. My builtin thing doesn't help at all, because it doesn't _need_ to 
help that case. You can already push as many branches as you want with the 
old "git push".

My builtin thing helps push to multiple _repositories_, not to multiple 
branches. I push out my stuff to two separate repos at master.kernel.org, 
and one at an osdl.org server machine, because if I lose my tree, I want 
to be sure I have alternate backups.

> In an attempt to get away from rsync, I even tried putting multiple 
> git:// entries into .git/remotes/origin, then doing "git push" with no 
> args, without success.

That's not how it works, and not how it has ever worked.

The way it works is that

	"git push repo"

with no branch specs will update all branches that are IN COMMON between 
your tree and the repository you're pushing to. In other words, it's meant 
to sync up the branches that you have already pushed.

If you want to push other branches, you need to do

	git push repo branch1 branch2 branch3 ...

or

	git push --all repo

where the latter does exactly what it says (use "--tags" instead of 
"--all" to just send all tags).

In all cases you will find that it's a hell of a lot more efficient than 
rsync ever has been.

		Linus

Re: [PATCH] git builtin "push"

[sean]

On Sun, 30 Apr 2006 13:07:42 -0700 (PDT)
Linus Torvalds <torvalds@osdl.org> wrote:

> The way it works is that
> 
> 	"git push repo"
> 
> with no branch specs will update all branches that are IN COMMON between 
> your tree and the repository you're pushing to. In other words, it's meant 
> to sync up the branches that you have already pushed.

True, as long as you don't have any Push: lines in your .git/remotes/repo
file.  Any such lines act just like you gave them on the command line
which afaik effectively removes the ability to say "update all branches
we have in common".

Sean

Re: [PATCH] git builtin "push"

[Linus Torvalds]

On Sun, 30 Apr 2006, sean wrote:

> On Sun, 30 Apr 2006 13:07:42 -0700 (PDT)
> Linus Torvalds <torvalds@osdl.org> wrote:
> 
> > The way it works is that
> > 
> > 	"git push repo"
> > 
> > with no branch specs will update all branches that are IN COMMON between 
> > your tree and the repository you're pushing to. In other words, it's meant 
> > to sync up the branches that you have already pushed.
> 
> True, as long as you don't have any Push: lines in your .git/remotes/repo
> file.  Any such lines act just like you gave them on the command line
> which afaik effectively removes the ability to say "update all branches
> we have in common".

Ok, so my built-in version "fixes" that "feature". 

		Linus

Re: [PATCH] git builtin "push"

[Jeff Garzik]

Linus Torvalds wrote:
> If you want to push other branches, you need to do
> 
> 	git push repo branch1 branch2 branch3 ...
> 
> or
> 
> 	git push --all repo
> 
> where the latter does exactly what it says (use "--tags" instead of 
> "--all" to just send all tags).

After experimenting, "--all" does indeed provide most of the features 
that rsync provides.  A few minor niggles:

1) Doesn't propagate local branch deletions to the remote, like rsync does.

2) git-push "-f" doesn't seem to work, but "--force" does.

3) You still have to provide a $repo argument to 'git pull $repo'. 
Would like to list the default remote push URL in 
.git/remotes/{somefile} so that I need only to do "git push --all" to 
have changes send to any number of remote servers.

4) Propagation of alternatives is unclear (at least in docs).  Without 
my current pack file pre-sharing and hardlinking, I fear needlessly 
uploading vanilla linux-2.6.git changes back to kernel.org, when I do a 
push.  Currently, pack files are downloaded _once_ from kernel.org to 
local, and never re-uploaded.

Regards,

	Jeff

Re: [PATCH] git builtin "push"

[Junio C Hamano]

Linus Torvalds <torvalds@osdl.org> writes:

>> True, as long as you don't have any Push: lines in your .git/remotes/repo
>> file.  Any such lines act just like you gave them on the command line
>> which afaik effectively removes the ability to say "update all branches
>> we have in common".
>
> Ok, so my built-in version "fixes" that "feature". 
>
> 		Linus

Alternatively/additionally you could do something like the
attached patch.  However,...

I suspect that destination repositories that you would want to
define Push: and the ones you do not want to have any Push: are
of different nature and you would not mix them.  When you do
want to have any Push: line for a repository, you would _never_
want to use the "matching refs" semantics for that repository.
So in that sense, the "fix" is not necessary, and fixing it
would probably lead to more confusion.

The Push: lines can do two related but independent things:

 - limit the set of refs that are published (hide the uncooked topics)
 - remap the source and destination refs (e.g. push master to origin)

The first can be done by not having Push: at all and by solely
relying on "matching refs" semantics.  Then usually you can just
keep doing "git push <destination>" until you have something new
to add, at that point you say "git push <destination>
<newbranch>" just once, and after that keep doing the "matching
refs" push.  You are having the remote repository remember what
the set of refs you would want there, instead of listing them on
the local "Push:" lines.  All should work well that way.

I use Push: lines to push into my private, non-bare, repository
at kernel.org machines for the second reason.  I have:

	Push: heads/master:heads/origin
        Push: heads/next:heads/next
        Push: +heads/pu:heads/pu
        Push: heads/maint:heads/maint

and leave the "master" branch checked out over there.  After
pushing, I log-in, and start my work by "git pull . origin".

The "matching refs" feature actively does a wrong thing to
remote repositories managed this way.

-- >8 --
[PATCH] git push --common <repo>

If you usually push only specific branches, it is handy to have
Push: lines only for those branches in your remotes/<repo> file.
But with that, you cannot access the underlying git-send-pack's
ability to push "only matching refs".  The flag --common can be
given to ignore Push: lines and push only matching refs.

---
diff --git a/git-push.sh b/git-push.sh
index f10cadb..381b76f 100755
--- a/git-push.sh
+++ b/git-push.sh
@@ -11,6 +11,7 @@ has_exec=
 has_thin=--thin
 remote=
 do_tags=
+do_common=
 
 while case "$#" in 0) break ;; esac
 do
@@ -19,6 +20,8 @@ do
 		has_all=--all ;;
 	--tags)
 		do_tags=yes ;;
+	--common)
+		do_common=yes ;;
 	--force)
 		has_force=--force ;;
 	--exec=*)
@@ -45,10 +48,21 @@ esac
 . git-parse-remote
 remote=$(get_remote_url "$@")
 
-case "$has_all" in
---all)
+case "$has_all,$do_common" in
+--all,yes)
+	die "Both --common and --all given." ;;
+--all,)
 	set x ;;
-'')
+,yes)
+	case "$do_tags,$#" in
+	,1)
+		set x ;;
+	yes,*)
+		die "Both --common and --tags given." ;;
+	*)
+		die "Both --common and an explicit refspec given." ;;
+	esac ;;
+,)
 	case "$do_tags,$#" in
 	yes,1)
 		set x $(cd "$GIT_DIR/refs" && find tags -type f -print) ;;