[RFC PATCH] Allow cloning branches selectively

[RFC PATCH] Allow cloning branches selectively

[Carlos Martín Nieto]

Sometimes it's useful to clone only a subset of branches from a remote
we're cloning. Teach clone the --fetch option to select which branches
should get fetched.

Each --fetch sets up a fetch refspec for that branch. Previously this
was only possible by initializing a repo and manually setting up the
config.
---

This is still a WIP, as clone will still always fetch and checkout the
remote's HEAD, which leaves you with a detached HEAD if you didn't
want that branch, which is clearly a bug.

Otherwise it works as expected. A better name for this feature would
also be nice, as in git you don't clone branches but repos. Maybe
something like "selectively fetch branches on clone"? If there isn't
an outcry against this I'll add --fetch to the manpage as well.

   cmn

 builtin/clone.c          |   82 ++++++++++++++++++++++++++++++++++------------
 t/t5702-clone-options.sh |   22 +++++++++++--
 2 files changed, 80 insertions(+), 24 deletions(-)

diff --git a/builtin/clone.c b/builtin/clone.c
index 86db954..f14ca2a 100644
--- a/builtin/clone.c
+++ b/builtin/clone.c
@@ -48,6 +48,17 @@ static int option_verbosity;
 static int option_progress;
 static struct string_list option_config;
 static struct string_list option_reference;
+static struct string_list option_fetch;
+static const char **refspecs;
+static int refspecs_nr;
+static int refspecs_alloc;
+
+static void add_refspec(const char *ref)
+{
+	refspecs_nr++;
+	ALLOC_GROW(refspecs, refspecs_nr, refspecs_alloc);
+	refspecs[refspecs_nr-1] = ref;
+}
 
 static int opt_parse_reference(const struct option *opt, const char *arg, int unset)
 {
@@ -88,6 +99,8 @@ static struct option builtin_clone_options[] = {
 		   "use <name> instead of 'origin' to track upstream"),
 	OPT_STRING('b', "branch", &option_branch, "branch",
 		   "checkout <branch> instead of the remote's HEAD"),
+	OPT_STRING_LIST(0, "fetch", &option_fetch, "refspec",
+		   "fetch <refspec> instead of all the branches"),
 	OPT_STRING('u', "upload-pack", &option_upload_pack, "path",
 		   "path to git-upload-pack on the remote"),
 	OPT_STRING(0, "depth", &option_depth, "depth",
@@ -421,13 +434,16 @@ static void remove_junk_on_signal(int signo)
 }
 
 static struct ref *wanted_peer_refs(const struct ref *refs,
-		struct refspec *refspec)
+		struct refspec *refspec, int refspec_nr)
 {
 	struct ref *head = copy_ref(find_ref_by_name(refs, "HEAD"));
 	struct ref *local_refs = head;
 	struct ref **tail = head ? &head->next : &local_refs;
+	int i;
+
+	for (i = 0; i < refspec_nr; i++)
+		get_fetch_map(refs, &refspec[i], &tail, 0);
 
-	get_fetch_map(refs, refspec, &tail, 0);
 	if (!option_mirror)
 		get_fetch_map(refs, tag_refspec, &tail, 0);
 
@@ -482,7 +498,6 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	int err = 0;
 
 	struct refspec *refspec;
-	const char *fetch_pattern;
 
 	junk_pid = getpid();
 
@@ -508,6 +523,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 		option_no_checkout = 1;
 	}
 
+	if (option_mirror && refspecs)
+		die(_("--mirror and --fetch options are incompatible"));
+
 	if (!option_origin)
 		option_origin = "origin";
 
@@ -594,30 +612,55 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	git_config(git_default_config, NULL);
 
 	if (option_bare) {
-		if (option_mirror)
+		git_config_set("core.bare", "true");
+		if (option_mirror) {
 			src_ref_prefix = "refs/";
-		strbuf_addstr(&branch_top, src_ref_prefix);
+			strbuf_addf(&key, "remote.%s.mirror", option_origin);
+			git_config_set(key.buf, "true");
+			strbuf_reset(&key);
+		}
 
-		git_config_set("core.bare", "true");
+		strbuf_addstr(&branch_top, src_ref_prefix);
 	} else {
 		strbuf_addf(&branch_top, "refs/remotes/%s/", option_origin);
 	}
 
-	strbuf_addf(&value, "+%s*:%s*", src_ref_prefix, branch_top.buf);
+	strbuf_reset(&key);
+	strbuf_addf(&key, "remote.%s.fetch", option_origin);
+
+	/* If the user didn't override it, use the default values */
+	if (option_fetch.nr == 0) {
+		strbuf_reset(&value);
+		strbuf_addf(&value, "+%s*:%s*", src_ref_prefix,
+			    branch_top.buf);
 
-	if (option_mirror || !option_bare) {
-		/* Configure the remote */
-		strbuf_addf(&key, "remote.%s.fetch", option_origin);
 		git_config_set_multivar(key.buf, value.buf, "^$", 0);
-		strbuf_reset(&key);
+		add_refspec(strbuf_detach(&value, NULL));
+	} else {
+		int i;
+		for (i = 0; i < option_fetch.nr; i++) {
+			const char *ref = option_fetch.items[i].string;
+			if (!valid_fetch_refspec(ref))
+				die(_("Not a valid fetch refspec: %s"), ref);
+
+			/* If we only got a branch name, make it a proper refspec */
+			if (!strchr(ref, ':')) {
+				strbuf_reset(&value);
+				strbuf_addf(&value, "refs/heads/%s:%s%s",
+					    ref, branch_top.buf, ref);
+				ref = value.buf;
+			}
 
-		if (option_mirror) {
-			strbuf_addf(&key, "remote.%s.mirror", option_origin);
-			git_config_set(key.buf, "true");
-			strbuf_reset(&key);
+			printf("Adding refpsec %s\n", ref);
+			git_config_set_multivar(key.buf, value.buf, "^$", 0);
+			add_refspec(strbuf_detach(&value, NULL));
 		}
+
+		strbuf_reset(&value);
+		strbuf_reset(&key);
 	}
 
+	strbuf_reset(&key);
 	strbuf_addf(&key, "remote.%s.url", option_origin);
 	git_config_set(key.buf, repo);
 	strbuf_reset(&key);
@@ -625,14 +668,11 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	if (option_reference.nr)
 		setup_reference();
 
-	fetch_pattern = value.buf;
-	refspec = parse_fetch_refspec(1, &fetch_pattern);
-
-	strbuf_reset(&value);
+	refspec = parse_fetch_refspec(refspecs_nr, refspecs);
 
 	if (is_local) {
 		refs = clone_local(path, git_dir);
-		mapped_refs = wanted_peer_refs(refs, refspec);
+		mapped_refs = wanted_peer_refs(refs, refspec, refspecs_nr);
 	} else {
 		struct remote *remote = remote_get(option_origin);
 		transport = transport_get(remote, remote->url[0]);
@@ -654,7 +694,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 
 		refs = transport_get_remote_refs(transport);
 		if (refs) {
-			mapped_refs = wanted_peer_refs(refs, refspec);
+			mapped_refs = wanted_peer_refs(refs, refspec, refspecs_nr);
 			transport_fetch_refs(transport, mapped_refs);
 		}
 	}
diff --git a/t/t5702-clone-options.sh b/t/t5702-clone-options.sh
index 02cb024..53d914f 100755
--- a/t/t5702-clone-options.sh
+++ b/t/t5702-clone-options.sh
@@ -8,15 +8,17 @@ test_expect_success 'setup' '
 	mkdir parent &&
 	(cd parent && git init &&
 	 echo one >file && git add file &&
-	 git commit -m one)
+	 git commit -m one &&
+	 git branch other)
 
 '
 
 test_expect_success 'clone -o' '
 
 	git clone -o foo parent clone-o &&
-	(cd clone-o && git rev-parse --verify refs/remotes/foo/master)
-
+	(cd clone-o &&
+	 git rev-parse --verify refs/remotes/foo/master &&
+	 git rev-parse --verify refs/remotes/foo/other)
 '
 
 test_expect_success 'redirected clone' '
@@ -33,4 +35,18 @@ test_expect_success 'redirected clone -v' '
 
 '
 
+test_expect_success 'select one branch to fetch' '
+	git clone --progress --fetch=master "file://$(pwd)/parent" clone-select-one &&
+	(cd clone-sel &&
+	 git rev-parse --verify refs/remotes/origin/master &&
+	 test_must_fail git rev-parse --verify refs/remotes/origin/other)
+'
+
+test_expect_success 'select several branches to fetch' '
+	git clone --progress --fetch=master --fetch=other "file://$(pwd)/parent" clone-select-many &&
+	(cd clone-sel2 &&
+	 git rev-parse --verify refs/remotes/origin/master &&
+	 git rev-parse --verify refs/remotes/origin/other)
+'
+
 test_done
-- 
1.7.8.352.g876a6f

Re: [RFC PATCH] Allow cloning branches selectively

[Junio C Hamano]

Carlos Martín Nieto <cmn@elego.de> writes:

> Sometimes it's useful to clone only a subset of branches from a remote
> we're cloning. Teach clone the --fetch option to select which branches
> should get fetched.

This is just a knee-jerk reaction without reading the patch text (which I
won't be doing over the holiday weekend anyway), but is the workflow of
the primarly intended audience to clone "a subset of branches" or "a
single branch"?

I have a slight suspicion that this started out as "I often want to create
a clone to _track_ a single branch, but because I am mucking with the code
related to cloning anyway, I might as well allow more than one to be
fetched, even though I do not have any need for that, somebody might find
it useful". And that is why it is important to answer the first question.

If the primary motivation is for a single branch, I suspect supporting
only a single branch and advertising the feature as "tracking only one
branch" might make it much easier to understand to the end users.

Upon "git clone --track cn/single-clone $there x.git", you would do
something like:

  it=cn/single-clone &&
  git init x.git &&
  cd x.git &&

  # configure "git fetch origin" to only get branch $it
  git config remote.origin.url "$there" &&
  git config remote.origin.fetch refs/heads/$it:refs/remotes/origin/$it &&

  # declare that the primary branch at origin is $it as far as we are concerned
  git symbolic-ref -m clone refs/remotes/origin/HEAD refs/remotes/origin/$it &&

  # Git aware prompt reminds us that this repository is to track branch $it
  git symbolic-ref -m clone HEAD refs/heads/$it &&

  # And Go!
  git fetch origin &&
  git reset --hard remotes/origin/$it &&
  git config branch.$it.remote origin &&
  git config branch.$it.merge $it

Of course you _could_ support more than one pretty easily, but the point
is that it is unclear how you explain to the end user what the feature
does and what it is for in easily understoodd terms, once you start doing
so. It will no longer be "this new clone is to track that branch", but
something else, and I do not know what that something else is.

And depending on what that "something else" is, which branch should be
checked out and what refs/remotes/origin/HEAD should name as the primary
branch of the remote would be different.

Thanks.

Re: [RFC PATCH] Allow cloning branches selectively

[Nguyen Thai Ngoc Duy]

On Sat, Dec 24, 2011 at 3:13 AM, Carlos Martín Nieto <cmn@elego.de> wrote:
> Sometimes it's useful to clone only a subset of branches from a remote
> we're cloning. Teach clone the --fetch option to select which branches
> should get fetched.

What about tags? Are all tags fetched or only ones part of the
selected branches?
-- 
Duy

Re: [RFC PATCH] Allow cloning branches selectively

[Jakub Narebski]

Carlos Martín Nieto <cmn@elego.de> writes:

> Sometimes it's useful to clone only a subset of branches from a remote
> we're cloning. Teach clone the --fetch option to select which branches
> should get fetched.
> 
> Each --fetch sets up a fetch refspec for that branch. Previously this
> was only possible by initializing a repo and manually setting up the
> config.

I haven't read the code, but I guess it should be possible to share
code with "git remote add", which allows to select which branches to
track, and which branch is to be 'main' on remote.

  git remote add [-t <branch>] [-m <master>] [-f] ...

-- 
Jakub Narebski

Re: [RFC PATCH] Allow cloning branches selectively

[Carlos Martín Nieto]

[-- Attachment #1: Type: text/plain, Size: 838 bytes --]

On Sun, Dec 25, 2011 at 08:28:39AM -0800, Jakub Narebski wrote:
> Carlos Martín Nieto <cmn@elego.de> writes:
> 
> > Sometimes it's useful to clone only a subset of branches from a remote
> > we're cloning. Teach clone the --fetch option to select which branches
> > should get fetched.
> > 
> > Each --fetch sets up a fetch refspec for that branch. Previously this
> > was only possible by initializing a repo and manually setting up the
> > config.
> 
> I haven't read the code, but I guess it should be possible to share
> code with "git remote add", which allows to select which branches to
> track, and which branch is to be 'main' on remote.
> 
>   git remote add [-t <branch>] [-m <master>] [-f] ...

Ah, very nice. I didn't know about those options. Hopefully I can use
it. Thanks for pointing it out.

   cmn

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 490 bytes --]

Re: [RFC PATCH] Allow cloning branches selectively

[Carlos Martín Nieto]

[-- Attachment #1: Type: text/plain, Size: 596 bytes --]

On Sat, Dec 24, 2011 at 11:31:51AM +0700, Nguyen Thai Ngoc Duy wrote:
> On Sat, Dec 24, 2011 at 3:13 AM, Carlos Martín Nieto <cmn@elego.de> wrote:
> > Sometimes it's useful to clone only a subset of branches from a remote
> > we're cloning. Teach clone the --fetch option to select which branches
> > should get fetched.
> 
> What about tags? Are all tags fetched or only ones part of the
> selected branches?

I haven't really touched that part of the code, so I think it'll fetch
every tag, as that's what clone does by default. Certainly something
that should get fixed.

   cmn

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 490 bytes --]

Re: [RFC PATCH] Allow cloning branches selectively

[Carlos Martín Nieto]

[-- Attachment #1: Type: text/plain, Size: 3455 bytes --]

On Fri, Dec 23, 2011 at 01:18:36PM -0800, Junio C Hamano wrote:
> Carlos Martín Nieto <cmn@elego.de> writes:
> 
> > Sometimes it's useful to clone only a subset of branches from a remote
> > we're cloning. Teach clone the --fetch option to select which branches
> > should get fetched.
> 
> This is just a knee-jerk reaction without reading the patch text (which I
> won't be doing over the holiday weekend anyway), but is the workflow of
> the primarly intended audience to clone "a subset of branches" or "a
> single branch"?
> 
> I have a slight suspicion that this started out as "I often want to create
> a clone to _track_ a single branch, but because I am mucking with the code
> related to cloning anyway, I might as well allow more than one to be
> fetched, even though I do not have any need for that, somebody might find
> it useful". And that is why it is important to answer the first question.

The main usefulness is indeed to track single branches. Limiting it to
one looked to me to be an arbitrary limitation and I don't like
those. Tracking between two and all branches is probably quite a niche
however, so I'll go with allowing one -t/--track option and if enough
people want to extend it, we'll see what we do then.

> 
> If the primary motivation is for a single branch, I suspect supporting
> only a single branch and advertising the feature as "tracking only one
> branch" might make it much easier to understand to the end users.
> 
> Upon "git clone --track cn/single-clone $there x.git", you would do
> something like:
> 
>   it=cn/single-clone &&
>   git init x.git &&
>   cd x.git &&
> 
>   # configure "git fetch origin" to only get branch $it
>   git config remote.origin.url "$there" &&
>   git config remote.origin.fetch refs/heads/$it:refs/remotes/origin/$it 
> 
>   # declare that the primary branch at origin is $it as far as we are concerned
>   git symbolic-ref -m clone refs/remotes/origin/HEAD refs/remotes/origin/$it &&

As Jakub pointed out, remote already has this option, so you can
replace these calls with

    git remote add origin $there -t $it -m $it

> 
>   # Git aware prompt reminds us that this repository is to track branch $it
>   git symbolic-ref -m clone HEAD refs/heads/$it &&
> 
>   # And Go!
>   git fetch origin &&
>   git reset --hard remotes/origin/$it &&
>   git config branch.$it.remote origin &&
>   git config branch.$it.merge $it

A lazier man (or one who doesn't work with internals every day) might do

    git checkout -t origin/$it

which the documentation claims would do the same thing. So it boils
down to doing three commands and it feels like it'd be much easier to
just write a small script than to modify the clone code. Putting it in
C will hopefully make the code a bit cleaner, if I use the code that
already exists in remote.

> 
> Of course you _could_ support more than one pretty easily, but the point
> is that it is unclear how you explain to the end user what the feature
> does and what it is for in easily understoodd terms, once you start doing
> so. It will no longer be "this new clone is to track that branch", but
> something else, and I do not know what that something else is.

Looking at the explanation for the -t (--track) option in git-remote,
it's certainly not very friendly unless you already know exactly what
a -t option would do if you were to implement it.

   cmn

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 490 bytes --]