[PATCH 1/2] refs_from_alternate: helper to use refs from alternates

[PATCH 1/2] refs_from_alternate: helper to use refs from alternates

[Junio C Hamano]

The receiving end of "git push" advertises the objects that the repository
itself does not use, but are at the tips of refs in other repositories
whose object databases are used as alternates for it. This helps it avoid
having to receive (and the pusher having to send) objects that are already
available to the receiving repository via the alternates mechanism.

Tweak the helper function that implements this feature, and move it to
transport.[ch] for future reuse by other programs.

The additional test demonstrates how this optimization is helping "git push",
and "git fetch" is ignorant about it.

Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 builtin/receive-pack.c           |   35 ++------------------
 t/t5501-fetch-push-alternates.sh |   66 ++++++++++++++++++++++++++++++++++++++
 transport.c                      |   34 +++++++++++++++++++
 transport.h                      |    3 ++
 4 files changed, 106 insertions(+), 32 deletions(-)
 create mode 100755 t/t5501-fetch-push-alternates.sh

diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c
index 760817d..a5b5fc2 100644
--- a/builtin/receive-pack.c
+++ b/builtin/receive-pack.c
@@ -731,43 +731,14 @@ static int delete_only(struct command *commands)
 	return 1;
 }
 
-static int add_refs_from_alternate(struct alternate_object_database *e, void *unused)
+static void add_one_alternate_ref(const struct ref *ref, void *unused)
 {
-	char *other;
-	size_t len;
-	struct remote *remote;
-	struct transport *transport;
-	const struct ref *extra;
-
-	e->name[-1] = '\0';
-	other = xstrdup(make_absolute_path(e->base));
-	e->name[-1] = '/';
-	len = strlen(other);
-
-	while (other[len-1] == '/')
-		other[--len] = '\0';
-	if (len < 8 || memcmp(other + len - 8, "/objects", 8))
-		return 0;
-	/* Is this a git repository with refs? */
-	memcpy(other + len - 8, "/refs", 6);
-	if (!is_directory(other))
-		return 0;
-	other[len - 8] = '\0';
-	remote = remote_get(other);
-	transport = transport_get(remote, other);
-	for (extra = transport_get_remote_refs(transport);
-	     extra;
-	     extra = extra->next) {
-		add_extra_ref(".have", extra->old_sha1, 0);
-	}
-	transport_disconnect(transport);
-	free(other);
-	return 0;
+	add_extra_ref(".have", ref->old_sha1, 0);
 }
 
 static void add_alternate_refs(void)
 {
-	foreach_alt_odb(add_refs_from_alternate, NULL);
+	foreach_alt_odb(refs_from_alternate_cb, add_one_alternate_ref);
 }
 
 int cmd_receive_pack(int argc, const char **argv, const char *prefix)
diff --git a/t/t5501-fetch-push-alternates.sh b/t/t5501-fetch-push-alternates.sh
new file mode 100755
index 0000000..564ef7c
--- /dev/null
+++ b/t/t5501-fetch-push-alternates.sh
@@ -0,0 +1,66 @@
+#!/bin/sh
+
+test_description='fetch/push involving alternates'
+. ./test-lib.sh
+
+count_objects () {
+	(
+		echo 0 &&
+		git count-objects -v |
+		sed -n -e 's/^count: \(.*\)/\1 +/p' \
+		    -e 's/^in-pack: \(.*\)/\1 +/p' &&
+		echo p
+	) | dc
+}
+
+
+test_expect_success setup '
+	(
+		git init original &&
+		cd original &&
+		i=0 &&
+		while test $i -le 100
+		do
+			echo "$i" >count &&
+			git add count &&
+			git commit -m "$i" || exit
+			i=$(($i + 1))
+		done
+	) &&
+	(
+		git clone --reference=original "file:///$(pwd)/original" one &&
+		cd one &&
+		echo Z >count &&
+		git add count &&
+		git commit -m Z &&
+		count_objects >../one.count
+	) &&
+	A=$(pwd)/original/.git/objects &&
+	git init receiver &&
+	echo "$A" >receiver/.git/objects/info/alternates &&
+	git init fetcher &&
+	echo "$A" >fetcher/.git/objects/info/alternates
+'
+
+test_expect_success 'pushing into a repository with the same alternate' '
+	(
+		cd one &&
+		git push ../receiver master:refs/heads/it
+	) &&
+	(
+		cd receiver &&
+		count_objects >../receiver.count
+	) &&
+	test_cmp one.count receiver.count
+'
+
+test_expect_failure 'fetching from a repository with the same alternate' '
+	(
+		cd fetcher &&
+		git fetch ../one master:refs/heads/it &&
+		count_objects >../fetcher.count
+	) &&
+	test_cmp one.count fetcher.count
+'
+
+test_done
diff --git a/transport.c b/transport.c
index 0078660..3dfbc6a 100644
--- a/transport.c
+++ b/transport.c
@@ -1189,3 +1189,37 @@ char *transport_anonymize_url(const char *url)
 literal_copy:
 	return xstrdup(url);
 }
+
+int refs_from_alternate_cb(struct alternate_object_database *e, void *cb)
+{
+	char *other;
+	size_t len;
+	struct remote *remote;
+	struct transport *transport;
+	const struct ref *extra;
+	alternate_ref_fn *ref_fn = cb;
+
+	e->name[-1] = '\0';
+	other = xstrdup(make_absolute_path(e->base));
+	e->name[-1] = '/';
+	len = strlen(other);
+
+	while (other[len-1] == '/')
+		other[--len] = '\0';
+	if (len < 8 || memcmp(other + len - 8, "/objects", 8))
+		return 0;
+	/* Is this a git repository with refs? */
+	memcpy(other + len - 8, "/refs", 6);
+	if (!is_directory(other))
+		return 0;
+	other[len - 8] = '\0';
+	remote = remote_get(other);
+	transport = transport_get(remote, other);
+	for (extra = transport_get_remote_refs(transport);
+	     extra;
+	     extra = extra->next)
+		ref_fn(extra, NULL);
+	transport_disconnect(transport);
+	free(other);
+	return 0;
+}
diff --git a/transport.h b/transport.h
index e803c0e..efb1968 100644
--- a/transport.h
+++ b/transport.h
@@ -166,4 +166,7 @@ int transport_refs_pushed(struct ref *ref);
 void transport_print_push_status(const char *dest, struct ref *refs,
 		  int verbose, int porcelain, int *nonfastforward);
 
+typedef void alternate_ref_fn(const struct ref *, void *);
+extern int refs_from_alternate_cb(struct alternate_object_database *e, void *cb);
+
 #endif
-- 
1.7.4.1.404.g62d316

[PATCH 2/2] fetch-pack: objects in our alternates are available to us

[Junio C Hamano]

Use the helper function split from receiving end of "git push" to allow
the same optimization on the receiving end of "git fetch".

Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 builtin/fetch-pack.c             |   12 ++++++++++++
 t/t5501-fetch-push-alternates.sh |    2 +-
 2 files changed, 13 insertions(+), 1 deletions(-)

diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c
index b999413..4c25968 100644
--- a/builtin/fetch-pack.c
+++ b/builtin/fetch-pack.c
@@ -9,6 +9,7 @@
 #include "fetch-pack.h"
 #include "remote.h"
 #include "run-command.h"
+#include "transport.h"
 
 static int transfer_unpack_limit = -1;
 static int fetch_unpack_limit = -1;
@@ -217,6 +218,16 @@ static void send_request(int fd, struct strbuf *buf)
 		safe_write(fd, buf->buf, buf->len);
 }
 
+static void insert_one_alternate_ref(const struct ref *ref, void *unused)
+{
+	rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
+}
+
+static void insert_alternate_refs(void)
+{
+	foreach_alt_odb(refs_from_alternate_cb, insert_one_alternate_ref);
+}
+
 static int find_common(int fd[2], unsigned char *result_sha1,
 		       struct ref *refs)
 {
@@ -235,6 +246,7 @@ static int find_common(int fd[2], unsigned char *result_sha1,
 	marked = 1;
 
 	for_each_ref(rev_list_insert_ref, NULL);
+	insert_alternate_refs();
 
 	fetching = 0;
 	for ( ; refs ; refs = refs->next) {
diff --git a/t/t5501-fetch-push-alternates.sh b/t/t5501-fetch-push-alternates.sh
index 564ef7c..a0f3644 100755
--- a/t/t5501-fetch-push-alternates.sh
+++ b/t/t5501-fetch-push-alternates.sh
@@ -54,7 +54,7 @@ test_expect_success 'pushing into a repository with the same alternate' '
 	test_cmp one.count receiver.count
 '
 
-test_expect_failure 'fetching from a repository with the same alternate' '
+test_expect_success 'fetching from a repository with the same alternate' '
 	(
 		cd fetcher &&
 		git fetch ../one master:refs/heads/it &&
-- 
1.7.4.1.404.g62d316

Re: [PATCH 1/2] refs_from_alternate: helper to use refs from alternates

[Johannes Sixt]

On Freitag, 11. März 2011, Junio C Hamano wrote:
> +count_objects () {
> +	(
> +		echo 0 &&
> +		git count-objects -v |
> +		sed -n -e 's/^count: \(.*\)/\1 +/p' \
> +		    -e 's/^in-pack: \(.*\)/\1 +/p' &&
> +		echo p
> +	) | dc
> +}

Can we have this without using dc? (We don't have it on Windows.)

diff --git a/t/t5501-fetch-push-alternates.sh 
b/t/t5501-fetch-push-alternates.sh
index 564ef7c..eb844cf 100755
--- a/t/t5501-fetch-push-alternates.sh
+++ b/t/t5501-fetch-push-alternates.sh
@@ -4,13 +4,11 @@ test_description='fetch/push involving alternates'
 . ./test-lib.sh
 
 count_objects () {
-	(
-		echo 0 &&
+	echo $(( $(
 		git count-objects -v |
 		sed -n -e 's/^count: \(.*\)/\1 +/p' \
-		    -e 's/^in-pack: \(.*\)/\1 +/p' &&
-		echo p
-	) | dc
+		    -e 's/^in-pack: \(.*\)/\1/p'
+		) ))
 }
 
 

Re: [PATCH 1/2] refs_from_alternate: helper to use refs from alternates

[Junio C Hamano]

Johannes Sixt <j6t@kdbg.org> writes:

> Can we have this without using dc? (We don't have it on Windows.)
>
> diff --git a/t/t5501-fetch-push-alternates.sh 
> b/t/t5501-fetch-push-alternates.sh
> index 564ef7c..eb844cf 100755
> --- a/t/t5501-fetch-push-alternates.sh
> +++ b/t/t5501-fetch-push-alternates.sh
> @@ -4,13 +4,11 @@ test_description='fetch/push involving alternates'
>  . ./test-lib.sh
>  
>  count_objects () {
> +	echo $(( $(
>  		git count-objects -v |
>  		sed -n -e 's/^count: \(.*\)/\1 +/p' \
> +		    -e 's/^in-pack: \(.*\)/\1/p'
> +		) ))
>  }

What does that do when there are only loose objects without packed ones?

Re: [PATCH 1/2] refs_from_alternate: helper to use refs from alternates

[Junio C Hamano]

Junio C Hamano <gitster@pobox.com> writes:

> Johannes Sixt <j6t@kdbg.org> writes:
>
>> Can we have this without using dc? (We don't have it on Windows.)
>>
>> diff --git a/t/t5501-fetch-push-alternates.sh 
>> b/t/t5501-fetch-push-alternates.sh
>> index 564ef7c..eb844cf 100755
>> --- a/t/t5501-fetch-push-alternates.sh
>> +++ b/t/t5501-fetch-push-alternates.sh
>> @@ -4,13 +4,11 @@ test_description='fetch/push involving alternates'
>>  . ./test-lib.sh
>>  
>>  count_objects () {
>> +	echo $(( $(
>>  		git count-objects -v |
>>  		sed -n -e 's/^count: \(.*\)/\1 +/p' \
>> +		    -e 's/^in-pack: \(.*\)/\1/p'
>> +		) ))
>>  }
>
> What does that do when there are only loose objects without packed ones?

Ahh, Ok, we always say "0", so that is not a problem.  Will squash in.

Thanks.

Re: [PATCH 2/2] fetch-pack: objects in our alternates are available to us

[Shawn Pearce]

On Fri, Mar 11, 2011 at 12:51, Junio C Hamano <gitster@pobox.com> wrote:
> Use the helper function split from receiving end of "git push" to allow
> the same optimization on the receiving end of "git fetch".
>
> Signed-off-by: Junio C Hamano <gitster@pobox.com>
> ---
>  builtin/fetch-pack.c             |   12 ++++++++++++
>  t/t5501-fetch-push-alternates.sh |    2 +-
>  2 files changed, 13 insertions(+), 1 deletions(-)

I apologize for the slow response; both patches
Acked-by: Shawn O. Pearce <spearce@spearce.org>

-- 
Shawn.