[PATCH] fetch: add config to avoid fetching every branch in shallow repo

["Harald Nordgren via GitGitGadget" <gitgitgadget@gmail.com>]

From: Harald Nordgren <haraldnordgren@gmail.com>

In a shallow, sparsely checked out clone of a repository with many
branches, plain git pull can take minutes or hang outright, even
though only one branch is actually being worked on.

Add fetch.shallow, off by default. When enabled, a fetch or pull for
a shallow repository that isn't already scoped to specific refs
fetches only the current branch's tracked upstream, instead of every
branch the remote has. git pull ultimately runs such a fetch under
the hood, so this fixes pull the same way. It has no effect once the
repository is no longer shallow, and no effect on a fetch of a remote
the current branch doesn't track, both fall back to the existing
behavior.

This is opt-in rather than automatic because it changes what a plain
fetch or pull leaves in refs/remotes/<name>/ for anyone who currently
relies on it syncing every branch of a shallow remote, not just the
one they are on. Scoping remote.<name>.fetch by hand already covers
this for a single remote, but that requires knowing the config exists
and applies it permanently, even to branches that are not currently
checked out.

The remote's recorded default branch (remotes/<name>/HEAD) is kept up
to date the same way it always is, only the other branches are
skipped.

Signed-off-by: Harald Nordgren <haraldnordgren@gmail.com>
---
    fetch: add fetch.shallow so pull doesn't fetch every branch on shallow
    repo
    
    Add fetch.shallow config for big shallow repo, so git fetch/pull doesn't
    hang by fetching every branch.

Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-2412%2FHaraldNordgren%2Ffetch-shallow-narrow-refspec-v1
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-2412/HaraldNordgren/fetch-shallow-narrow-refspec-v1
Pull-Request: https://github.com/git/git/pull/2412

 Documentation/config/fetch.adoc |  15 ++++
 builtin/fetch.c                 |  18 +++-
 t/t5537-fetch-shallow.sh        | 146 ++++++++++++++++++++++++++++++++
 3 files changed, 175 insertions(+), 4 deletions(-)

diff --git a/Documentation/config/fetch.adoc b/Documentation/config/fetch.adoc
index 00435e9a16..e63e54a463 100644
--- a/Documentation/config/fetch.adoc
+++ b/Documentation/config/fetch.adoc
@@ -145,3 +145,18 @@ remove the value for the `fetch.bundleCreationToken` value before fetching.
 `never`;;
 	Never create or modify the `remotes/<name>/HEAD` symbolic-ref.
 --
+
+`fetch.shallow`::
+	If true, and the repository is a shallow repository (see
+	linkgit:git-clone[1] `--depth`), a fetch or `git pull` that names no
+	explicit refspec and would otherwise fall back to the remote's
+	configured `remote.<name>.fetch` refspec instead fetches only the
+	current branch's upstream, when that upstream is on the remote being
+	fetched. This avoids negotiating history for every branch the remote
+	advertises, which can be slow on a shallow repository that tracks
+	many disjoint shallow histories. It has no effect on a fetch that
+	names an explicit remote or refspec, and no effect on a repository
+	that is not shallow. Defaults to false.
++
+`remotes/<name>/HEAD` is still kept up to date per `fetch.followRemoteHEAD`
+while this is in effect, only the other branches are skipped.
diff --git a/builtin/fetch.c b/builtin/fetch.c
index 533fdfe7d8..b22f7fe5f4 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -111,6 +111,7 @@ struct fetch_config {
 	int recurse_submodules;
 	int parallel;
 	int submodule_fetch_jobs;
+	int shallow;
 };
 
 static int git_fetch_config(const char *k, const char *v,
@@ -175,6 +176,11 @@ static int git_fetch_config(const char *k, const char *v,
 		return 0;
 	}
 
+	if (!strcmp(k, "fetch.shallow")) {
+		fetch_config->shallow = git_config_bool(k, v);
+		return 0;
+	}
+
 	if (!strcmp(k, "fetch.followremotehead")) {
 		if (!v)
 			return config_error_nonbool(k);
@@ -1958,15 +1964,19 @@ static int do_fetch(struct transport *transport,
 		refspec_ref_prefixes(rs, &transport_ls_refs_options.ref_prefixes);
 	} else {
 		struct branch *branch = branch_get(NULL);
+		int tracks_this_remote = branch && branch_has_merge_config(branch) &&
+			!strcmp(branch->remote_name, transport->remote->name);
+		int narrow_to_tracked_ref = config->shallow &&
+			is_repository_shallow(the_repository) && tracks_this_remote;
 
 		if (transport->remote->fetch.nr) {
-			refspec_ref_prefixes(&transport->remote->fetch,
-					     &transport_ls_refs_options.ref_prefixes);
+			if (!narrow_to_tracked_ref)
+				refspec_ref_prefixes(&transport->remote->fetch,
+						     &transport_ls_refs_options.ref_prefixes);
 			if (follow_remote_head != FOLLOW_REMOTE_NEVER)
 				do_set_head = 1;
 		}
-		if (branch && branch_has_merge_config(branch) &&
-		    !strcmp(branch->remote_name, transport->remote->name)) {
+		if (tracks_this_remote) {
 			int i;
 			for (i = 0; i < branch->merge_nr; i++) {
 				strvec_push(&transport_ls_refs_options.ref_prefixes,
diff --git a/t/t5537-fetch-shallow.sh b/t/t5537-fetch-shallow.sh
index f323ceebd2..8143af9fc3 100755
--- a/t/t5537-fetch-shallow.sh
+++ b/t/t5537-fetch-shallow.sh
@@ -13,6 +13,24 @@ commit() {
 	git commit -m "$1"
 }
 
+check_upstream_refs () {
+	git for-each-ref --format="%(refname)" refs/remotes/upstream/ >actual &&
+	cat >expect &&
+	test_cmp expect actual
+}
+
+check_upstream_head () {
+	git symbolic-ref refs/remotes/upstream/HEAD >actual &&
+	echo "refs/remotes/upstream/$1" >expect &&
+	test_cmp expect actual
+}
+
+check_same_tip () {
+	git log --oneline -1 "$1" >expect &&
+	git -C "$2" log --oneline -1 "$3" >actual &&
+	test_cmp expect actual
+}
+
 test_expect_success 'setup' '
 	commit 1 &&
 	commit 2 &&
@@ -261,6 +279,134 @@ test_expect_success 'fetch --deepen does not truncate' '
 	test_cmp expect actual
 '
 
+test_expect_success 'fetch.shallow setup' '
+	git branch narrow-side &&
+	git clone --no-local --depth=1 --branch main --single-branch \
+		.git narrow-default &&
+	git clone --no-local --depth=1 --branch main --single-branch \
+		.git narrow-enabled &&
+	(
+		cd narrow-default &&
+		git remote add upstream ../.git &&
+		git fetch --depth=1 upstream main:refs/remotes/upstream/main &&
+		git branch --set-upstream-to=upstream/main main
+	) &&
+	(
+		cd narrow-enabled &&
+		git remote add upstream ../.git &&
+		git fetch --depth=1 upstream main:refs/remotes/upstream/main &&
+		git branch --set-upstream-to=upstream/main main &&
+		git config fetch.shallow true
+	)
+'
+
+test_expect_success 'a refspec-less fetch expands to the configured refspec by default' '
+	(
+		cd narrow-default &&
+		git fetch upstream &&
+		check_upstream_refs <<-\EOF
+		refs/remotes/upstream/HEAD
+		refs/remotes/upstream/main
+		refs/remotes/upstream/narrow-side
+		EOF
+	)
+'
+
+test_expect_success 'fetch.shallow=true limits a refspec-less fetch to the tracked branch' '
+	(
+		cd narrow-enabled &&
+		git fetch upstream &&
+		check_upstream_refs <<-\EOF
+		refs/remotes/upstream/HEAD
+		refs/remotes/upstream/main
+		EOF
+	)
+'
+
+test_expect_success 'fetch.shallow=true still creates refs/remotes/<remote>/HEAD' '
+	(
+		cd narrow-enabled &&
+		git symbolic-ref -d refs/remotes/upstream/HEAD &&
+		git fetch upstream &&
+		check_upstream_head main
+	)
+'
+
+test_expect_success 'fetch.shallow=true with followRemoteHEAD=always corrects a stale HEAD' '
+	test_when_finished \
+		"git -C narrow-enabled update-ref -d refs/remotes/upstream/stale-branch" &&
+	(
+		cd narrow-enabled &&
+		git update-ref refs/remotes/upstream/stale-branch refs/remotes/upstream/main &&
+		git symbolic-ref refs/remotes/upstream/HEAD refs/remotes/upstream/stale-branch &&
+		git -c fetch.followRemoteHEAD=always fetch upstream &&
+		check_upstream_head main
+	)
+'
+
+test_expect_success 'fetch.shallow=true still updates the tracked branch' '
+	commit 5 &&
+	git -C narrow-enabled fetch upstream &&
+	check_same_tip main narrow-enabled refs/remotes/upstream/main
+'
+
+test_expect_success 'fetch.shallow=true keeps git pull narrowed too' '
+	test_when_finished "git branch -D narrow-side" &&
+	commit 6 &&
+	(
+		cd narrow-enabled &&
+		git pull &&
+		check_upstream_refs <<-\EOF
+		refs/remotes/upstream/HEAD
+		refs/remotes/upstream/main
+		EOF
+	) &&
+	check_same_tip main narrow-enabled HEAD
+'
+
+test_expect_success 'fetch.shallow=true has no effect on a non-shallow repository' '
+	git clone --no-local --branch main --single-branch .git narrow-full &&
+	(
+		cd narrow-full &&
+		git rev-parse --is-shallow-repository >actual &&
+		echo false >expect &&
+		test_cmp expect actual &&
+		git remote add upstream ../.git &&
+		git fetch upstream &&
+		git branch --set-upstream-to=upstream/main main &&
+		git config fetch.shallow true
+	) &&
+	test_when_finished "git branch -D narrow-full-side" &&
+	git branch narrow-full-side &&
+	(
+		cd narrow-full &&
+		git fetch upstream &&
+		check_upstream_refs <<-\EOF
+		refs/remotes/upstream/HEAD
+		refs/remotes/upstream/main
+		refs/remotes/upstream/narrow-full-side
+		EOF
+	)
+'
+
+test_expect_success 'fetch.shallow=true only narrows a fetch of the tracked remote' '
+	test_when_finished "git branch -D other-side" &&
+	git branch other-side &&
+	git clone --no-local --depth=1 --branch main --single-branch \
+		.git narrow-other-remote &&
+	(
+		cd narrow-other-remote &&
+		git remote add upstream ../.git &&
+		git config fetch.shallow true &&
+		git fetch upstream &&
+		check_upstream_refs <<-\EOF
+		refs/remotes/upstream/HEAD
+		refs/remotes/upstream/main
+		refs/remotes/upstream/other-side
+		EOF
+	)
+'
+
 . "$TEST_DIRECTORY"/lib-httpd.sh
 start_httpd
 

base-commit: d38352cd43ab9745686d697872408bc3249a153f
-- 
gitgitgadget