[PATCH v9 0/5] fetch: avoid fetching every branch of a new remote in a shallow repo

["Harald Nordgren via GitGitGadget" <gitgitgadget@gmail.com>]

Avoid fetching every branch of a new remote in a shallow repo.

Changes in v9:

 * Moved the do_fetch() if/else-if/else split out of the preliminary
   helper-extraction commit and into the refmap-inference commit that needs
   it. Preliminary commit is now a simpler refactor.

Changes in v8:

 * Split into a preliminary collect_upstream_from_remote() helper commit
   plus a simplified refmap-inference commit.
 * Fixed a regression that could drop the current branch's @{upstream} when
   remote.<name>.fetch was narrowed.
 * Threaded struct repository * through instead of hardcoding
   the_repository.
 * Minor naming and comment cleanups.

Changes in v7:

 * Fixed fatal: --refmap option is only meaningful with command-line
   refspec(s) firing just from configuring remote.<name>.refmap alone, with
   nothing else to fetch.
 * When both remote.<name>.fetch and remote.<name>.refmap are configured, a
   refspec-less fetch follows .fetch as usual, .refmap only remaps something
   already being fetched by name. Added tests.

Changes in v6:

 * Remove leftover reference to deleted default-branch logic in commit
   message.

Changes in v5:

 * Renumber test file from t5585 to t5586.

Changes in v4:

 * Removed the automatic default-branch fetch. A fresh remote fetches
   nothing until you track a branch explicitly.
 * Fixed fetch report showing "new ref HEAD" instead of "new branch "
 * Reworded remote..refmap docs and commit message.

Changes in v3:

 * Replace the special ":"/"+:" fetch refspec with remote.<name>.refmap,
   reusing git's existing --refmap mechanism instead of inventing new
   refspec syntax.
 * Split the change into 4 commits.

Changes in v2:

 * Replaced the opt-in fetch.shallow config entirely with a new special
   fetch refspec (+:) that git remote add now defaults new remotes to in a
   shallow repository. The new refspec fetches whichever branches any local
   branch tracks at that remote, plus the remote's default branch.

Harald Nordgren (5):
  fetch: add remote.<name>.refmap
  fetch: extract collect_upstream_from_remote() helper
  fetch: infer branches to fetch from a refmap-only remote
  remote: add "git remote add --limited-fetch"
  remote: default to --limited-fetch in a shallow repository

 Documentation/config/remote.adoc |   8 +++
 Documentation/fetch-options.adoc |   5 ++
 Documentation/git-remote.adoc    |  14 +++-
 builtin/fetch.c                  |  84 ++++++++++++++++++----
 builtin/remote.c                 |  31 ++++++--
 remote.c                         |  51 ++++++++++++-
 remote.h                         |  19 +++++
 t/meson.build                    |   1 +
 t/t5505-remote.sh                |  76 ++++++++++++++++++++
 t/t5510-fetch.sh                 |  63 ++++++++++++++++
 t/t5586-fetch-refmap.sh          | 119 +++++++++++++++++++++++++++++++
 11 files changed, 448 insertions(+), 23 deletions(-)
 create mode 100755 t/t5586-fetch-refmap.sh


base-commit: 6de20f6092dcf9bdb1c8efe03db4b70c82b423dd
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-2412%2FHaraldNordgren%2Ffetch-shallow-narrow-refspec-v9
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-2412/HaraldNordgren/fetch-shallow-narrow-refspec-v9
Pull-Request: https://github.com/git/git/pull/2412

Range-diff vs v8:

 1:  2a48c21bc8 = 1:  2a48c21bc8 fetch: add remote.<name>.refmap
 2:  08fea07a8b ! 2:  50382bce7e fetch: extract collect_upstream_from_remote() helper
     @@ Commit message
      
       ## builtin/fetch.c ##
      @@ builtin/fetch.c: static int do_fetch(struct transport *transport,
     - 
       	if (rs->nr) {
       		refspec_ref_prefixes(rs, &transport_ls_refs_options.ref_prefixes);
     -+	} else if (transport->remote->fetch.nr) {
     -+		struct string_list tracked = STRING_LIST_INIT_DUP;
     -+		struct string_list_item *item;
     -+
     -+		refspec_ref_prefixes(&transport->remote->fetch,
     -+				     &transport_ls_refs_options.ref_prefixes);
     -+		if (follow_remote_head != FOLLOW_REMOTE_NEVER)
     -+			do_set_head = 1;
     -+
     -+		/*
     -+		 * The configured refspec may not cover the current
     -+		 * branch's upstream (e.g. a narrowed -t refspec), so
     -+		 * make sure we can still fetch it regardless.
     -+		 */
     -+		collect_upstream_from_remote(the_repository, &tracked,
     -+					      transport->remote, NULL);
     -+		for_each_string_list_item(item, &tracked)
     -+			strvec_push(&transport_ls_refs_options.ref_prefixes,
     -+				    item->string);
     -+		string_list_clear(&tracked, 0);
       	} else {
      -		struct branch *branch = branch_get(NULL);
      +		struct string_list tracked = STRING_LIST_INIT_DUP;
      +		struct string_list_item *item;
       
     --		if (transport->remote->fetch.nr) {
     --			refspec_ref_prefixes(&transport->remote->fetch,
     --					     &transport_ls_refs_options.ref_prefixes);
     --			if (follow_remote_head != FOLLOW_REMOTE_NEVER)
     --				do_set_head = 1;
     --		}
     + 		if (transport->remote->fetch.nr) {
     + 			refspec_ref_prefixes(&transport->remote->fetch,
     +@@ builtin/fetch.c: static int do_fetch(struct transport *transport,
     + 			if (follow_remote_head != FOLLOW_REMOTE_NEVER)
     + 				do_set_head = 1;
     + 		}
      -		if (branch && branch_has_merge_config(branch) &&
      -		    !strcmp(branch->remote_name, transport->remote->name)) {
      -			int i;
     @@ builtin/fetch.c: static int do_fetch(struct transport *transport,
      -					    branch->merge[i]->src);
      -			}
      -		}
     ++
     ++		/*
     ++		 * The configured refspec may not cover the current
     ++		 * branch's upstream (e.g. a narrowed -t refspec), so
     ++		 * make sure we can still fetch it regardless.
     ++		 */
      +		collect_upstream_from_remote(the_repository, &tracked,
      +					      transport->remote, NULL);
      +		for_each_string_list_item(item, &tracked)
 3:  a39316d233 ! 3:  9ad8ad3eb4 fetch: infer branches to fetch from a refmap-only remote
     @@ builtin/fetch.c: static struct ref *get_ref_map(struct remote *remote,
       }
       
      @@ builtin/fetch.c: static int do_fetch(struct transport *transport,
     + 
     + 	if (rs->nr) {
     + 		refspec_ref_prefixes(rs, &transport_ls_refs_options.ref_prefixes);
     +-	} else {
     ++	} else if (transport->remote->fetch.nr) {
     + 		struct string_list tracked = STRING_LIST_INIT_DUP;
     + 		struct string_list_item *item;
     + 
     +-		if (transport->remote->fetch.nr) {
     +-			refspec_ref_prefixes(&transport->remote->fetch,
     +-					     &transport_ls_refs_options.ref_prefixes);
     +-			if (follow_remote_head != FOLLOW_REMOTE_NEVER)
     +-				do_set_head = 1;
     +-		}
     ++		refspec_ref_prefixes(&transport->remote->fetch,
     ++				     &transport_ls_refs_options.ref_prefixes);
     ++		if (follow_remote_head != FOLLOW_REMOTE_NEVER)
     ++			do_set_head = 1;
     + 
     + 		/*
     + 		 * The configured refspec may not cover the current
     +@@ builtin/fetch.c: static int do_fetch(struct transport *transport,
     + 			strvec_push(&transport_ls_refs_options.ref_prefixes,
       				    item->string);
       		string_list_clear(&tracked, 0);
     - 	} else {
     ++	} else {
      +		/*
      +		 * The --refmap command line option, if given, takes
      +		 * precedence over remote.<name>.refmap.
      +		 */
      +		struct refspec *effective_refmap = refmap.nr ? &refmap :
      +			&transport->remote->refmap;
     - 		struct string_list tracked = STRING_LIST_INIT_DUP;
     - 		struct string_list_item *item;
     - 
     --		collect_upstream_from_remote(the_repository, &tracked,
     --					      transport->remote, NULL);
     ++		struct string_list tracked = STRING_LIST_INIT_DUP;
     ++		struct string_list_item *item;
     ++
      +		if (effective_refmap->nr) {
      +			branches_tracking_remote(the_repository,
      +						  transport->remote, &tracked);
     @@ builtin/fetch.c: static int do_fetch(struct transport *transport,
      +			collect_upstream_from_remote(the_repository, &tracked,
      +						      transport->remote, NULL);
      +		}
     - 		for_each_string_list_item(item, &tracked)
     - 			strvec_push(&transport_ls_refs_options.ref_prefixes,
     - 				    item->string);
     ++		for_each_string_list_item(item, &tracked)
     ++			strvec_push(&transport_ls_refs_options.ref_prefixes,
     ++				    item->string);
     ++		string_list_clear(&tracked, 0);
     + 
     + 		/*
     + 		 * If there are no refs specified to fetch, then we just
      
       ## remote.c ##
      @@ remote.c: void collect_upstream_from_remote(struct repository *repo,
 4:  2b974bbe0a = 4:  75169f668e remote: add "git remote add --limited-fetch"
 5:  db27c29086 = 5:  9ee090d3d0 remote: default to --limited-fetch in a shallow repository

-- 
gitgitgadget