[PATCH v8 0/5] fetch: avoid fetching every branch of a new remote in a shallow repo

["Harald Nordgren via GitGitGadget" <gitgitgadget@gmail.com>]

Avoid fetching every branch of a new remote in a shallow repo.

Changes in v8:

 * Split into a preliminary collect_upstream_from_remote() helper commit
   plus a simplified refmap-inference commit.
 * Fixed a regression that could drop the current branch's @{upstream} when
   remote.<name>.fetch was narrowed.
 * Threaded struct repository * through instead of hardcoding
   the_repository.
 * Minor naming and comment cleanups.

Changes in v7:

 * Fixed fatal: --refmap option is only meaningful with command-line
   refspec(s) firing just from configuring remote.<name>.refmap alone, with
   nothing else to fetch.
 * When both remote.<name>.fetch and remote.<name>.refmap are configured, a
   refspec-less fetch follows .fetch as usual, .refmap only remaps something
   already being fetched by name. Added tests.

Changes in v6:

 * Remove leftover reference to deleted default-branch logic in commit
   message.

Changes in v5:

 * Renumber test file from t5585 to t5586.

Changes in v4:

 * Removed the automatic default-branch fetch. A fresh remote fetches
   nothing until you track a branch explicitly.
 * Fixed fetch report showing "new ref HEAD" instead of "new branch "
 * Reworded remote..refmap docs and commit message.

Changes in v3:

 * Replace the special ":"/"+:" fetch refspec with remote.<name>.refmap,
   reusing git's existing --refmap mechanism instead of inventing new
   refspec syntax.
 * Split the change into 4 commits.

Changes in v2:

 * Replaced the opt-in fetch.shallow config entirely with a new special
   fetch refspec (+:) that git remote add now defaults new remotes to in a
   shallow repository. The new refspec fetches whichever branches any local
   branch tracks at that remote, plus the remote's default branch.

Harald Nordgren (5):
  fetch: add remote.<name>.refmap
  fetch: extract collect_upstream_from_remote() helper
  fetch: infer branches to fetch from a refmap-only remote
  remote: add "git remote add --limited-fetch"
  remote: default to --limited-fetch in a shallow repository

 Documentation/config/remote.adoc |   8 +++
 Documentation/fetch-options.adoc |   5 ++
 Documentation/git-remote.adoc    |  14 +++-
 builtin/fetch.c                  |  84 ++++++++++++++++++----
 builtin/remote.c                 |  31 ++++++--
 remote.c                         |  51 ++++++++++++-
 remote.h                         |  19 +++++
 t/meson.build                    |   1 +
 t/t5505-remote.sh                |  76 ++++++++++++++++++++
 t/t5510-fetch.sh                 |  63 ++++++++++++++++
 t/t5586-fetch-refmap.sh          | 119 +++++++++++++++++++++++++++++++
 11 files changed, 448 insertions(+), 23 deletions(-)
 create mode 100755 t/t5586-fetch-refmap.sh


base-commit: 6de20f6092dcf9bdb1c8efe03db4b70c82b423dd
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-2412%2FHaraldNordgren%2Ffetch-shallow-narrow-refspec-v8
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-2412/HaraldNordgren/fetch-shallow-narrow-refspec-v8
Pull-Request: https://github.com/git/git/pull/2412

Range-diff vs v7:

 1:  b5db64d56f ! 1:  2a48c21bc8 fetch: add remote.<name>.refmap
     @@ builtin/fetch.c: static struct ref *get_ref_map(struct remote *remote,
       	struct ref *rm;
       	struct ref *ref_map = NULL;
       	struct ref **tail = &ref_map;
     ++	/*
     ++	 * The --refmap command line option, if given, takes precedence
     ++	 * over remote.<name>.refmap.
     ++	 */
      +	struct refspec *effective_refmap =
      +		refmap.nr ? &refmap : remote ? &remote->refmap : NULL;
       
 -:  ---------- > 2:  08fea07a8b fetch: extract collect_upstream_from_remote() helper
 2:  fd6864daaf ! 3:  a39316d233 fetch: infer branches to fetch from a refmap-only remote
     @@ Documentation/fetch-options.adoc: endif::git-pull[]
      -`remote.<name>.refmap` provides the default value for this option, the
      -same way `remote.<name>.fetch` provides the default refspecs to fetch.
      +When a refmap is active (from `--refmap` or `remote.<name>.refmap`) but
     -+there is nothing to fetch, neither on the command line nor from
     -+`remote.<name>.fetch`, Git infers what to fetch from the local branches
     -+whose `@{upstream}` is on that remote.
     ++nothing to fetch is specified on the command line, nor is there a
     ++`remote.<name>.fetch`, branches from the remote that are used as the
     ++`@{upstream}` of our local branches are fetched.
       
       `-t`::
       `--tags`::
      
       ## builtin/fetch.c ##
      @@ builtin/fetch.c: static struct ref *get_ref_map(struct remote *remote,
     - 	struct ref **tail = &ref_map;
     + 	 */
       	struct refspec *effective_refmap =
       		refmap.nr ? &refmap : remote ? &remote->refmap : NULL;
      +	struct refspec inferred_rs;
     -+	int inferred_branches = 0;
     ++	int infer_from_refmap = 0;
       
       	/* opportunistically-updated references: */
       	struct ref *orefs = NULL, **oref_tail = &orefs;
     @@ builtin/fetch.c: static struct ref *get_ref_map(struct remote *remote,
      +		struct string_list tracked = STRING_LIST_INIT_DUP;
      +		struct string_list_item *item;
      +
     -+		branches_tracking_remote(remote, &tracked);
     ++		branches_tracking_remote(the_repository, remote, &tracked);
      +		for_each_string_list_item(item, &tracked)
      +			refspec_append(&inferred_rs, item->string);
      +		string_list_clear(&tracked, 0);
      +
      +		rs = &inferred_rs;
     -+		inferred_branches = 1;
     ++		infer_from_refmap = 1;
      +	}
      +
       	if (rs->nr) {
     @@ builtin/fetch.c: static struct ref *get_ref_map(struct remote *remote,
       		for (i = 0; i < rs->nr; i++) {
      -			get_fetch_map(remote_refs, &rs->items[i], &tail, 0);
      +			get_fetch_map(remote_refs, &rs->items[i], &tail,
     -+				      inferred_branches);
     ++				      infer_from_refmap);
       			if (rs->items[i].dst && rs->items[i].dst[0])
       				*autotags = 1;
       		}
     @@ builtin/fetch.c: static struct ref *get_ref_map(struct remote *remote,
       
       		for (i = 0; i < fetch_refspec->nr; i++)
       			get_fetch_map(ref_map, &fetch_refspec->items[i], &oref_tail, 1);
     -+	} else if (inferred_branches) {
     ++	} else if (infer_from_refmap) {
      +		/* Already fully handled above. */
       	} else if (refmap.nr) {
       		die("--refmap option is only meaningful with command-line refspec(s)");
     @@ builtin/fetch.c: static struct ref *get_ref_map(struct remote *remote,
       }
       
      @@ builtin/fetch.c: static int do_fetch(struct transport *transport,
     - 		refspec_ref_prefixes(rs, &transport_ls_refs_options.ref_prefixes);
     + 				    item->string);
     + 		string_list_clear(&tracked, 0);
       	} else {
     - 		struct branch *branch = branch_get(NULL);
     --
     --		if (transport->remote->fetch.nr) {
     -+		int tracks_this_remote = branch && branch_has_merge_config(branch) &&
     -+			!strcmp(branch->remote_name, transport->remote->name);
     ++		/*
     ++		 * The --refmap command line option, if given, takes
     ++		 * precedence over remote.<name>.refmap.
     ++		 */
      +		struct refspec *effective_refmap = refmap.nr ? &refmap :
      +			&transport->remote->refmap;
     -+		int inferred_branches = !transport->remote->fetch.nr &&
     -+			effective_refmap->nr;
     -+
     -+		if (inferred_branches) {
     -+			struct string_list tracked = STRING_LIST_INIT_DUP;
     -+			struct string_list_item *item;
     -+
     -+			branches_tracking_remote(transport->remote, &tracked);
     -+			for_each_string_list_item(item, &tracked)
     -+				strvec_push(&transport_ls_refs_options.ref_prefixes,
     -+					    item->string);
     -+			string_list_clear(&tracked, 0);
     -+		} else if (transport->remote->fetch.nr) {
     - 			refspec_ref_prefixes(&transport->remote->fetch,
     - 					     &transport_ls_refs_options.ref_prefixes);
     --			if (follow_remote_head != FOLLOW_REMOTE_NEVER)
     --				do_set_head = 1;
     - 		}
     --		if (branch && branch_has_merge_config(branch) &&
     --		    !strcmp(branch->remote_name, transport->remote->name)) {
     -+		if ((transport->remote->fetch.nr || inferred_branches) &&
     -+		    follow_remote_head != FOLLOW_REMOTE_NEVER)
     -+			do_set_head = 1;
     -+		if (tracks_this_remote) {
     - 			int i;
     - 			for (i = 0; i < branch->merge_nr; i++) {
     - 				strvec_push(&transport_ls_refs_options.ref_prefixes,
     -@@ builtin/fetch.c: static int do_fetch(struct transport *transport,
     - 
     - 	ref_map = get_ref_map(transport->remote, remote_refs, rs,
     - 			      tags, &autotags);
     -+
     - 	if (!update_head_ok)
     - 		check_not_current_branch(ref_map);
     + 		struct string_list tracked = STRING_LIST_INIT_DUP;
     + 		struct string_list_item *item;
       
     +-		collect_upstream_from_remote(the_repository, &tracked,
     +-					      transport->remote, NULL);
     ++		if (effective_refmap->nr) {
     ++			branches_tracking_remote(the_repository,
     ++						  transport->remote, &tracked);
     ++			if (follow_remote_head != FOLLOW_REMOTE_NEVER)
     ++				do_set_head = 1;
     ++		} else {
     ++			collect_upstream_from_remote(the_repository, &tracked,
     ++						      transport->remote, NULL);
     ++		}
     + 		for_each_string_list_item(item, &tracked)
     + 			strvec_push(&transport_ls_refs_options.ref_prefixes,
     + 				    item->string);
      
       ## remote.c ##
     -@@ remote.c: int branch_merge_matches(struct branch *branch,
     - 	return refname_match(branch->merge[i]->src, refname);
     +@@ remote.c: void collect_upstream_from_remote(struct repository *repo,
     + 		string_list_insert(tracked, branch->merge[i]->src);
       }
       
      +struct branches_tracking_remote_cb_data {
     ++	struct repository *repo;
      +	struct remote *remote;
      +	struct string_list *tracked;
      +};
     @@ remote.c: int branch_merge_matches(struct branch *branch,
      +static int add_if_tracking_remote(const struct reference *ref, void *cb_data)
      +{
      +	struct branches_tracking_remote_cb_data *data = cb_data;
     -+	struct branch *branch;
     -+
     -+	branch = branch_get(ref->name);
     -+	if (!branch_has_merge_config(branch) ||
     -+	    strcmp(branch->remote_name, data->remote->name))
     -+		return 0;
     -+
     -+	for (int i = 0; i < branch->merge_nr; i++)
     -+		string_list_insert(data->tracked, branch->merge[i]->src);
      +
     ++	collect_upstream_from_remote(data->repo, data->tracked, data->remote,
     ++				      ref->name);
      +	return 0;
      +}
      +
     -+void branches_tracking_remote(struct remote *remote, struct string_list *tracked)
     ++void branches_tracking_remote(struct repository *repo, struct remote *remote,
     ++			       struct string_list *tracked)
      +{
     -+	struct branches_tracking_remote_cb_data data = { remote, tracked };
     ++	struct branches_tracking_remote_cb_data data = { repo, remote, tracked };
      +
     -+	refs_for_each_branch_ref(get_main_ref_store(the_repository),
     ++	refs_for_each_branch_ref(get_main_ref_store(repo),
      +				  add_if_tracking_remote, &data);
      +}
      +
     @@ remote.c: int branch_merge_matches(struct branch *branch,
       {
      
       ## remote.h ##
     -@@ remote.h: int branch_has_merge_config(struct branch *branch);
     - 
     - int branch_merge_matches(struct branch *, int n, const char *);
     +@@ remote.h: void collect_upstream_from_remote(struct repository *repo,
     + 				   struct remote *remote,
     + 				   const char *refname);
       
      +/* fills tracked with the refname of every local branch's upstream on remote */
     -+void branches_tracking_remote(struct remote *remote, struct string_list *tracked);
     ++void branches_tracking_remote(struct repository *repo, struct remote *remote,
     ++			       struct string_list *tracked);
      +
       /* list of the remote in a group as configured */
       struct remote_group_data {
 3:  a2208875b6 = 4:  2b974bbe0a remote: add "git remote add --limited-fetch"
 4:  c8fd073de3 = 5:  db27c29086 remote: default to --limited-fetch in a shallow repository

-- 
gitgitgadget