[PATCH 2/3] worktree: add --recurse-submodules flag to worktree add

[PATCH 2/3] worktree: add --recurse-submodules flag to worktree add

[Jimmy Aguilar Mena]

When "git submodule update --init" is run inside a linked worktree,
submodule_name_to_gitdir() returns a per-worktree path:

   $GIT_COMMON_DIR/worktrees/<id>/modules/<name>

rather than the main worktree's:

   $GIT_COMMON_DIR/modules/<name>

If the main worktree already has the submodule cloned, the per-worktree
gitdir does not yet exist and clone_submodule() falls through to fetch
from the remote URL -- wasting bandwidth and disk space when all the
objects are already present locally.

Detect this case: when the target sm_gitdir does not exist, differs
from the common-dir path, and the common-dir path is a valid git
directory, skip the remote clone and instead run:

   git clone --local --no-checkout --separate-git-dir <sm_gitdir> \
             <common_sm_gitdir> <working_tree_path>

"--local" makes Git use hardlinks for pack files and loose objects, so
no extra disk space is consumed for data that is already present.  The
per-worktree gitdir gets its own HEAD, index, and config, just like any
other worktree, while sharing the object store with the main worktree's
submodule repository.  Mutable per-worktree files (HEAD, refs, index,
config) start out as hardlinks but become independent on the first write
via Git's atomic rename(2) approach.

Signed-off-by: Jimmy Aguilar Mena <kratsbinovish@gmail.com>
---
  builtin/submodule--helper.c | 54 +++++++++++++++++++++++++++++++++++++
  1 file changed, 54 insertions(+)

diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index 2f589e3b37..2da59e8c93 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -1914,6 +1914,59 @@ static int clone_submodule(const struct module_clone_data *clone_data,
  		clone_data_path = to_free = xstrfmt("%s/%s", repo_get_work_tree(the_repository),
  						    clone_data->path);
  
+	/*
+	 * If we are populating a submodule in a linked worktree and the main
+	 * worktree already has this submodule cloned, reuse its objects via a
+	 * local clone (hardlinks) instead of fetching from the network.  The
+	 * common-dir path "$GIT_COMMON_DIR/modules/<name>" is where the main
+	 * worktree stores the submodule gitdir; the per-worktree path returned
+	 * by submodule_name_to_gitdir() diverges from it only in linked
+	 * worktrees.
+	 */
+	{
+		struct strbuf common_sm_gitdir = STRBUF_INIT;
+		strbuf_addf(&common_sm_gitdir, "%s/modules/%s",
+			    the_repository->commondir, clone_data->name);
+
+		if (!file_exists(sm_gitdir) &&
+		    strcmp(sm_gitdir, common_sm_gitdir.buf) &&
+		    is_git_directory(common_sm_gitdir.buf)) {
+			/*
+			 * Main worktree has the submodule; reuse it locally.
+			 * git clone --local creates hardlinks for pack files so
+			 * no extra disk space is needed for existing objects.
+			 */
+			if (clone_data->require_init && !stat(clone_data_path, &st) &&
+			    !is_empty_dir(clone_data_path))
+				die(_("directory not empty: '%s'"), clone_data_path);
+
+			if (safe_create_leading_directories_const(the_repository, sm_gitdir) < 0)
+				die(_("could not create directory '%s'"), sm_gitdir);
+
+			strvec_push(&cp.args, "clone");
+			strvec_push(&cp.args, "--local");
+			strvec_push(&cp.args, "--no-checkout");
+			if (clone_data->quiet)
+				strvec_push(&cp.args, "--quiet");
+			strvec_pushl(&cp.args, "--separate-git-dir", sm_gitdir, NULL);
+			strvec_push(&cp.args, "--");
+			strvec_push(&cp.args, common_sm_gitdir.buf);
+			strvec_push(&cp.args, clone_data_path);
+
+			cp.git_cmd = 1;
+			prepare_submodule_repo_env(&cp.env);
+			cp.no_stdin = 1;
+
+			if (run_command(&cp))
+				die(_("local clone of '%s' into submodule path '%s' failed"),
+				    common_sm_gitdir.buf, clone_data_path);
+
+			strbuf_release(&common_sm_gitdir);
+			goto connect_wt;
+		}
+		strbuf_release(&common_sm_gitdir);
+	}
+
  	if (!file_exists(sm_gitdir)) {
  		if (clone_data->require_init && !stat(clone_data_path, &st) &&
  		    !is_empty_dir(clone_data_path))
@@ -2005,6 +2058,7 @@ static int clone_submodule(const struct module_clone_data *clone_data,
  		    sm_gitdir);
  	}
  
+connect_wt:
  	connect_work_tree_and_git_dir(clone_data_path, sm_gitdir, 0);
  
  	p = repo_submodule_path(the_repository, clone_data_path, "config");

Re: [PATCH 2/3] worktree: add --recurse-submodules flag to worktree add

[Junio C Hamano]

Jimmy Aguilar Mena <kratsbinovish@gmail.com> writes:

> +	/*
> +	 * If we are populating a submodule in a linked worktree and the main
> +	 * worktree already has this submodule cloned, reuse its objects via a
> +	 * local clone (hardlinks) instead of fetching from the network.  The
> +	 * common-dir path "$GIT_COMMON_DIR/modules/<name>" is where the main
> +	 * worktree stores the submodule gitdir; the per-worktree path returned
> +	 * by submodule_name_to_gitdir() diverges from it only in linked
> +	 * worktrees.
> +	 */
> +	{

I think you identified the right place to hook into.  But instead of
"clone --local", shoudln't you be running "git worktree add" here?

"clone --local" means you have two logically separate repositories
for this single submodule that are used by two worktrees of the
superproject.  If you add a new commit to one, shouldn't that commit
become available in the other?  Two separate repositories created by
"clone --local" will not allow you to do so.

> +		struct strbuf common_sm_gitdir = STRBUF_INIT;
> +		strbuf_addf(&common_sm_gitdir, "%s/modules/%s",
> +			    the_repository->commondir, clone_data->name);
> +
> +		if (!file_exists(sm_gitdir) &&
> +		    strcmp(sm_gitdir, common_sm_gitdir.buf) &&
> +		    is_git_directory(common_sm_gitdir.buf)) {
> +			/*
> +			 * Main worktree has the submodule; reuse it locally.
> +			 * git clone --local creates hardlinks for pack files so
> +			 * no extra disk space is needed for existing objects.
> +			 */
> +			if (clone_data->require_init && !stat(clone_data_path, &st) &&
> +			    !is_empty_dir(clone_data_path))
> +				die(_("directory not empty: '%s'"), clone_data_path);
> +
> +			if (safe_create_leading_directories_const(the_repository, sm_gitdir) < 0)
> +				die(_("could not create directory '%s'"), sm_gitdir);
> +
> +			strvec_push(&cp.args, "clone");
> +			strvec_push(&cp.args, "--local");
> +			strvec_push(&cp.args, "--no-checkout");
> +			if (clone_data->quiet)
> +				strvec_push(&cp.args, "--quiet");
> +			strvec_pushl(&cp.args, "--separate-git-dir", sm_gitdir, NULL);
> +			strvec_push(&cp.args, "--");
> +			strvec_push(&cp.args, common_sm_gitdir.buf);
> +			strvec_push(&cp.args, clone_data_path);
> +
> +			cp.git_cmd = 1;
> +			prepare_submodule_repo_env(&cp.env);
> +			cp.no_stdin = 1;
> +
> +			if (run_command(&cp))
> +				die(_("local clone of '%s' into submodule path '%s' failed"),
> +				    common_sm_gitdir.buf, clone_data_path);
> +
> +			strbuf_release(&common_sm_gitdir);
> +			goto connect_wt;
> +		}
> +		strbuf_release(&common_sm_gitdir);
> +	}
> +
>   	if (!file_exists(sm_gitdir)) {
>   		if (clone_data->require_init && !stat(clone_data_path, &st) &&
>   		    !is_empty_dir(clone_data_path))
> @@ -2005,6 +2058,7 @@ static int clone_submodule(const struct module_clone_data *clone_data,
>   		    sm_gitdir);
>   	}
>   
> +connect_wt:
>   	connect_work_tree_and_git_dir(clone_data_path, sm_gitdir, 0);
>   
>   	p = repo_submodule_path(the_repository, clone_data_path, "config");