[PATCH v3 20/28] receive-pack: allow pushing with new shallow roots

["Nguyễn Thái Ngọc Duy" <pclouds@gmail.com>]

This is more expensive than the current mode and potentially
invalidates caches like pack bitmaps. It's only enabled if
receive.shallowupdate is on.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 Documentation/config.txt |   4 ++
 builtin/receive-pack.c   | 111 ++++++++++++++++++++++++++++++++++++++++++++---
 t/t5537-push-shallow.sh  |  16 +++++++
 3 files changed, 125 insertions(+), 6 deletions(-)

diff --git a/Documentation/config.txt b/Documentation/config.txt
index ab26963..1a0bd0d 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -2026,6 +2026,10 @@ receive.updateserverinfo::
 	If set to true, git-receive-pack will run git-update-server-info
 	after receiving data from git-push and updating refs.
 
+receive.shallowupdate::
+	If set to true, .git/shallow can be updated when new refs
+	require new shallow roots. Otherwise those refs are rejected.
+
 remote.pushdefault::
 	The remote to push to by default.  Overrides
 	`branch.<name>.remote` for all branches, and is overridden by
diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c
index 254feff..366ecde 100644
--- a/builtin/receive-pack.c
+++ b/builtin/receive-pack.c
@@ -43,7 +43,7 @@ static int fix_thin = 1;
 static const char *head_name;
 static void *head_name_to_free;
 static int sent_capabilities;
-static int shallow_push;
+static int shallow_push, shallow_update;
 static const char* alternate_shallow_file;
 static struct extra_have_objects shallow;
 
@@ -124,6 +124,11 @@ static int receive_pack_config(const char *var, const char *value, void *cb)
 		return 0;
 	}
 
+	if (strcmp(var, "receive.shallowupdate") == 0) {
+		shallow_update = git_config_bool(var, value);
+		return 0;
+	}
+
 	return git_default_config(var, value, cb);
 }
 
@@ -191,6 +196,7 @@ struct command {
 	struct command *next;
 	const char *error_string;
 	unsigned int skip_update:1,
+		     checked_connectivity:1,
 		     did_not_exist:1;
 	int index;
 	unsigned char old_sha1[20];
@@ -424,7 +430,44 @@ static void refuse_unconfigured_deny_delete_current(void)
 		rp_error("%s", refuse_unconfigured_deny_delete_current_msg[i]);
 }
 
-static const char *update(struct command *cmd)
+static int command_singleton_iterator(void *cb_data, unsigned char sha1[20]);
+static int update_ref_shallow(struct command *cmd, uint32_t **used_shallow)
+{
+	static struct lock_file shallow_lock;
+	struct extra_have_objects extra;
+	const char *alt_file;
+	uint32_t mask = 1 << (cmd->index % 32);
+	int i;
+
+	memset(&extra, 0, sizeof(extra));
+	for (i = 0; i < shallow.nr; i++)
+		if (used_shallow[i] &&
+		    used_shallow[i][cmd->index / 32] & mask)
+			add_extra_have(&extra, shallow.array[i]);
+
+	setup_alternate_shallow(&shallow_lock, &alt_file, &extra);
+	if (check_shallow_connected(command_singleton_iterator,
+				     0, cmd, alt_file)) {
+		rollback_lock_file(&shallow_lock);
+		free(extra.array);
+		return -1;
+	}
+
+	commit_lock_file(&shallow_lock);
+
+	/*
+	 * Make sure setup_alternate_shallow() for the next ref does
+	 * not lose these new roots..
+	 */
+	for (i = 0; i < extra.nr; i++)
+		register_shallow(extra.array[i]);
+
+	cmd->checked_connectivity = 1;
+	free(extra.array);
+	return 0;
+}
+
+static const char *update(struct command *cmd, uint32_t **used_shallow)
 {
 	const char *name = cmd->ref_name;
 	struct strbuf namespaced_name_buf = STRBUF_INIT;
@@ -532,6 +575,10 @@ static const char *update(struct command *cmd)
 		return NULL; /* good */
 	}
 	else {
+		if (shallow_push && !cmd->checked_connectivity &&
+		    update_ref_shallow(cmd, used_shallow))
+			return "shallow error";
+
 		lock = lock_any_ref_for_update(namespaced_name, old_sha1,
 					       0, NULL);
 		if (!lock) {
@@ -678,6 +725,8 @@ static void set_connectivity_errors(struct command *commands)
 
 	for (cmd = commands; cmd; cmd = cmd->next) {
 		struct command *singleton = cmd;
+		if (shallow_push && !cmd->checked_connectivity)
+			continue;
 		if (!check_everything_connected(command_singleton_iterator,
 						0, &singleton))
 			continue;
@@ -691,7 +740,8 @@ static int iterate_receive_command_list(void *cb_data, unsigned char sha1[20])
 	struct command *cmd = *cmd_list;
 
 	while (cmd) {
-		if (!is_null_sha1(cmd->new_sha1) && !cmd->skip_update) {
+		if (!is_null_sha1(cmd->new_sha1) && !cmd->skip_update &&
+		    (!shallow_push || cmd->checked_connectivity)) {
 			hashcpy(sha1, cmd->new_sha1);
 			*cmd_list = cmd->next;
 			return 0;
@@ -733,9 +783,10 @@ static void filter_shallow_refs(struct command *commands,
 
 static void execute_commands(struct command *commands, const char *unpacker_error)
 {
-	int *ref_status = NULL;
+	int *ref_status = NULL, checked_connectivity;
 	struct extra_have_objects ref;
 	struct command *cmd;
+	uint32_t **used_shallow = NULL;
 	unsigned char sha1[20];
 
 	if (unpacker_error) {
@@ -754,8 +805,39 @@ static void execute_commands(struct command *commands, const char *unpacker_erro
 			}
 
 		ref_status = xmalloc(sizeof(*ref_status) * ref.nr);
-		if (mark_new_shallow_refs(&ref, ref_status, NULL, &shallow))
+		if (shallow_update)
+			used_shallow = xmalloc(sizeof(*used_shallow) * shallow.nr);
+		if (!mark_new_shallow_refs(&ref, ref_status, used_shallow,
+					   &shallow))
+			shallow_push = 0;
+		else if (!shallow_update) {
 			filter_shallow_refs(commands, ref_status, ref.nr);
+			shallow_push = 0;
+		}
+	}
+
+	/*
+	 * If shallow_push is still on at this point, we know some of
+	 * the new refs may require extra shallow roots. But we don't
+	 * know yet which ref will be accepted because hooks may
+	 * reject some of them. So we can't add all new shallow roots
+	 * in. Go through ref by ref and only add relevant shallow
+	 * roots right before write_ref_sha1.
+	 */
+	if (shallow_push) {
+		int i;
+		/* keep hooks happy */
+		setenv(GIT_SHALLOW_FILE_ENVIRONMENT, alternate_shallow_file, 1);
+		for (i = 0, cmd = commands; i < ref.nr; i++, cmd = cmd->next) {
+			/*
+			 * marker for iterate_receive_command_list.
+			 * All safe refs are run through the next
+			 * check_everything_connected() the rest one
+			 * by one in update()
+			 */
+			cmd->checked_connectivity = !ref_status[i];
+			cmd->index = i;
+		}
 	}
 
 	cmd = commands;
@@ -778,6 +860,7 @@ static void execute_commands(struct command *commands, const char *unpacker_erro
 	free(head_name_to_free);
 	head_name = head_name_to_free = resolve_refdup("HEAD", sha1, 0, NULL);
 
+	checked_connectivity = 1;
 	for (cmd = commands; cmd; cmd = cmd->next) {
 		if (cmd->error_string)
 			continue;
@@ -785,10 +868,26 @@ static void execute_commands(struct command *commands, const char *unpacker_erro
 		if (cmd->skip_update)
 			continue;
 
-		cmd->error_string = update(cmd);
+		cmd->error_string = update(cmd, used_shallow);
+		if (shallow_push && !cmd->error_string &&
+		    !cmd->checked_connectivity) {
+			error("BUG: connectivity check has not been run on ref %s",
+			      cmd->ref_name);
+			checked_connectivity = 0;
+		}
+	}
+
+	if (shallow_push) {
+		if (!checked_connectivity)
+			error("BUG: run 'git fsck' for safety.\n"
+			      "If there are errors, try to remove "
+			      "the reported refs above");
+		if (*alternate_shallow_file)
+			unlink(alternate_shallow_file);
 	}
 	free(ref.array);
 	free(ref_status);
+	free(used_shallow);
 }
 
 static struct command *read_head_info(void)
diff --git a/t/t5537-push-shallow.sh b/t/t5537-push-shallow.sh
index 650c31a..0084a31 100755
--- a/t/t5537-push-shallow.sh
+++ b/t/t5537-push-shallow.sh
@@ -67,4 +67,20 @@ test_expect_success 'push from shallow clone, with grafted roots' '
 	git fsck
 '
 
+test_expect_success 'add new shallow root with receive.updateshallow on' '
+	git config receive.shallowupdate true &&
+	(
+	cd shallow2 &&
+	GIT_TRACE=2 git push ../.git +master:refs/remotes/shallow2/master
+	) &&
+	git log --format=%s shallow2/master >actual &&
+	git fsck &&
+	cat <<EOF >expect &&
+c
+b
+EOF
+	test_cmp expect actual &&
+	git config receive.shallowupdate false
+'
+
 test_done
-- 
1.8.2.83.gc99314b