[PATCH v3 17/28] Support pushing from a shallow clone

["Nguyễn Thái Ngọc Duy" <pclouds@gmail.com>]

Pushing from a shallow clone using today's send-pack and receive-pack
may work, if the transferred pack does not end up at any graft
points. If it does, recent receive-pack that does connectivity check
will reject the push. If receive-pack is old and does not have the
connectivity check, the upstream repo becomes corrupt.

The pack protocol is updated and send-pack now sends all shallow
grafts before it sends the commands, if the repo is shallow. This
protocol extension will break current receive-pack, which is intended,
mostly to stop corrupting the upstream repo.

Changes on the receiver are similar to what has been done in
fetch-pack, i.e. filter out refs that require new shallow roots then
go along as usual.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 Documentation/technical/pack-protocol.txt |  4 +-
 builtin/receive-pack.c                    | 68 +++++++++++++++++++++++++++++-
 builtin/send-pack.c                       |  2 +-
 send-pack.c                               |  3 ++
 t/t5537-push-shallow.sh (new +x)          | 70 +++++++++++++++++++++++++++++++
 5 files changed, 143 insertions(+), 4 deletions(-)
 create mode 100755 t/t5537-push-shallow.sh

diff --git a/Documentation/technical/pack-protocol.txt b/Documentation/technical/pack-protocol.txt
index eb8edd1..c73b62f 100644
--- a/Documentation/technical/pack-protocol.txt
+++ b/Documentation/technical/pack-protocol.txt
@@ -464,7 +464,9 @@ contain all the objects that the server will need to complete the new
 references.
 
 ----
-  update-request    =  command-list [pack-file]
+  update-request    =  *shallow command-list [pack-file]
+
+  shallow           =  PKT-LINE("shallow" SP obj-id)
 
   command-list      =  PKT-LINE(command NUL capability-list LF)
 		       *PKT-LINE(command LF)
diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c
index 22e162d..254feff 100644
--- a/builtin/receive-pack.c
+++ b/builtin/receive-pack.c
@@ -43,6 +43,9 @@ static int fix_thin = 1;
 static const char *head_name;
 static void *head_name_to_free;
 static int sent_capabilities;
+static int shallow_push;
+static const char* alternate_shallow_file;
+static struct extra_have_objects shallow;
 
 static enum deny_action parse_deny_action(const char *var, const char *value)
 {
@@ -189,6 +192,7 @@ struct command {
 	const char *error_string;
 	unsigned int skip_update:1,
 		     did_not_exist:1;
+	int index;
 	unsigned char old_sha1[20];
 	unsigned char new_sha1[20];
 	char ref_name[FLEX_ARRAY]; /* more */
@@ -687,7 +691,7 @@ static int iterate_receive_command_list(void *cb_data, unsigned char sha1[20])
 	struct command *cmd = *cmd_list;
 
 	while (cmd) {
-		if (!is_null_sha1(cmd->new_sha1)) {
+		if (!is_null_sha1(cmd->new_sha1) && !cmd->skip_update) {
 			hashcpy(sha1, cmd->new_sha1);
 			*cmd_list = cmd->next;
 			return 0;
@@ -712,8 +716,25 @@ static void reject_updates_to_hidden(struct command *commands)
 	}
 }
 
+static void filter_shallow_refs(struct command *commands,
+				const int *ref_status, int nr_ref)
+{
+	struct command *cmd;
+	for (cmd = commands; cmd; cmd = cmd->next)
+		if (!is_null_sha1(cmd->new_sha1) && ref_status[cmd->index]) {
+			cmd->error_string = "shallow update not allowed";
+			cmd->skip_update = 1;
+		}
+	if (*alternate_shallow_file) {
+		unlink(alternate_shallow_file);
+		alternate_shallow_file = NULL;
+	}
+}
+
 static void execute_commands(struct command *commands, const char *unpacker_error)
 {
+	int *ref_status = NULL;
+	struct extra_have_objects ref;
 	struct command *cmd;
 	unsigned char sha1[20];
 
@@ -723,6 +744,20 @@ static void execute_commands(struct command *commands, const char *unpacker_erro
 		return;
 	}
 
+	memset(&ref, 0, sizeof(ref));
+	if (shallow_push) {
+		int i;
+		for (i = 0, cmd = commands; cmd; i++, cmd = cmd->next)
+			if (!is_null_sha1(cmd->new_sha1)) {
+				add_extra_have(&ref, cmd->new_sha1);
+				cmd->index = i;
+			}
+
+		ref_status = xmalloc(sizeof(*ref_status) * ref.nr);
+		if (mark_new_shallow_refs(&ref, ref_status, NULL, &shallow))
+			filter_shallow_refs(commands, ref_status, ref.nr);
+	}
+
 	cmd = commands;
 	if (check_everything_connected(iterate_receive_command_list,
 				       0, &cmd))
@@ -752,6 +787,8 @@ static void execute_commands(struct command *commands, const char *unpacker_erro
 
 		cmd->error_string = update(cmd);
 	}
+	free(ref.array);
+	free(ref_status);
 }
 
 static struct command *read_head_info(void)
@@ -768,6 +805,15 @@ static struct command *read_head_info(void)
 		line = packet_read_line(0, &len);
 		if (!line)
 			break;
+
+		if (len == 48 && !prefixcmp(line, "shallow ")) {
+			if (get_sha1_hex(line + 8, old_sha1))
+				die("protocol error: expected shallow sha, got '%s'", line + 8);
+			add_extra_have(&shallow, old_sha1);
+			shallow_push = 1;
+			continue;
+		}
+
 		if (len < 83 ||
 		    line[40] != ' ' ||
 		    line[81] != ' ' ||
@@ -822,7 +868,7 @@ static const char *pack_lockfile;
 static const char *unpack(int err_fd)
 {
 	struct pack_header hdr;
-	const char *argv[7];
+	const char *argv[9];
 	const char *hdr_err;
 	int status, ac = 0;
 	char hdr_arg[38];
@@ -843,6 +889,24 @@ static const char *unpack(int err_fd)
 			"--pack_header=%"PRIu32",%"PRIu32,
 			ntohl(hdr.hdr_version), ntohl(hdr.hdr_entries));
 
+	if (shallow_push) {
+		struct extra_have_objects extra;
+		memset(&extra, 0, sizeof(extra));
+		remove_reachable_shallow_points(&extra, &shallow);
+		if (extra.nr) {
+			alternate_shallow_file = setup_temporary_shallow(&extra);
+			free(shallow.array);
+			shallow = extra;
+		} else {
+			alternate_shallow_file = NULL;
+			shallow_push = 0;
+		}
+		if (alternate_shallow_file && *alternate_shallow_file) {
+			argv[ac++] = "--shallow-file";
+			argv[ac++] = alternate_shallow_file;
+		}
+	}
+
 	memset(&child, 0, sizeof(child));
 	if (ntohl(hdr.hdr_entries) < unpack_limit) {
 		argv[ac++] = "unpack-objects";
diff --git a/builtin/send-pack.c b/builtin/send-pack.c
index bfa9253..5db1311 100644
--- a/builtin/send-pack.c
+++ b/builtin/send-pack.c
@@ -207,7 +207,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix)
 	    (send_all && args.send_mirror))
 		usage(send_pack_usage);
 
-	if (is_repository_shallow())
+	if (is_repository_shallow() && args.stateless_rpc)
 		die("attempt to push from a shallow repository");
 
 	if (remote_name) {
diff --git a/send-pack.c b/send-pack.c
index fab62e3..8b5571c 100644
--- a/send-pack.c
+++ b/send-pack.c
@@ -213,6 +213,9 @@ int send_pack(struct send_pack_args *args,
 		return 0;
 	}
 
+	if (!args->dry_run)
+		advertise_shallow_grafts(out);
+
 	/*
 	 * Finally, tell the other end!
 	 */
diff --git a/t/t5537-push-shallow.sh b/t/t5537-push-shallow.sh
new file mode 100755
index 0000000..650c31a
--- /dev/null
+++ b/t/t5537-push-shallow.sh
@@ -0,0 +1,70 @@
+#!/bin/sh
+
+test_description='push from/to a shallow clone'
+
+. ./test-lib.sh
+
+commit() {
+	echo "$1" >tracked &&
+	git add tracked &&
+	git commit -m "$1"
+}
+
+test_expect_success 'setup' '
+	git config --global transfer.fsckObjects true &&
+	commit 1 &&
+	commit 2 &&
+	commit 3 &&
+	commit 4 &&
+	(
+	git init full-abc &&
+	cd full-abc &&
+	commit a &&
+	commit b &&
+	commit c
+	) &&
+	git clone --no-local --depth=2 .git shallow &&
+	git --git-dir=shallow/.git log --format=%s >actual &&
+	cat <<EOF >expect &&
+4
+3
+EOF
+	test_cmp expect actual &&
+	git clone --no-local --depth=2 full-abc/.git shallow2 &&
+	git --git-dir=shallow2/.git log --format=%s >actual &&
+	cat <<EOF >expect &&
+c
+b
+EOF
+	test_cmp expect actual
+'
+
+test_expect_success 'push from shallow clone' '
+	(
+	cd shallow &&
+	commit 5 &&
+	git push ../.git +master:refs/remotes/shallow/master
+	) &&
+	git log --format=%s shallow/master >actual &&
+	git fsck &&
+	cat <<EOF >expect &&
+5
+4
+3
+2
+1
+EOF
+	test_cmp expect actual
+'
+
+test_expect_success 'push from shallow clone, with grafted roots' '
+	(
+	cd shallow2 &&
+	test_must_fail git push ../.git +master:refs/remotes/shallow2/master 2>err &&
+	grep "shallow2/master.*shallow update not allowed" err
+	) &&
+	test_must_fail git rev-parse shallow2/master &&
+	git fsck
+'
+
+test_done
-- 
1.8.2.83.gc99314b