[PATCH v2 5/5] Use --not-so-strict on all pack transfer for connectivity check

["Nguyễn Thái Ngọc Duy" <pclouds@gmail.com>]

This replaces check_everything_connected() with --not-so-strict, which
accomplishes the same thing and is generally cheaper when index-pack
or unpack-objects are used. All other cases fall back to
check_everything_connected.

This could help reduce the impact of check_everything_connected() on
pack transport. For example, cloning file://../linux-2.6.git

        before       after
real    4m23.664s    3m47.280s
user    4m55.613s    4m39.530s
sys     0m14.805s    0m17.728s

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 builtin/clone.c                 | 22 ++++++++++--------
 builtin/fetch.c                 | 11 +++++----
 builtin/receive-pack.c          | 51 ++++-------------------------------------
 fetch-pack.c                    |  2 ++
 fetch-pack.h                    |  3 ++-
 t/t5504-fetch-receive-strict.sh |  2 +-
 transport.c                     |  2 ++
 transport.h                     |  1 +
 8 files changed, 32 insertions(+), 62 deletions(-)

diff --git a/builtin/clone.c b/builtin/clone.c
index dad4265..1cd37c1 100644
--- a/builtin/clone.c
+++ b/builtin/clone.c
@@ -497,6 +497,8 @@ static void write_remote_refs(const struct ref *local_refs)
 	for (r = local_refs; r; r = r->next) {
 		if (!r->peer_ref)
 			continue;
+		if (!has_sha1_file(r->old_sha1))
+			die(_("remote did not send all necessary objects"));
 		add_packed_ref(r->peer_ref->name, r->old_sha1);
 	}
 
@@ -544,15 +546,6 @@ static void update_remote_refs(const struct ref *refs,
 			       const char *branch_top,
 			       const char *msg)
 {
-	const struct ref *rm = mapped_refs;
-
-	if (0 <= option_verbosity)
-		printf(_("Checking connectivity... "));
-	if (check_everything_connected(iterate_ref_map, 0, &rm))
-		die(_("remote did not send all necessary objects"));
-	if (0 <= option_verbosity)
-		printf(_("done\n"));
-
 	if (refs) {
 		write_remote_refs(mapped_refs);
 		if (option_single_branch)
@@ -953,6 +946,17 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	else if (refs && complete_refs_before_fetch)
 		transport_fetch_refs(transport, mapped_refs);
 
+	if (!transport->smart_options ||
+	    !transport->smart_options->connected) {
+		const struct ref *rm = mapped_refs;
+
+		if (0 <= option_verbosity)
+			printf(_("Checking connectivity... "));
+		if (check_everything_connected(iterate_ref_map, 0, &rm))
+			die(_("remote did not send all necessary objects"));
+		if (0 <= option_verbosity)
+			printf(_("done\n"));
+	}
 	update_remote_refs(refs, mapped_refs, remote_head_points_at,
 			   branch_top.buf, reflog_msg.buf);
 
diff --git a/builtin/fetch.c b/builtin/fetch.c
index 4b6b1df..e013d3f 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -380,7 +380,7 @@ static int iterate_ref_map(void *cb_data, unsigned char sha1[20])
 }
 
 static int store_updated_refs(const char *raw_url, const char *remote_name,
-		struct ref *ref_map)
+			      struct ref *ref_map, int need_check)
 {
 	FILE *fp;
 	struct commit *commit;
@@ -401,7 +401,8 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
 		url = xstrdup("foreign");
 
 	rm = ref_map;
-	if (check_everything_connected(iterate_ref_map, 0, &rm)) {
+	if (need_check &&
+	    check_everything_connected(iterate_ref_map, 0, &rm)) {
 		rc = error(_("%s did not send all necessary objects\n"), url);
 		goto abort;
 	}
@@ -538,8 +539,10 @@ static int fetch_refs(struct transport *transport, struct ref *ref_map)
 		ret = transport_fetch_refs(transport, ref_map);
 	if (!ret)
 		ret |= store_updated_refs(transport->url,
-				transport->remote->name,
-				ref_map);
+					  transport->remote->name,
+					  ref_map,
+					  !transport->smart_options ||
+					  !transport->smart_options->connected);
 	transport_unlock_pack(transport);
 	return ret;
 }
diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c
index e3eb5fc..2157c1a 100644
--- a/builtin/receive-pack.c
+++ b/builtin/receive-pack.c
@@ -651,48 +651,6 @@ static void check_aliased_updates(struct command *commands)
 	string_list_clear(&ref_list, 0);
 }
 
-static int command_singleton_iterator(void *cb_data, unsigned char sha1[20])
-{
-	struct command **cmd_list = cb_data;
-	struct command *cmd = *cmd_list;
-
-	if (!cmd || is_null_sha1(cmd->new_sha1))
-		return -1; /* end of list */
-	*cmd_list = NULL; /* this returns only one */
-	hashcpy(sha1, cmd->new_sha1);
-	return 0;
-}
-
-static void set_connectivity_errors(struct command *commands)
-{
-	struct command *cmd;
-
-	for (cmd = commands; cmd; cmd = cmd->next) {
-		struct command *singleton = cmd;
-		if (!check_everything_connected(command_singleton_iterator,
-						0, &singleton))
-			continue;
-		cmd->error_string = "missing necessary objects";
-	}
-}
-
-static int iterate_receive_command_list(void *cb_data, unsigned char sha1[20])
-{
-	struct command **cmd_list = cb_data;
-	struct command *cmd = *cmd_list;
-
-	while (cmd) {
-		if (!is_null_sha1(cmd->new_sha1)) {
-			hashcpy(sha1, cmd->new_sha1);
-			*cmd_list = cmd->next;
-			return 0;
-		}
-		cmd = cmd->next;
-	}
-	*cmd_list = NULL;
-	return -1; /* end of list */
-}
-
 static void reject_updates_to_hidden(struct command *commands)
 {
 	struct command *cmd;
@@ -718,11 +676,6 @@ static void execute_commands(struct command *commands, const char *unpacker_erro
 		return;
 	}
 
-	cmd = commands;
-	if (check_everything_connected(iterate_receive_command_list,
-				       0, &cmd))
-		set_connectivity_errors(commands);
-
 	reject_updates_to_hidden(commands);
 
 	if (run_receive_hook(commands, "pre-receive", 0)) {
@@ -844,6 +797,8 @@ static const char *unpack(int err_fd)
 			unpacker[i++] = "-q";
 		if (fsck_objects)
 			unpacker[i++] = "--strict";
+		else
+			unpacker[i++] = "--not-so-strict";
 		unpacker[i++] = hdr_arg;
 		unpacker[i++] = NULL;
 		memset(&child, 0, sizeof(child));
@@ -869,6 +824,8 @@ static const char *unpack(int err_fd)
 		keeper[i++] = "--stdin";
 		if (fsck_objects)
 			keeper[i++] = "--strict";
+		else
+			keeper[i++] = "--not-so-strict";
 		keeper[i++] = "--fix-thin";
 		keeper[i++] = hdr_arg;
 		keeper[i++] = keep_arg;
diff --git a/fetch-pack.c b/fetch-pack.c
index 17cfa88..cdd9e2d 100644
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -755,6 +755,8 @@ static int get_pack(struct fetch_pack_args *args,
 	    ? transfer_fsck_objects
 	    : 0)
 		*av++ = "--strict";
+	else if (args->check_connectivity)
+		*av++ = "--not-so-strict";
 	*av++ = NULL;
 
 	cmd.in = demux.out;
diff --git a/fetch-pack.h b/fetch-pack.h
index dc5266c..824136d 100644
--- a/fetch-pack.h
+++ b/fetch-pack.h
@@ -16,7 +16,8 @@ struct fetch_pack_args {
 		verbose:1,
 		no_progress:1,
 		include_tag:1,
-		stateless_rpc:1;
+		stateless_rpc:1,
+		check_connectivity:1;
 };
 
 /*
diff --git a/t/t5504-fetch-receive-strict.sh b/t/t5504-fetch-receive-strict.sh
index 69ee13c..14d2935 100755
--- a/t/t5504-fetch-receive-strict.sh
+++ b/t/t5504-fetch-receive-strict.sh
@@ -60,7 +60,7 @@ test_expect_success 'fetch with transfer.fsckobjects' '
 
 cat >exp <<EOF
 To dst
-!	refs/heads/master:refs/heads/test	[remote rejected] (missing necessary objects)
+!	refs/heads/master:refs/heads/test	[remote rejected] (unpacker error)
 EOF
 
 test_expect_success 'push without strict' '
diff --git a/transport.c b/transport.c
index ba5d8af..72e6fa3 100644
--- a/transport.c
+++ b/transport.c
@@ -534,6 +534,7 @@ static int fetch_refs_via_pack(struct transport *transport,
 	args.quiet = (transport->verbose < 0);
 	args.no_progress = !transport->progress;
 	args.depth = data->options.depth;
+	args.check_connectivity = 1;
 
 	if (!data->got_remote_heads) {
 		connect_setup(transport, 0, 0);
@@ -551,6 +552,7 @@ static int fetch_refs_via_pack(struct transport *transport,
 		refs = NULL;
 	data->conn = NULL;
 	data->got_remote_heads = 0;
+	data->options.connected = 1;
 
 	free_refs(refs_tmp);
 
diff --git a/transport.h b/transport.h
index fcb1d25..e127224 100644
--- a/transport.h
+++ b/transport.h
@@ -8,6 +8,7 @@ struct git_transport_options {
 	unsigned thin : 1;
 	unsigned keep : 1;
 	unsigned followtags : 1;
+	unsigned connected : 1;
 	int depth;
 	const char *uploadpack;
 	const char *receivepack;
-- 
1.8.2.83.gc99314b