Partial removal of fetching from git-clone

Partial removal of fetching from git-clone

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

This was needed for doing submodules in git-fetch,
but since I moved the cloning of submodules to git-checkout,
I no longer need these changes.
Still, I know Junio wants something like this, so they
could be used as a starting point for completely
removing all fetching from git-clone.

skimo

[PATCH 1/3] builtin-fetch--tool: extend "native-store" for use in cloning

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 builtin-fetch--tool.c |   52 +++++++++++++++++++++++++++++++++++++++++++-----
 1 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c
index ed4d5de..b3ed3da 100644
--- a/builtin-fetch--tool.c
+++ b/builtin-fetch--tool.c
@@ -207,6 +207,32 @@ static void remove_keep_on_signal(int signo)
 	raise(signo);
 }
 
+static char *construct_local_name(const char *remote_ref, const char *remote_nick,
+				  int use_separate_remote)
+{
+	static char local_ref[PATH_MAX];
+	int len = strlen(remote_ref);
+
+	if (len >= 3 && !memcmp(remote_ref+len-3, "^{}", 3))
+		return NULL;
+	if (!strcmp(remote_ref, "HEAD"))
+		return "REMOTE_HEAD";
+	if (!prefixcmp(remote_ref, "refs/heads/")) {
+		if (snprintf(local_ref, sizeof(local_ref), "refs/%s%s/%s",
+			    use_separate_remote ? "remotes/" : "heads",
+			    use_separate_remote ? remote_nick : "",
+			    remote_ref+11) > sizeof(local_ref))
+			die("Local branchname too long");
+	} else if (!prefixcmp(remote_ref, "refs/tags/")) {
+		if (snprintf(local_ref, sizeof(local_ref), "refs/tags/%s",
+			    remote_ref+10) > sizeof(local_ref))
+			die("Local branchname too long");
+	} else
+		return NULL;
+
+	return local_ref;
+}
+
 static char *find_local_name(const char *remote_name, const char *refs,
 			     int *force_p, int *not_for_merge_p)
 {
@@ -261,7 +287,8 @@ static int fetch_native_store(FILE *fp,
 			      const char *remote,
 			      const char *remote_nick,
 			      const char *refs,
-			      int verbose, int force)
+			      int verbose, int force,
+			      int all, int use_separate_remote)
 {
 	char buffer[1024];
 	int err = 0;
@@ -273,7 +300,7 @@ static int fetch_native_store(FILE *fp,
 		int len;
 		char *cp;
 		char *local_name;
-		int single_force, not_for_merge;
+		int single_force = force, not_for_merge = 0;
 
 		for (cp = buffer; *cp && !isspace(*cp); cp++)
 			;
@@ -294,14 +321,18 @@ static int fetch_native_store(FILE *fp,
 			continue;
 		}
 
-		local_name = find_local_name(cp, refs,
-					     &single_force, &not_for_merge);
+		if (all)
+			local_name = construct_local_name(cp, remote_nick,
+							  use_separate_remote);
+		else
+			local_name = find_local_name(cp, refs,
+						     &single_force, &not_for_merge);
 		if (!local_name)
 			continue;
 		err |= append_fetch_head(fp,
 					 buffer, remote, cp, remote_nick,
 					 local_name, not_for_merge,
-					 verbose, force || single_force);
+					 verbose, single_force);
 	}
 	return err;
 }
@@ -514,6 +545,8 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix)
 	int verbose = 0;
 	int force = 0;
 	int sopt = 0;
+	int all = 0;
+	int use_separate_remote = 1;
 
 	while (1 < argc) {
 		const char *arg = argv[1];
@@ -523,6 +556,12 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix)
 			force = 1;
 		else if (!strcmp("-s", arg))
 			sopt = 1;
+		else if (!strcmp("--all", arg))
+			all = 1;
+		else if (!strcmp("--use-separate-remote", arg))
+			use_separate_remote = 1;
+		else if (!strcmp("--no-separate-remote", arg))
+			use_separate_remote = 0;
 		else
 			break;
 		argc--;
@@ -554,7 +593,8 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix)
 			return error("fetch-native-store takes 3 args");
 		fp = fopen(git_path("FETCH_HEAD"), "a");
 		result = fetch_native_store(fp, argv[2], argv[3], argv[4],
-					    verbose, force);
+					    verbose, force, all,
+					    use_separate_remote);
 		fclose(fp);
 		return result;
 	}
-- 
1.5.1.5.g8fc2

[PATCH 2/3] git-clone: rely on git-fetch for fetching for most protocols

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 git-clone.sh |   20 ++++++++++++--------
 git-fetch.sh |   28 ++++++++++++++++++++++------
 2 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/git-clone.sh b/git-clone.sh
index fdd354f..823b973 100755
--- a/git-clone.sh
+++ b/git-clone.sh
@@ -159,6 +159,11 @@ then
 	no_checkout=yes
 	use_separate_remote=
 fi
+if test t = "$use_separate_remote"; then
+	separate_remote_flag="--use-separate-remote"
+else
+	separate_remote_flag="--no-separate-remote"
+fi
 
 if test -z "$origin"
 then
@@ -219,6 +224,10 @@ then
 	fi
 fi
 
+# Write out $origin URL
+GIT_CONFIG="$GIT_DIR/config"
+git-config remote."$origin".url "$repo" || exit
+
 rm -f "$GIT_DIR/CLONE_HEAD"
 
 # We do local magic only when the user tells us to.
@@ -299,11 +308,9 @@ yes,yes)
 		fi
 		;;
 	*)
-		case "$upload_pack" in
-		'') git-fetch-pack --all -k $quiet $depth $no_progress "$repo";;
-		*) git-fetch-pack --all -k $quiet "$upload_pack" $depth $no_progress "$repo" ;;
-		esac >"$GIT_DIR/CLONE_HEAD" ||
-			die "fetch-pack from '$repo' failed."
+		git-fetch --all -k $quiet  ${upload_pack:+"$upload_pack"} $depth \
+			$separate_remote_flag "$origin" ||
+			die "fetch from '$repo' failed."
 		;;
 	esac
 	;;
@@ -387,9 +394,6 @@ then
 		origin_track="$remote_top/$head_points_at" &&
 		git-update-ref HEAD "$head_sha1" &&
 
-		# Upstream URL
-		git-config remote."$origin".url "$repo" &&
-
 		# Set up the mappings to track the remote branches.
 		git-config remote."$origin".fetch \
 			"+refs/heads/*:$remote_top/*" '^$' &&
diff --git a/git-fetch.sh b/git-fetch.sh
index 0e05cf1..dcb6985 100755
--- a/git-fetch.sh
+++ b/git-fetch.sh
@@ -15,6 +15,7 @@ LF='
 '
 IFS="$LF"
 
+all=
 no_tags=
 tags=
 append=
@@ -25,6 +26,7 @@ exec=
 keep=
 shallow_depth=
 no_progress=
+use_separate_remote=
 test -t 1 || no_progress=--no-progress
 quiet=
 while case "$#" in 0) break ;; esac
@@ -33,6 +35,9 @@ do
 	-a|--a|--ap|--app|--appe|--appen|--append)
 		append=t
 		;;
+	--al|--all)
+		all=--all
+		;;
 	--upl|--uplo|--uploa|--upload|--upload-|--upload-p|\
 	--upload-pa|--upload-pac|--upload-pack)
 		shift
@@ -63,6 +68,12 @@ do
 	-v|--verbose)
 		verbose=Yes
 		;;
+	--use-separate-remote)
+		use_separate_remote="--use-separate-remote"
+		;;
+	--no-separate-remote)
+		use_separate_remote="--no-separate-remote"
+		;;
 	-k|--k|--ke|--kee|--keep)
 		keep='-k -k'
 		;;
@@ -141,7 +152,9 @@ esac
 # branches file, and just fetch those and refspecs explicitly given.
 # Otherwise we do what we always did.
 
-reflist=$(get_remote_refs_for_fetch "$@")
+if test -z "$all"; then
+	reflist=$(get_remote_refs_for_fetch "$@")
+fi
 if test "$tags"
 then
 	taglist=`IFS='	' &&
@@ -163,8 +176,10 @@ fi
 
 fetch_all_at_once () {
 
-  eval=$(echo "$1" | git-fetch--tool parse-reflist "-")
-  eval "$eval"
+    if test -z "$all"; then
+	eval=$(echo "$1" | git-fetch--tool parse-reflist "-")
+	eval "$eval"
+    fi
 
     ( : subshell because we muck with IFS
       IFS=" 	$LF"
@@ -177,7 +192,8 @@ fetch_all_at_once () {
 	    git-bundle unbundle "$remote" $rref ||
 	    echo failed "$remote"
 	else
-		if	test -d "$remote" &&
+		if	test -z "$all" &&
+			test -d "$remote" &&
 
 			# The remote might be our alternate.  With
 			# this optimization we will bypass fetch-pack
@@ -201,7 +217,7 @@ fetch_all_at_once () {
 			echo "$ls_remote_result" | \
 				git-fetch--tool pick-rref "$rref" "-"
 		else
-			git-fetch-pack --thin $exec $keep $shallow_depth \
+			git-fetch-pack --thin $all $exec $keep $shallow_depth \
 				$quiet $no_progress "$remote" $rref ||
 			echo failed "$remote"
 		fi
@@ -212,7 +228,7 @@ fetch_all_at_once () {
 	test -n "$verbose" && flags="$flags -v"
 	test -n "$force" && flags="$flags -f"
 	GIT_REFLOG_ACTION="$GIT_REFLOG_ACTION" \
-		git-fetch--tool $flags native-store \
+		git-fetch--tool $flags $all $use_separate_remote native-store \
 			"$remote" "$remote_nick" "$refs"
       )
     ) || exit
-- 
1.5.1.5.g8fc2

[PATCH 3/3] git-clone: rely on git-fetch for non-bare fetching over http

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 git-clone.sh |    6 +++---
 git-fetch.sh |   20 ++++++++++++++++++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/git-clone.sh b/git-clone.sh
index 823b973..8d5dc05 100755
--- a/git-clone.sh
+++ b/git-clone.sh
@@ -262,8 +262,8 @@ yes,yes)
 	git-ls-remote "$repo" >"$GIT_DIR/CLONE_HEAD" || exit 1
 	;;
 *)
-	case "$repo" in
-	rsync://*)
+	case "$bare,$repo" in
+	*,rsync://*)
 		case "$depth" in
 		"") ;;
 		*) die "shallow over rsync not supported" ;;
@@ -295,7 +295,7 @@ yes,yes)
 		fi
 		git-ls-remote "$repo" >"$GIT_DIR/CLONE_HEAD" || exit 1
 		;;
-	https://*|http://*|ftp://*)
+	yes,https://*|yes,http://*|yes,ftp://*)
 		case "$depth" in
 		"") ;;
 		*) die "shallow over http or ftp not supported" ;;
diff --git a/git-fetch.sh b/git-fetch.sh
index dcb6985..68706db 100755
--- a/git-fetch.sh
+++ b/git-fetch.sh
@@ -235,11 +235,31 @@ fetch_all_at_once () {
 
 }
 
+http_fetch () {
+	if [ -n "$GIT_SSL_NO_VERIFY" ]; then
+		curl_extra_args="-k"
+	fi
+	if [ -n "$GIT_CURL_FTP_NO_EPSV" -o \
+		"`git-config --bool http.noEPSV`" = true ]; then
+		curl_extra_args="${curl_extra_args} --disable-epsv"
+	fi
+
+	# $1 = Remote, $2 = Local
+	curl -nsfL $curl_extra_args "$1" >"$2"
+}
+
 fetch_per_ref () {
   reflist="$1"
   refs=
   rref=
 
+    if test -n "$all"; then
+	reflist=$(canon_refs_list_for_fetch -d "$remote_nick" \
+			"+refs/heads/*:refs/remotes/$remote_nick/*")
+	http_fetch "$remote/HEAD" "$GIT_DIR/REMOTE_HEAD" ||
+	rm -f "$GIT_DIR/REMOTE_HEAD"
+    fi
+
   for ref in $reflist
   do
       refs="$refs$LF$ref"
-- 
1.5.1.5.g8fc2