Second round of support for cloning submodules

Second round of support for cloning submodules

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

This patch series implements a mechanism for cloning submodules.
Each submodule is specified by a 'submodule.<submodule>.url'
configuration option, e.g.,

bash-3.00$ ./git-config --remote=http://www.liacs.nl/~sverdool/isa.git --get-regexp 'submodule\..*\.url' 
submodule.cloog.url /home/sverdool/public_html/cloog.git
submodule.cloog.url http://www.liacs.nl/~sverdool/cloog.git

git-clone will use the first url that works.
E.g., a

git clone --submodules ssh://liacs/~/public_html/isa.git

(which only works for me), will use the first url, while a

git clone --submodules http://www.liacs.nl/~sverdool/isa.git

will use the second.

The cloning of submodules is now handled inside git-fetch.

skimo

[PATCH 01/16] Add dump-config

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

This command dumps the config of a repository and will be used
to read config options from a remote site.

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 .gitignore                        |    1 +
 Documentation/cmd-list.perl       |    1 +
 Documentation/git-dump-config.txt |   37 +++++++++++++++++++++++++++++++++++++
 Makefile                          |    1 +
 daemon.c                          |    7 +++++++
 dump-config.c                     |   29 +++++++++++++++++++++++++++++
 6 files changed, 76 insertions(+), 0 deletions(-)
 create mode 100644 Documentation/git-dump-config.txt
 create mode 100644 dump-config.c

diff --git a/.gitignore b/.gitignore
index 4dc0c39..d4e5492 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,6 +38,7 @@ git-diff-files
 git-diff-index
 git-diff-tree
 git-describe
+git-dump-config
 git-fast-import
 git-fetch
 git-fetch--tool
diff --git a/Documentation/cmd-list.perl b/Documentation/cmd-list.perl
index 443802a..fa04615 100755
--- a/Documentation/cmd-list.perl
+++ b/Documentation/cmd-list.perl
@@ -103,6 +103,7 @@ git-diff-files                          plumbinginterrogators
 git-diff-index                          plumbinginterrogators
 git-diff                                mainporcelain
 git-diff-tree                           plumbinginterrogators
+git-dump-config                         synchelpers
 git-fast-import				ancillarymanipulators
 git-fetch                               mainporcelain
 git-fetch-pack                          synchingrepositories
diff --git a/Documentation/git-dump-config.txt b/Documentation/git-dump-config.txt
new file mode 100644
index 0000000..370781c
--- /dev/null
+++ b/Documentation/git-dump-config.txt
@@ -0,0 +1,37 @@
+git-dump-config(1)
+====================
+
+NAME
+----
+git-dump-config - Dump config options
+
+
+SYNOPSIS
+--------
+'git-dump-config' <directory>
+
+DESCRIPTION
+-----------
+Invoked by 'git-config --remote' and dumps the config file to the
+other end over the git protocol.
+
+This command is usually not invoked directly by the end user.  The UI
+for the protocol is on the 'git-config' side, where it is used to get
+options from a remote repository.
+
+OPTIONS
+-------
+<directory>::
+	The repository to get the config options from.
+
+Author
+------
+Written by Sven Verdoolaege.
+
+Documentation
+--------------
+Documentation by Sven Verdoolaege.
+
+GIT
+---
+Part of the gitlink:git[7] suite
diff --git a/Makefile b/Makefile
index 29243c6..37eb861 100644
--- a/Makefile
+++ b/Makefile
@@ -240,6 +240,7 @@ PROGRAMS = \
 	git-fast-import$X \
 	git-merge-base$X \
 	git-daemon$X \
+	git-dump-config$X \
 	git-merge-index$X git-mktag$X git-mktree$X git-patch-id$X \
 	git-peek-remote$X git-receive-pack$X \
 	git-send-pack$X git-shell$X \
diff --git a/daemon.c b/daemon.c
index e74ecac..3e5ebf3 100644
--- a/daemon.c
+++ b/daemon.c
@@ -378,10 +378,17 @@ static int receive_pack(void)
 	return -1;
 }
 
+static int dump_config(void)
+{
+	execl_git_cmd("dump-config", ".", NULL);
+	return -1;
+}
+
 static struct daemon_service daemon_service[] = {
 	{ "upload-archive", "uploadarch", upload_archive, 0, 1 },
 	{ "upload-pack", "uploadpack", upload_pack, 1, 1 },
 	{ "receive-pack", "receivepack", receive_pack, 0, 1 },
+	{ "dump-config", "dumpconfig", dump_config, 0, 1 },
 };
 
 static void enable_service(const char *name, int ena) {
diff --git a/dump-config.c b/dump-config.c
new file mode 100644
index 0000000..355920d
--- /dev/null
+++ b/dump-config.c
@@ -0,0 +1,29 @@
+#include "git-compat-util.h"
+#include "cache.h"
+#include "pkt-line.h"
+
+static const char dump_config_usage[] = "git-dump-config <dir>";
+
+static int dump_config(const char *var, const char *value)
+{
+	packet_write(1, "%s", var);
+	packet_write(1, "%s", value);
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	char *dir;
+
+	if (argc != 2)
+		usage(dump_config_usage);
+
+	dir = argv[1];
+	if (!enter_repo(dir, 0))
+		die("'%s': unable to chdir or not a git archive", dir);
+
+	git_config(dump_config);
+	packet_flush(1);
+
+	return 0;
+}
-- 
1.5.2.rc3.783.gc7476-dirty

[PATCH 02/16] git-config: add --remote option for reading config from remote repo

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 Documentation/git-config.txt |   33 +++++++++++++++++++++---------
 builtin-config.c             |   44 ++++++++++++++++++++++++++++++++---------
 cache.h                      |    1 +
 config.c                     |   26 ++++++++++++++++++++++++
 4 files changed, 84 insertions(+), 20 deletions(-)

diff --git a/Documentation/git-config.txt b/Documentation/git-config.txt
index 280ef20..76398ab 100644
--- a/Documentation/git-config.txt
+++ b/Documentation/git-config.txt
@@ -9,16 +9,25 @@ git-config - Get and set repository or global options
 SYNOPSIS
 --------
 [verse]
-'git-config' [--system | --global] [type] name [value [value_regex]]
-'git-config' [--system | --global] [type] --add name value
-'git-config' [--system | --global] [type] --replace-all name [value [value_regex]]
-'git-config' [--system | --global] [type] --get name [value_regex]
-'git-config' [--system | --global] [type] --get-all name [value_regex]
-'git-config' [--system | --global] [type] --unset name [value_regex]
-'git-config' [--system | --global] [type] --unset-all name [value_regex]
-'git-config' [--system | --global] [type] --rename-section old_name new_name
-'git-config' [--system | --global] [type] --remove-section name
-'git-config' [--system | --global] -l | --list
+'git-config' [--system | --global | --remote=[<host>:]<directory ]
+	     [type] name [value [value_regex]]
+'git-config' [--system | --global | --remote=[<host>:]<directory ]
+	     [type] --add name value
+'git-config' [--system | --global | --remote=[<host>:]<directory ]
+	     [type] --replace-all name [value [value_regex]]
+'git-config' [--system | --global | --remote=[<host>:]<directory ]
+	     [type] --get name [value_regex]
+'git-config' [--system | --global | --remote=[<host>:]<directory ]
+	     [type] --get-all name [value_regex]
+'git-config' [--system | --global | --remote=[<host>:]<directory ]
+	     [type] --unset name [value_regex]
+'git-config' [--system | --global | --remote=[<host>:]<directory ]
+	     [type] --unset-all name [value_regex]
+'git-config' [--system | --global | --remote=[<host>:]<directory ]
+	     [type] --rename-section old_name new_name
+'git-config' [--system | --global | --remote=[<host>:]<directory ]
+	     [type] --remove-section name
+'git-config' [--system | --global | --remote=[<host>:]<directory ] -l | --list
 
 DESCRIPTION
 -----------
@@ -80,6 +89,10 @@ OPTIONS
 	Use system-wide $(prefix)/etc/gitconfig rather than the repository
 	.git/config.
 
+--remote=[<host>:]<directory
+	Use remote config instead of the repository .git/config.
+	Only available for reading options.
+
 --remove-section::
 	Remove the given section from the configuration file.
 
diff --git a/builtin-config.c b/builtin-config.c
index b2515f7..3a1e86c 100644
--- a/builtin-config.c
+++ b/builtin-config.c
@@ -2,8 +2,10 @@
 #include "cache.h"
 
 static const char git_config_set_usage[] =
-"git-config [ --global | --system ] [ --bool | --int ] [--get | --get-all | --get-regexp | --replace-all | --add | --unset | --unset-all] name [value [value_regex]] | --rename-section old_name new_name | --remove-section name | --list";
+"git-config [ --global | --system | --remote=[<host>:]<directory ] "
+"[ --bool | --int ] [--get | --get-all | --get-regexp | --replace-all | --add | --unset | --unset-all] name [value [value_regex]] | --rename-section old_name new_name | --remove-section name | --list";
 
+static char *dest;
 static char *key;
 static regex_t *key_regexp;
 static regex_t *regexp;
@@ -104,15 +106,19 @@ static int get_value(const char* key_, const char* regex_)
 		}
 	}
 
-	if (do_all && system_wide)
-		git_config_from_file(show_config, system_wide);
-	if (do_all && global)
-		git_config_from_file(show_config, global);
-	git_config_from_file(show_config, local);
-	if (!do_all && !seen && global)
-		git_config_from_file(show_config, global);
-	if (!do_all && !seen && system_wide)
-		git_config_from_file(show_config, system_wide);
+	if (dest)
+		git_config_from_remote(show_config, dest);
+	else {
+		if (do_all && system_wide)
+			git_config_from_file(show_config, system_wide);
+		if (do_all && global)
+			git_config_from_file(show_config, global);
+		git_config_from_file(show_config, local);
+		if (!do_all && !seen && global)
+			git_config_from_file(show_config, global);
+		if (!do_all && !seen && system_wide)
+			git_config_from_file(show_config, system_wide);
+	}
 
 	free(key);
 	if (regexp) {
@@ -155,8 +161,14 @@ int cmd_config(int argc, const char **argv, const char *prefix)
 		}
 		else if (!strcmp(argv[1], "--system"))
 			setenv("GIT_CONFIG", ETC_GITCONFIG, 1);
+		else if (!prefixcmp(argv[1], "--remote="))
+			dest = xstrdup(argv[1]+9);
 		else if (!strcmp(argv[1], "--rename-section")) {
 			int ret;
+			if (dest) {
+				fprintf(stderr, "Cannot rename on remote\n");
+				return 1;
+			}
 			if (argc != 4)
 				usage(git_config_set_usage);
 			ret = git_config_rename_section(argv[2], argv[3]);
@@ -170,6 +182,10 @@ int cmd_config(int argc, const char **argv, const char *prefix)
 		}
 		else if (!strcmp(argv[1], "--remove-section")) {
 			int ret;
+			if (dest) {
+				fprintf(stderr, "Cannot remove on remote\n");
+				return 1;
+			}
 			if (argc != 3)
 				usage(git_config_set_usage);
 			ret = git_config_rename_section(argv[2], NULL);
@@ -191,6 +207,10 @@ int cmd_config(int argc, const char **argv, const char *prefix)
 	case 2:
 		return get_value(argv[1], NULL);
 	case 3:
+		if (dest && prefixcmp(argv[1], "--get")) {
+			fprintf(stderr, "Cannot (un)set on remote\n");
+			return 1;
+		}
 		if (!strcmp(argv[1], "--unset"))
 			return git_config_set(argv[2], NULL);
 		else if (!strcmp(argv[1], "--unset-all"))
@@ -209,6 +229,10 @@ int cmd_config(int argc, const char **argv, const char *prefix)
 
 			return git_config_set(argv[1], argv[2]);
 	case 4:
+		if (dest && prefixcmp(argv[1], "--get")) {
+			fprintf(stderr, "Cannot (un)set on remote\n");
+			return 1;
+		}
 		if (!strcmp(argv[1], "--unset"))
 			return git_config_set_multivar(argv[2], NULL, argv[3], 0);
 		else if (!strcmp(argv[1], "--unset-all"))
diff --git a/cache.h b/cache.h
index e34958b..6acc330 100644
--- a/cache.h
+++ b/cache.h
@@ -501,6 +501,7 @@ extern int update_server_info(int);
 typedef int (*config_fn_t)(const char *, const char *);
 extern int git_default_config(const char *, const char *);
 extern int git_config_from_file(config_fn_t fn, const char *);
+extern int git_config_from_remote(config_fn_t fn, char *dest);
 extern int git_config(config_fn_t fn);
 extern int git_config_int(const char *, const char *);
 extern int git_config_bool(const char *, const char *);
diff --git a/config.c b/config.c
index 0614c2b..dbfae3f 100644
--- a/config.c
+++ b/config.c
@@ -6,9 +6,12 @@
  *
  */
 #include "cache.h"
+#include "pkt-line.h"
 
 #define MAXNAME (256)
 
+static const char *dumpconfig = "git-dump-config";
+
 static FILE *config_file;
 static const char *config_file_name;
 static int config_linenr;
@@ -403,6 +406,29 @@ int git_config_from_file(config_fn_t fn, const char *filename)
 	return ret;
 }
 
+int git_config_from_remote(config_fn_t fn, char *dest)
+{
+	int ret;
+	int fd[2];
+	pid_t pid;
+	static char var[MAXNAME];
+	static char value[1024];
+
+	pid = git_connect(fd, dest, dumpconfig);
+	if (pid < 0)
+		return 1;
+	ret = 0;
+	while (packet_read_line(fd[0], var, sizeof(var))) {
+		if (!packet_read_line(fd[0], value, sizeof(value)))
+			die("Missing value");
+		fn(var, value);
+	}
+	close(fd[0]);
+	close(fd[1]);
+	ret |= finish_connect(pid);
+	return !!ret;
+}
+
 int git_config(config_fn_t fn)
 {
 	int ret = 0;
-- 
1.5.2.rc3.783.gc7476-dirty

[PATCH 03/16] http.h: make fill_active_slots a function pointer

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

This allows us to use the methods provided by http.c
from within libgit, in particular config.c.

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 http-fetch.c |    5 ++++-
 http-push.c  |    5 ++++-
 http.h       |    2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/http-fetch.c b/http-fetch.c
index 09baedc..53fb2a9 100644
--- a/http-fetch.c
+++ b/http-fetch.c
@@ -317,7 +317,7 @@ static void release_object_request(struct object_request *obj_req)
 }
 
 #ifdef USE_CURL_MULTI
-void fill_active_slots(void)
+static void fetch_fill_active_slots(void)
 {
 	struct object_request *obj_req = object_queue_head;
 	struct active_request_slot *slot = active_queue_head;
@@ -1031,6 +1031,9 @@ int main(int argc, const char **argv)
 	}
 	url = argv[arg];
 
+#ifdef USE_CURL_MULTI
+	fill_active_slots = fetch_fill_active_slots;
+#endif
 	http_init();
 
 	no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
diff --git a/http-push.c b/http-push.c
index e3f7675..d4c850b 100644
--- a/http-push.c
+++ b/http-push.c
@@ -794,7 +794,7 @@ static void finish_request(struct transfer_request *request)
 }
 
 #ifdef USE_CURL_MULTI
-void fill_active_slots(void)
+static void push_fill_active_slots(void)
 {
 	struct transfer_request *request = request_queue_head;
 	struct transfer_request *next;
@@ -2355,6 +2355,9 @@ int main(int argc, char **argv)
 
 	memset(remote_dir_exists, -1, 256);
 
+#ifdef USE_CURL_MULTI
+	fill_active_slots = push_fill_active_slots;
+#endif
 	http_init();
 
 	no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
diff --git a/http.h b/http.h
index 69b6b66..7a41cde 100644
--- a/http.h
+++ b/http.h
@@ -69,7 +69,7 @@ extern void finish_all_active_slots(void);
 extern void release_active_slot(struct active_request_slot *slot);
 
 #ifdef USE_CURL_MULTI
-extern void fill_active_slots(void);
+extern void (*fill_active_slots)(void);
 extern void step_active_slots(void);
 #endif
 
-- 
1.5.2.rc3.783.gc7476-dirty

[PATCH 04/16] git-config: read remote config files over HTTP

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 Makefile           |    7 ++++++-
 builtin-config.c   |    8 ++++++--
 config.c           |   16 +++++++++++++++-
 http.c             |   10 ++++++----
 http.h             |    2 +-
 http_config.h      |    1 +
 http_config_curl.c |   52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 http_config_none.c |    6 ++++++
 8 files changed, 93 insertions(+), 9 deletions(-)
 create mode 100644 http_config.h
 create mode 100644 http_config_curl.c
 create mode 100644 http_config_none.c

diff --git a/Makefile b/Makefile
index 37eb861..bce8514 100644
--- a/Makefile
+++ b/Makefile
@@ -319,7 +319,8 @@ LIB_OBJS = \
 	write_or_die.o trace.o list-objects.o grep.o match-trees.o \
 	alloc.o merge-file.o path-list.o help.o unpack-trees.o $(DIFF_OBJS) \
 	color.o wt-status.o archive-zip.o archive-tar.o shallow.o utf8.o \
-	convert.o attr.o decorate.o progress.o mailmap.o symlinks.o
+	convert.o attr.o decorate.o progress.o mailmap.o symlinks.o \
+	$(HTTP_CONFIG_OBJ)
 
 BUILTIN_OBJS = \
 	builtin-add.o \
@@ -526,6 +527,10 @@ ifndef NO_CURL
 	ifndef NO_EXPAT
 		EXPAT_LIBEXPAT = -lexpat
 	endif
+	HTTP_CONFIG_OBJ = http_config_curl.o http.o
+	EXTLIBS += $(CURL_LIBCURL)
+else
+	HTTP_CONFIG_OBJ = http_config_none.o
 endif
 
 ifndef NO_OPENSSL
diff --git a/builtin-config.c b/builtin-config.c
index 3a1e86c..7e18f73 100644
--- a/builtin-config.c
+++ b/builtin-config.c
@@ -147,8 +147,12 @@ int cmd_config(int argc, const char **argv, const char *prefix)
 			type = T_INT;
 		else if (!strcmp(argv[1], "--bool"))
 			type = T_BOOL;
-		else if (!strcmp(argv[1], "--list") || !strcmp(argv[1], "-l"))
-			return git_config(show_all_config);
+		else if (!strcmp(argv[1], "--list") || !strcmp(argv[1], "-l")) {
+			if (dest)
+				return git_config_from_remote(show_all_config, dest);
+			else
+				return git_config(show_all_config);
+		}
 		else if (!strcmp(argv[1], "--global")) {
 			char *home = getenv("HOME");
 			if (home) {
diff --git a/config.c b/config.c
index dbfae3f..fc2162b 100644
--- a/config.c
+++ b/config.c
@@ -7,6 +7,7 @@
  */
 #include "cache.h"
 #include "pkt-line.h"
+#include "http_config.h"
 
 #define MAXNAME (256)
 
@@ -406,6 +407,16 @@ int git_config_from_file(config_fn_t fn, const char *filename)
 	return ret;
 }
 
+static int config_from_http(config_fn_t fn, char *dest)
+{
+	char config_temp[50];
+	if (git_http_fetch_config(dest, config_temp, sizeof(config_temp)))
+		return 1;
+	git_config_from_file(fn, config_temp);
+	unlink(config_temp);
+	return 0;
+}
+
 int git_config_from_remote(config_fn_t fn, char *dest)
 {
 	int ret;
@@ -414,7 +425,10 @@ int git_config_from_remote(config_fn_t fn, char *dest)
 	static char var[MAXNAME];
 	static char value[1024];
 
-	pid = git_connect(fd, dest, dumpconfig);
+	if (!prefixcmp(dest, "http://"))
+		return config_from_http(fn, dest);
+
+	pid = git_connect(fd, dest, dumpconfig, 0);
 	if (pid < 0)
 		return 1;
 	ret = 0;
diff --git a/http.c b/http.c
index ae27e0c..c8237cb 100644
--- a/http.c
+++ b/http.c
@@ -25,6 +25,8 @@ long curl_low_speed_limit = -1;
 long curl_low_speed_time = -1;
 int curl_ftp_no_epsv = 0;
 
+void (*fill_active_slots)(void) = NULL;
+
 struct curl_slist *pragma_header;
 
 struct active_request_slot *active_queue_head = NULL;
@@ -394,7 +396,8 @@ void step_active_slots(void)
 	} while (curlm_result == CURLM_CALL_MULTI_PERFORM);
 	if (num_transfers < active_requests) {
 		process_curl_messages();
-		fill_active_slots();
+		if (fill_active_slots)
+			fill_active_slots();
 	}
 }
 #endif
@@ -458,9 +461,8 @@ void release_active_slot(struct active_request_slot *slot)
 		curl_easy_cleanup(slot->curl);
 		slot->curl = NULL;
 	}
-#ifdef USE_CURL_MULTI
-	fill_active_slots();
-#endif
+	if (fill_active_slots)
+		fill_active_slots();
 }
 
 static void finish_active_slot(struct active_request_slot *slot)
diff --git a/http.h b/http.h
index 7a41cde..7f29ff8 100644
--- a/http.h
+++ b/http.h
@@ -68,8 +68,8 @@ extern void run_active_slot(struct active_request_slot *slot);
 extern void finish_all_active_slots(void);
 extern void release_active_slot(struct active_request_slot *slot);
 
-#ifdef USE_CURL_MULTI
 extern void (*fill_active_slots)(void);
+#ifdef USE_CURL_MULTI
 extern void step_active_slots(void);
 #endif
 
diff --git a/http_config.h b/http_config.h
new file mode 100644
index 0000000..25f5c19
--- /dev/null
+++ b/http_config.h
@@ -0,0 +1 @@
+int git_http_fetch_config(const char *repo, char *config_file, int len);
diff --git a/http_config_curl.c b/http_config_curl.c
new file mode 100644
index 0000000..88317cf
--- /dev/null
+++ b/http_config_curl.c
@@ -0,0 +1,52 @@
+#include "http_config.h"
+#include "http.h"
+
+int git_http_fetch_config(const char *repo, char *config, int config_len)
+{
+	char url[PATH_MAX];
+	int len = strlen(repo);
+
+	int fd;
+	FILE *configfile;
+	struct active_request_slot *slot;
+	struct slot_results results;
+
+	strcpy(url, repo);
+	while (len > 0 && url[len-1] == '/')
+		--len;
+	snprintf(url+len, sizeof(url)-len, "/config");
+
+	fd = git_mkstemp(config, config_len, ".config_XXXXXX");
+	if (fd >= 0)
+		configfile = fdopen(fd, "w");
+	if (fd < 0 || !configfile)
+		return error("Unable to open local file %s for config",
+			     config);
+
+	http_init();
+
+	slot = get_active_slot();
+	slot->results = &results;
+	curl_easy_setopt(slot->curl, CURLOPT_FILE, configfile);
+	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
+	curl_easy_setopt(slot->curl, CURLOPT_URL, url);
+	slot->local = configfile;
+
+	if (start_active_slot(slot)) {
+		run_active_slot(slot);
+		if (results.curl_result != CURLE_OK) {
+			fclose(configfile);
+			warning("Unable to get config %s\n%s", url,
+				 curl_errorstr);
+		}
+	} else {
+		fclose(configfile);
+		return error("Unable to start request");
+	}
+
+	http_cleanup();
+
+	fclose(configfile);
+
+	return 0;
+}
diff --git a/http_config_none.c b/http_config_none.c
new file mode 100644
index 0000000..860ae84
--- /dev/null
+++ b/http_config_none.c
@@ -0,0 +1,6 @@
+#include "http_config.h"
+
+int git_http_fetch_config(const char *repo, char *config_file, int len)
+{
+	return error("Reading http config files not supported");
+}
-- 
1.5.2.rc3.783.gc7476-dirty

[PATCH 05/16] unpack-trees.c: verify_uptodate: remove dead code

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

This code was killed by commit fcc387db9bc453dc7e07a262873481af2ee9e5c8.

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 unpack-trees.c |    4 ----
 1 files changed, 0 insertions(+), 4 deletions(-)

diff --git a/unpack-trees.c b/unpack-trees.c
index 906ce69..cac2411 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -414,10 +414,6 @@ static void verify_uptodate(struct cache_entry *ce,
 			return;
 		errno = 0;
 	}
-	if (o->reset) {
-		ce->ce_flags |= htons(CE_UPDATE);
-		return;
-	}
 	if (errno == ENOENT)
 		return;
 	die("Entry '%s' not uptodate. Cannot merge.", ce->name);
-- 
1.5.2.rc3.783.gc7476-dirty

Re: [PATCH 05/16] unpack-trees.c: verify_uptodate: remove dead code

[Junio C Hamano]

skimo@liacs.nl writes:

> From: Sven Verdoolaege <skimo@kotnet.org>
>
> This code was killed by commit fcc387db9bc453dc7e07a262873481af2ee9e5c8.
>
> Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
> ---
>  unpack-trees.c |    4 ----
>  1 files changed, 0 insertions(+), 4 deletions(-)
>
> diff --git a/unpack-trees.c b/unpack-trees.c
> index 906ce69..cac2411 100644
> --- a/unpack-trees.c
> +++ b/unpack-trees.c
> @@ -414,10 +414,6 @@ static void verify_uptodate(struct cache_entry *ce,
>  			return;
>  		errno = 0;
>  	}
> -	if (o->reset) {
> -		ce->ce_flags |= htons(CE_UPDATE);
> -		return;
> -	}
>  	if (errno == ENOENT)
>  		return;
>  	die("Entry '%s' not uptodate. Cannot merge.", ce->name);
> -- 
> 1.5.2.rc3.783.gc7476-dirty

Hmmm.

I am not absolutely sure if the fcc387db change was correct
anymore, but in any case, this removal of dead code should not
break anything.

But this does not belong to your series either.

Perhaps I should apply this to 'master' regardless of the rest
of the series.

[PATCH 06/16] unpack-trees.c: pass cache_entry * to verify_absent rather than just the name

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

We will need the full cache_entry later to figure out if we are dealing
with a submodule.

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 unpack-trees.c |   32 ++++++++++++++++----------------
 1 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/unpack-trees.c b/unpack-trees.c
index cac2411..3dac150 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -487,7 +487,7 @@ static int verify_clean_subdirectory(const char *path, const char *action,
  * We do not want to remove or overwrite a working tree file that
  * is not tracked, unless it is ignored.
  */
-static void verify_absent(const char *path, const char *action,
+static void verify_absent(struct cache_entry *ce, const char *action,
 		struct unpack_trees_options *o)
 {
 	struct stat st;
@@ -495,12 +495,12 @@ static void verify_absent(const char *path, const char *action,
 	if (o->index_only || o->reset || !o->update)
 		return;
 
-	if (!lstat(path, &st)) {
+	if (!lstat(ce->name, &st)) {
 		int cnt;
 
-		if (o->dir && excluded(o->dir, path))
+		if (o->dir && excluded(o->dir, ce->name))
 			/*
-			 * path is explicitly excluded, so it is Ok to
+			 * ce->name is explicitly excluded, so it is Ok to
 			 * overwrite it.
 			 */
 			return;
@@ -512,7 +512,7 @@ static void verify_absent(const char *path, const char *action,
 			 * files that are in "foo/" we would lose
 			 * it.
 			 */
-			cnt = verify_clean_subdirectory(path, action, o);
+			cnt = verify_clean_subdirectory(ce->name, action, o);
 
 			/*
 			 * If this removed entries from the index,
@@ -540,7 +540,7 @@ static void verify_absent(const char *path, const char *action,
 		 * delete this path, which is in a subdirectory that
 		 * is being replaced with a blob.
 		 */
-		cnt = cache_name_pos(path, strlen(path));
+		cnt = cache_name_pos(ce->name, strlen(ce->name));
 		if (0 <= cnt) {
 			struct cache_entry *ce = active_cache[cnt];
 			if (!ce_stage(ce) && !ce->ce_mode)
@@ -548,7 +548,7 @@ static void verify_absent(const char *path, const char *action,
 		}
 
 		die("Untracked working tree file '%s' "
-		    "would be %s by merge.", path, action);
+		    "would be %s by merge.", ce->name, action);
 	}
 }
 
@@ -572,7 +572,7 @@ static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
 		}
 	}
 	else {
-		verify_absent(merge->name, "overwritten", o);
+		verify_absent(merge, "overwritten", o);
 		invalidate_ce_path(merge);
 	}
 
@@ -587,7 +587,7 @@ static int deleted_entry(struct cache_entry *ce, struct cache_entry *old,
 	if (old)
 		verify_uptodate(old, o);
 	else
-		verify_absent(ce->name, "removed", o);
+		verify_absent(ce, "removed", o);
 	ce->ce_mode = 0;
 	add_cache_entry(ce, ADD_CACHE_OK_TO_ADD|ADD_CACHE_OK_TO_REPLACE);
 	invalidate_ce_path(ce);
@@ -704,18 +704,18 @@ int threeway_merge(struct cache_entry **stages,
 	if (o->aggressive) {
 		int head_deleted = !head && !df_conflict_head;
 		int remote_deleted = !remote && !df_conflict_remote;
-		const char *path = NULL;
+		struct cache_entry *ce = NULL;
 
 		if (index)
-			path = index->name;
+			ce = index;
 		else if (head)
-			path = head->name;
+			ce = head;
 		else if (remote)
-			path = remote->name;
+			ce = remote;
 		else {
 			for (i = 1; i < o->head_idx; i++) {
 				if (stages[i] && stages[i] != o->df_conflict_entry) {
-					path = stages[i]->name;
+					ce = stages[i];
 					break;
 				}
 			}
@@ -730,8 +730,8 @@ int threeway_merge(struct cache_entry **stages,
 		    (remote_deleted && head && head_match)) {
 			if (index)
 				return deleted_entry(index, index, o);
-			else if (path && !head_deleted)
-				verify_absent(path, "removed", o);
+			else if (ce && !head_deleted)
+				verify_absent(ce, "removed", o);
 			return 0;
 		}
 		/*
-- 
1.5.2.rc3.783.gc7476-dirty

[PATCH 07/16] git-read-tree: take --submodules option

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

This option currently has no effect.

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 builtin-read-tree.c |   25 ++++++++++++++++++++++---
 cache.h             |    3 ++-
 unpack-trees.c      |    1 +
 unpack-trees.h      |    1 +
 4 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/builtin-read-tree.c b/builtin-read-tree.c
index 316fb0f..929dd95 100644
--- a/builtin-read-tree.c
+++ b/builtin-read-tree.c
@@ -87,14 +87,23 @@ static void prime_cache_tree(void)
 static const char read_tree_usage[] = "git-read-tree (<sha> | [[-m [--aggressive] | --reset | --prefix=<prefix>] [-u | -i]] [--exclude-per-directory=<gitignore>] [--index-output=<file>] <sha1> [<sha2> [<sha3>]])";
 
 static struct lock_file lock_file;
+static struct unpack_trees_options opts;
+
+static int git_read_tree_config(const char *var, const char *value)
+{
+	if (!strcmp(var, "core.submodules")) {
+		opts.submodules = git_config_bool(var, value);
+		return 0;
+	}
+
+	return git_default_config(var, value);
+}
 
 int cmd_read_tree(int argc, const char **argv, const char *unused_prefix)
 {
 	int i, newfd, stage = 0;
 	unsigned char sha1[20];
-	struct unpack_trees_options opts;
 
-	memset(&opts, 0, sizeof(opts));
 	opts.head_idx = -1;
 
 	setup_git_directory();
@@ -102,7 +111,7 @@ int cmd_read_tree(int argc, const char **argv, const char *unused_prefix)
 
 	newfd = hold_locked_index(&lock_file, 1);
 
-	git_config(git_default_config);
+	git_config(git_read_tree_config);
 
 	for (i = 1; i < argc; i++) {
 		const char *arg = argv[i];
@@ -172,6 +181,16 @@ int cmd_read_tree(int argc, const char **argv, const char *unused_prefix)
 			continue;
 		}
 
+		if (!strcmp(arg, "--no-submodules")) {
+			opts.submodules = 0;
+			continue;
+		}
+
+		if (!strcmp(arg, "--submodules")) {
+			opts.submodules = 1;
+			continue;
+		}
+
 		/* "-m" stands for "merge", meaning we start in stage 1 */
 		if (!strcmp(arg, "-m")) {
 			if (stage || opts.merge || opts.prefix)
diff --git a/cache.h b/cache.h
index 6acc330..42a275e 100644
--- a/cache.h
+++ b/cache.h
@@ -406,7 +406,8 @@ struct checkout {
 	unsigned force:1,
 		 quiet:1,
 		 not_new:1,
-		 refresh_cache:1;
+		 refresh_cache:1,
+		 submodules:1;
 };
 
 extern int checkout_entry(struct cache_entry *ce, const struct checkout *state, char *topath);
diff --git a/unpack-trees.c b/unpack-trees.c
index 3dac150..5fa637a 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -352,6 +352,7 @@ int unpack_trees(struct object_list *trees, struct unpack_trees_options *o)
 	state.force = 1;
 	state.quiet = 1;
 	state.refresh_cache = 1;
+	state.submodules = o->submodules;
 
 	o->merge_size = len;
 
diff --git a/unpack-trees.h b/unpack-trees.h
index fee7da4..21005d9 100644
--- a/unpack-trees.h
+++ b/unpack-trees.h
@@ -15,6 +15,7 @@ struct unpack_trees_options {
 	int trivial_merges_only;
 	int verbose_update;
 	int aggressive;
+	int submodules;
 	const char *prefix;
 	int pos;
 	struct dir_struct *dir;
-- 
1.5.2.rc3.783.gc7476-dirty

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Alex Riesen]

skimo@liacs.nl, Fri, May 18, 2007 21:24:56 +0200:
> 
> This option currently has no effect.
> 

Can we have this option (and corresponding support in the following
patches, of course) first? It is enough to have subprojects working
locally, and people can start using them immediately: anyone can clone
the subprojects manually if he wishes so.

Cloning of subprojects is still unclear, and frankly I'm not sure it
should be done at all. Not even with an option which is off by
default.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Sven Verdoolaege]

I noticed there's a whole thread about subprojects that I haven't read yet,
so this may have been addressed already ....

On Fri, May 18, 2007 at 11:53:12PM +0200, Alex Riesen wrote:
> Can we have this option (and corresponding support in the following
> patches, of course) first?

That's why the clone thing comes last.

> It is enough to have subprojects working
> locally, and people can start using them immediately: anyone can clone
> the subprojects manually if he wishes so.

Anyone can run git-write-tree and git-commit-tree is she wishes so...

> Cloning of subprojects is still unclear, and frankly I'm not sure it
> should be done at all. Not even with an option which is off by
> default.

Then don't use it.

The reason for not putting this in shouldn't be that someone doesn't
think it is useful; the reason should be that my code is crap.

skimo

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Alex Riesen]

Sven Verdoolaege, Sat, May 19, 2007 00:08:26 +0200:
> I noticed there's a whole thread about subprojects that I haven't read yet,
> so this may have been addressed already ....

Not the checkout, which is strange. It's mostly about cloning.

> On Fri, May 18, 2007 at 11:53:12PM +0200, Alex Riesen wrote:
> > Can we have this option (and corresponding support in the following
> > patches, of course) first?
> 
> That's why the clone thing comes last.
> 
> > It is enough to have subprojects working
> > locally, and people can start using them immediately: anyone can clone
> > the subprojects manually if he wishes so.
> 
> Anyone can run git-write-tree and git-commit-tree is she wishes so...

It is much more tedious. It have to be done recursively, and with
right SHA and you have to cd into right direcotry first and it is
git-read-tree and git-checkout-index, BTW.

IOW, it is hard.

> The reason for not putting this in shouldn't be that someone doesn't
> think it is useful; the reason should be that my code is crap.

The code is not a problem. It can be also discarded because you
implemented something no one wants.

I just meant to say, that even if no one wants your subproject cloning
code, _I_ support your checkout effort and I am asking for it to be
put in.

"First", as the cloning discussion does not seem to be finished (and,
as I said, I am not interested in cloning anyway).

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Alex Riesen <raa.lkml@gmail.com> writes:

> Sven Verdoolaege, Sat, May 19, 2007 00:08:26 +0200:
> ...
>> The reason for not putting this in shouldn't be that someone doesn't
>> think it is useful; the reason should be that my code is crap.
>
> The code is not a problem. It can be also discarded because you
> implemented something no one wants.

More specifically, at the very high level, what you do and what
people want to happen might share the description (e.g. "this
allows checkout of subprojects", or "this implements clone of
superproject to recurse") but with semantics that may be
different from what people would want (I am not saying that is
the case, as I do not think the current discussion concluded
yet).  The _first_ implementation that goes in the mainline
pretty much sets the _semantics_ so we would need to be extra
careful, all the more so as this is a feature many people seem
to want.

> I just meant to say, that even if no one wants your subproject cloning
> code, _I_ support your checkout effort and I am asking for it to be
> put in.
>
> "First", as the cloning discussion does not seem to be finished (and,
> as I said, I am not interested in cloning anyway).

It was partly my fault that I mentioned "clone" example in the
original message, and then let the discussion drifted to a
tangent of the clone topic, namely, how the URL would be
determined to clone the subproject from.

I would agree that checkout is a more fundamental operation, and
I wanted to make that clear in my message, but checkout and
clone has certain chicken-and-egg factor between them.  After
the clone of superproject, checking it out recursively would
need cloning the subprojects.  Also after a clone of
superproject without the recursive behaviour, when the user
explicitly asks a subproject to be checked out, somebody needs
to do a clone before the subject can be checked out.

Having said that, let's throw out an strawman for checkout
proper and then merge.

The user may or may not want to deal with subprojects, and for
something truly large like the KDE case, which is where the
superproject support is really needed, a large On/Off switch
where an option --subproject makes everything checked out and no
subproject is checked out without it is not a usable option.
I've already outlined how the .git/config file can be used to
define which subprojects are of interested so that the Porcelain
layer can decide which ones to recurse into and which ones to
leave alone.  The design is NOT the only possible/sensible one,
and I am sure other people will come up with much nicer
organization, but I would consider that is just the matter of
details.

Now, suppose "git checkout" needs to recurse into one
subdirectory that is to have a subproject.  There are three
cases:

 (1) There is no git repository yet (the plumbing layer already
     makes sure there is a directory, but does not do anything
     else).

 (2) There already is a git repository there, which is the
     correct repository (perhaps determined by .gitmodules and
     .git/config in the superproject, or presense of the commit
     that is recorded in the superproject's index).

 (3) There is a git repository but it is not the correct one.

We've discussed in the other thread about what to do in case
(1) to some degree.

For case (2), I think what should happen there is an equivalent
of this:

	$ commit=$(git-rev-parse :subproject)
        $ cd subproject
	$ git-rev-parse --verify $commit || git fetch || barf
        $ git checkout $commit

That is,

 - figure out what commit should be checked out from
   superproject index;

 - make sure the named commit exists, or fetch to make it exist.

 - go there and check out that commit; this implies two things:

   1. if there are local changes, it will be carried along and we
      checkout the named commit;

   2. the repository's HEAD becomes detached;

It is entirely possible that the repository is the _correct_ one
but not quite up to date, and you haven't fetched $commit.  This
is really a variant of (1) -- before being able to check out,
somebody has to clone the subproject.  Before being able to
check out to update the latest, somebody has to fetch in the
subproject.

If there are local changes, we would not at least lose them.  If
you want to get to a clean slate, you can cd there and perform
"git reset --hard".  If you want to mark that commit in the
subproject, you may want to do "git checkout -b branch" after
the recursive checkout from the superproject detached the HEAD
to the commit.

There is another variant that has already been suggested.  The
superproject tree and index could record 0{40} object name for
the subproject, and say "whatever commit happens to be at the
tip of the branch of subproject" (and most likely that URL and
branch information would come from .gitmodules and confirmed in
the .git/config file).  In such a case, the above outline would
be adjusted _BUT_ I think what would be checked out will not be
the named branch (e.g. refs/heads/master) itself, but the remote
branch that tracks it (e.g. refs/remotes/origin/master).

While I am at it, let me think aloud as to what I _think_ should
happen in a superproject merge.

 - Carry out the tree-level 3-way merge.  If it trivially
   resolves at the tree-level, we are happy.

 - There could be a case where the commit fetched/merged branch
   has and what the current branch has are different.  If one is
   a fast forward of the other, take it.

 - All other cases will leave the superproject index unmerged.

When the merge is cleanly done, the resulting commit is what we
should check out in the subproject directory (if we are
recursing into it, of course).

It is likely that in some cases you would want go to the
subproject directory and merge the commits at the subproject
from our branch and their branch in the superproject's index,
and make the resulting commit as the result of the merge for
that subproject path in the superproject, but I do not think it
is the only valid solution.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Shawn O. Pearce]

Junio C Hamano <junkio@cox.net> wrote:
> Now, suppose "git checkout" needs to recurse into one
> subdirectory that is to have a subproject.  There are three
> cases:

So I've implemented my own Git subproject support in a Java based
tool we use internally.  Its actually driving the Git plumbing (as
JGit isn't complete enough to do the job) but applies quite a bit
to this discussion as it is a working system that implements this
"checkout in superproject and recurse into subproject".

First I don't use the subproject support in the core plumbing,
because that came along from Linus about 2 days after I wrote
this implementation.  Our data file looks like:

	use-component: subproject1 >=df8cfac815...
	use-component: subproject2 >=af9b543820...

Or really anything that is a valid commit-ish, and often these are
actually just annotated tag names.

>  (1) There is no git repository yet (the plumbing layer already
>      makes sure there is a directory, but does not do anything
>      else).

During our build process we scan the root project's data file,
and clone by the relative URL anything we cannot find locally:

	$(git config remote.origin.url)/../component-links/subproject1.git

to get the subproject repository.  We don't require that the
subproject1 directory actually be called subproject1 in the
superproject, its just a recommendation.  That data file is also
our build-system driver and the build system driver is pretty darn
smart about guessing what is going on.  ;-)

You'll notice however that we (more or less) have a very flat
structure.  The component-links directory is really just a set of
symlinks pointing back up a level, as sometimes a component is not
stored in a repository named the component name, but the component
name matters to the build system.

>  (2) There already is a git repository there, which is the
>      correct repository (perhaps determined by .gitmodules and
>      .git/config in the superproject, or presense of the commit
>      that is recorded in the superproject's index).
> 
>  (3) There is a git repository but it is not the correct one.
> 
> For case (2), I think what should happen there is an equivalent
> of this:
> 
> 	$ commit=$(git-rev-parse :subproject)
>         $ cd subproject
> 	$ git-rev-parse --verify $commit || git fetch || barf
>         $ git checkout $commit

Yes.  Except we do a few things differently:

 - Only update the subproject if its a strict fast-forward.

 - Abort on a dirty working directory in the subproject or if a merge
 would be required to keep the current commit and the new commit.
 Yes, we don't really support going "backwards".

 - The merge aborting thing is probably wrong for some users,
 but blindly switching to the target commit feels somewhat wrong
 in our own uses.  Sometimes you need the current version of the
 subproject to help you debug an older version of the superproject,
 or sibling subproject.

 - You can't just checkout $commit if you can rev-parse it.
 You need to verify it and its entire reachable object set exists.
 See the local fetch fast-path thing you did recently in e3c6f240fd.

 - We update the user's current branch.  Because we are doing
 a strict fast-forward we're also assuming the user wants the
 current branch to stay correlated to the superproject branch.  Why?
 Most of our users keep the same branch name in all repositories.

-- 
Shawn.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Alex Riesen]

Junio C Hamano, Sat, May 19, 2007 05:59:48 +0200:
> 
>  - figure out what commit should be checked out from
>    superproject index;
> 
>  - make sure the named commit exists, or fetch to make it exist.

What if the fetch is not possible? You can't checkout? What about the
other subprojects, where the checkout succeeded? Will they be reset to
the previuos state?

To me, the fetch sounds pretty dangerous. Maybe the checkout should
be two stage: first - we check all subprojects to be checked out if it
is possible, second - either fail (default) or checkout what possible,
warn the user, leave the incomplete subprojects changed (but not
update the index with them, so that they wont be accidentally
committed).

>  - go there and check out that commit; this implies two things:
> 
>    1. if there are local changes, it will be carried along and we
>       checkout the named commit;

Shouldn't that depend on "-m" option given to git-checkout in
superproject? Sometime the user have to be sure he can checkout
everything as it were, but without breaking the local state (like what
current git-checkout without "-m" does).

>    2. the repository's HEAD becomes detached;
> 

Universally agreed upon

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Sven Verdoolaege]

On Fri, May 18, 2007 at 08:59:48PM -0700, Junio C Hamano wrote:
> Now, suppose "git checkout" needs to recurse into one
> subdirectory that is to have a subproject.  There are three
> cases:
> 
>  (1) There is no git repository yet (the plumbing layer already
>      makes sure there is a directory, but does not do anything
>      else).
> 
>  (2) There already is a git repository there, which is the
>      correct repository (perhaps determined by .gitmodules and
>      .git/config in the superproject, or presense of the commit
>      that is recorded in the superproject's index).
> 
>  (3) There is a git repository but it is not the correct one.
> 
> We've discussed in the other thread about what to do in case
> (1) to some degree.
> 
> For case (2), I think what should happen there is an equivalent
> of this:
> 
> 	$ commit=$(git-rev-parse :subproject)
>         $ cd subproject
> 	$ git-rev-parse --verify $commit || git fetch || barf
>         $ git checkout $commit

Does everyone agree that we should fetch (possibly after asking
for confirmation from the use) _during_ the checkout ?
I now only fetch submodules during a fetch of the supermodule
(actually, in my current patch set, I only fetch a submodule
the first time I see it, but that's a bug), but if there is
a consensus on this, I can switch to fetching during checkout.

As to the key to use to lookup the URL in the config, right
now I simply use the directory name where it is attached
(which seems like a useful default to me).
I'm not all that convinced that we should store a default URL
in history, so AFAICS, the only thing we need to store is a
mapping between directory names and subproject names.
It has been suggested to do that in .gitattributes.
Is that OK for everyone, or do we really need a separate .gitmodules ?

skimo

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Sven Verdoolaege <skimo@kotnet.org> writes:

> Does everyone agree that we should fetch (possibly after asking
> for confirmation from the use) _during_ the checkout ?
> I now only fetch submodules during a fetch of the supermodule
> (actually, in my current patch set, I only fetch a submodule
> the first time I see it, but that's a bug), but if there is
> a consensus on this, I can switch to fetching during checkout.

I think fetching of subproject during fetch or clone of
superproject would not make much sense.  Making it part of
superproject checkout would probably be the way we will end up
going.  The detail of "which part of the checkout" would need to
be defined, and I tend to agree with Alex that checkout itself
would need to be multi-phased, but I think that is a minor
implementation detail we can discuss after how the overall flows
should look like.

> As to the key to use to lookup the URL in the config, right
> now I simply use the directory name where it is attached
> (which seems like a useful default to me).

> I'm not all that convinced that we should store a default URL
> in history, so AFAICS, the only thing we need to store is a
> mapping between directory names and subproject names.
> It has been suggested to do that in .gitattributes.
> Is that OK for everyone, or do we really need a separate .gitmodules ?

If your (super)repository is _the_ only repository that knows
about the aggregation it is doing, I do not think you need
anything, as .git/config in the subproject would know where to
get updates from.  Otherwise, Project wide suggestions need to
be there in some machine readable form if you were to allow such
superproject distributed, be it in .gitmodules or
.gitattributes, don't they?

And frankly, I think .gitattributes is a wrong place to store
it, as its semantics is to give attributes to paths that MATCH
the glob.  You can argue that a pattern in .gitattributes can be
written to match only one path, but it still feels conceptually
wrong.  Something more concrete and exact, like the second level
key of .git/config and .gitmodules file format, is preferable.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Jan Hudec]

[-- Attachment #1: Type: text/plain, Size: 3583 bytes --]

On Sat, May 19, 2007 at 11:20:12 -0700, Junio C Hamano wrote:
> Sven Verdoolaege <skimo@kotnet.org> writes:
> 
> > Does everyone agree that we should fetch (possibly after asking
> > for confirmation from the use) _during_ the checkout ?
> > I now only fetch submodules during a fetch of the supermodule
> > (actually, in my current patch set, I only fetch a submodule
> > the first time I see it, but that's a bug), but if there is
> > a consensus on this, I can switch to fetching during checkout.
> 
> I think fetching of subproject during fetch or clone of
> superproject would not make much sense.  Making it part of
> superproject checkout would probably be the way we will end up
> going.  The detail of "which part of the checkout" would need to
> be defined, and I tend to agree with Alex that checkout itself
> would need to be multi-phased, but I think that is a minor
> implementation detail we can discuss after how the overall flows
> should look like.

IMHO it makes more sense to fetch during fetch of superproject:

 - If you don't fetch the superproject, it won't start refering to
   unavailable commit of subproject. So should only need to fetch subproject
   after fetching superproject.

 - If you fetch from more than one location, you want to fetch subproject
   from location corresponding to where you fetch superproject from.
   
   Let's have a repository of project P with remotes PA and PB. Let it have
   a subproject S with remotes SA and SB.

   Whenever I pull P from PA, it might refer to commit of S, that is only
   available from SA (because that's what PA owner uses). Whenever I pull
   P from PB, it might refer to commit of S, that is only available from SB
   (again because that's what PB owner uses).

   Now checkout does not know, whether I pulled the target revision from PA
   or PB, so:
    - Either it has to fetch both. But say the commit I want is in SB and SA
      contains a lot of new stuff, which will slow the thing down, though
      I don't need it.
    - Or it has to guess by looking whether any heads in remotes/PA or
      remotes/PB are descendants of the commit being checked out. But that
      feels rather hacky.

   I see several options:
    - Fetch will recurse. This should work ok and is IMHO least magic. We can
      also add some way to specify refspecs for the subproject, giving user
      control over what is fetched.
    - Fetch will store a "pending fetch from" note in the subproject and
      checkout, if it does not find the revision, will try fetching from all
      sources pointed to by those notes. There is still a problem with what
      exactly to fetch (user can specify in config).
    - Checkout will ask all subproject repositories whether they have given
      commit and pull the first one that does. This would get the needed
      commit most certainly. It would be slower though, because it would need
      to ask all the repositories whether they have the particular object.
      It also leaves the tracking branches in subproject in somewhat random
      state (maybe both repositories had the commit, so it pulled from the
      other one that user would etc.).

> > As to the key to use to lookup the URL in the config, right
> > now I simply use the directory name where it is attached
> > (which seems like a useful default to me).

The extra level of indirection has the advantage, that you can describe
moving the same subproject to a different directory.

-- 
						 Jan 'Bulb' Hudec <bulb@ucw.cz>

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Jan Hudec <bulb@ucw.cz> writes:

> IMHO it makes more sense to fetch during fetch of superproject:
>
>  - If you don't fetch the superproject, it won't start refering to
>    unavailable commit of subproject. So should only need to fetch subproject
>    after fetching superproject.

Eh, I was suggesting that the subproject fetch would come after
checkout in "fetch and then checkout" sequence of the
superproject, and if you are arguing against it, you should
justify why it should not happen before checkout, as we both
agree it should come after fetch of superproject.  Your argument
is like saying you have to git-init before doing anything so
you should fetch when you git-init.  That's not a justification.

>  - If you fetch from more than one location, you want to fetch subproject
>    from location corresponding to where you fetch superproject from.

Not at all.  There is no reason to believe that the case that
superproject and subproject come from related URLs is more
common.  One of the reasons to do a separated project
organization is to allow looser bindings of the project from
project administrative viewpoint. The integrator may not
necessarily have any control over what the subproject guys do,
and more importantly, the subproject people do not even care nor
be aware of the fact that their project is sometimes bound
inside other peoples' superprojects.  Think of the embedded
appliance vendor binding the kernel, libc and busybox in their
superproject that holds them together with the build
infrastructure. The kernel folks certainly do not particularly
care about the vendor.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Sven Verdoolaege]

On Sun, May 20, 2007 at 11:33:17AM -0700, Junio C Hamano wrote:
> Jan Hudec <bulb@ucw.cz> writes:
> >  - If you fetch from more than one location, you want to fetch subproject
> >    from location corresponding to where you fetch superproject from.
> 
> Not at all.  There is no reason to believe that the case that
> superproject and subproject come from related URLs is more
> common.  One of the reasons to do a separated project
> organization is to allow looser bindings of the project from
> project administrative viewpoint. The integrator may not
> necessarily have any control over what the subproject guys do,
> and more importantly, the subproject people do not even care nor
> be aware of the fact that their project is sometimes bound
> inside other peoples' superprojects.  Think of the embedded
> appliance vendor binding the kernel, libc and busybox in their
> superproject that holds them together with the build
> infrastructure. The kernel folks certainly do not particularly
> care about the vendor.

I think what Jan means is that if there are two (or more) copies
of the superproject then it is more likely that the subproject
commit can be found in the subproject repo "pointed to" (e.g.,
through my submodule.*.url) by the superproject repo you fetched from.

skimo

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Jan Hudec]

[-- Attachment #1: Type: text/plain, Size: 6611 bytes --]

Hello,

I gave the problem some more thought, and though I follow up on my previous
comment below, I can now see this:

So far it was discussed what should happen in fetch (+ checkout). But I think
the following are the interesting cases. Please read ALL before responding,
they are in somewhat random order:

 For following, assume there is a repo of project super has two branches,
 master and next. The next branch adds subproject sublib. In that state, I fetch
 refs/heads/*:refs/remotes/origin/* and check out master. So I have
 a repository, that contains revision with submodule, but I did not check it
 out yet.

 - Some time later, *without* fetching again, I simply
      git checkout --submodules -b feature1 remotes/origin/next
   Obviously it needs to give me the module.

 - Should that checkout work without network access?

 - Ok, now I start hacking on feature1 and find a bug in sublib, that I need
   to fix for it to work. Therefore I change something within sublib.

   However few days later I am asked to fix a bug in stable release of super.
   Therefore I: git checkout master Now, where does sublib go? It contains
   precious data!

   For the worst case situation assume, that the master branch also has
   directory sublib, so it can't stay where it was as unversioned.

 - The fix in master is done, back to our feature1, right?  git checkout
   --submodules feature1 Obviously re-fetching from upstram won't work. The
   head feature1 now refers to a commit that I made and only exists localy.

 - Now the maintainer of super wants to test the feature1. However sublib
   upstream did not accept the bugfix yet (and is perhaps waiting for
   confirmation, that the fixed version really works well for super, so we
   have to test).

   Therefore I push feature1 to my public repo, set up a public repo with my
   fixes to sublib and configure my public super repo to know about it.

   The maintainer already has a repo of super including sublib submodule. But
   when he pulls from me, he does not have the repo with my fixes.

 - The maintainer reviewed my feature1 and now needs to work on feature2.
   That however requires new upstream version of super. Therefore he needs to
   pull alternatively from both upstream and my repo with super, depending on
   what he works on.

   For the most complex case, assume here that I add more fixes to sublib
   while author of feature2 uses more and more bleeding edge stuff, so the
   maintainer really needs futher changes in sublib from both repos.

 - Also git has to fail safe if I forget to push the sublib, so when the
   maintainer tries to pull super, the refered revision of sublib simply
   won't be found.

I am not sure how to handle these cases. But they are cases that can happen
in real life and should be handled somehow. Even if some of them just require
some manual configuration.

Here is one possible idea:

We could store the GIT_DIR of submodule within the GIT_DIR of the
superproject instead of the submodule directory itself. So instead of:
 /
 /.git
 /subdir
 /subdir/.git

There would be:
 /
 /.git
 /subdir
 /.git/submodules/submodule-name.git

This would require changes to the logic how git finds GIT_DIR (which would be
really deep change), but it would provide place to store the submodule data
while the submodule is not being checked out. 

This does not address the last two cases above with mutliple sources, each
containing some revisions. There I see two options:

 - The submodules are fetched during superproject fetch (based on them being
   configured, even if they are not checked out) and the URL might depend on
   url configured for superproject. That is:
      git fetch --submodules foobar
   would do roughty:
      for GIT_DIR in $GIT_DIR/submodules/*.git; do
         git fetch foobar || git fetch
      done
   So if you configured source of the same name for the subproject, it would
   be pulled, otherwise the default one would.

   Checkout would then be local-only operation, because subprojects are
   up-to-date.

 - The superproject checkout would try fetching all sources of the
   subproject, until the requested revision is found.

   This could be extended to normal checkout doing it as well --
   "git checkout sha1" would try fetching all configured sources if the
   revision was not found.

Perhaps we could actually do both. That is, "git fetch --subprojects" to
also fetch all of "$GIT_DIR/submodules/*.git" and checkout to try fetching if
it can't find the desired revision.

On Sun, May 20, 2007 at 11:33:17 -0700, Junio C Hamano wrote:
> Jan Hudec <bulb@ucw.cz> writes:
> > IMHO it makes more sense to fetch during fetch of superproject:
> >
> >  - If you don't fetch the superproject, it won't start refering to
> >    unavailable commit of subproject. So should only need to fetch subproject
> >    after fetching superproject.
> 
> Eh, I was suggesting that the subproject fetch would come after
> checkout in "fetch and then checkout" sequence of the
> superproject, and if you are arguing against it, you should
> justify why it should not happen before checkout, as we both
> agree it should come after fetch of superproject.  Your argument
> is like saying you have to git-init before doing anything so
> you should fetch when you git-init.  That's not a justification.

It definitely has to come after fetch on superproject. My original thought
was, that it would be weird if it was part of the checkout itself, meaning
even checkout that does not follow a fetch. However I thought about it some
more and that might conflict with other requirements.

> >  - If you fetch from more than one location, you want to fetch subproject
> >    from location corresponding to where you fetch superproject from.
> 
> Not at all.  There is no reason to believe that the case that
> superproject and subproject come from related URLs is more
> common.  One of the reasons to do a separated project

I definitely don't think it's more common. But it's the harder case and it
might happen. Generally it will happen if some people work on both the
superproject and the subproject. Of course the argument is that than it
should not be separate projects, but maybe the teams just partly overlap.

Example of this situation is given above. IMHO it needs to be handled
somehow (probably git would have to check all potential sources whether they
have the revision in question).

-- 
						 Jan 'Bulb' Hudec <bulb@ucw.cz>

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Sven Verdoolaege]

On Mon, May 21, 2007 at 06:59:38PM +0200, Jan Hudec wrote:
> We could store the GIT_DIR of submodule within the GIT_DIR of the
> superproject instead of the submodule directory itself. So instead of:
>  /
>  /.git
>  /subdir
>  /subdir/.git
> 
> There would be:
>  /
>  /.git
>  /subdir
>  /.git/submodules/submodule-name.git

I have this already, except that I use /.git/submodules/submodule-name/.git
because I was too lazy to figure out how to get clone to use the above
without using --bare, because --bare disables separate-remotes.

Is there any good reason, btw for --bare not to do separate-remotes ?
We could throw out a lot of speical cases, especially the --bare http
fetch if we would simply always do a separate-remotes.

> This would require changes to the logic how git finds GIT_DIR (which would be
> really deep change),

Euhm.... I just add a symlink...

skimo

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Sven Verdoolaege <skimo@kotnet.org> writes:

> On Mon, May 21, 2007 at 06:59:38PM +0200, Jan Hudec wrote:
>> We could store the GIT_DIR of submodule within the GIT_DIR of the
>> superproject instead of the submodule directory itself. So instead of:
>>  /
>>  /.git
>>  /subdir
>>  /subdir/.git
>> 
>> There would be:
>>  /
>>  /.git
>>  /subdir
>>  /.git/submodules/submodule-name.git
>
> I have this already, except that I use /.git/submodules/submodule-name/.git
> because I was too lazy to figure out how to get clone to use the above
> without using --bare, because --bare disables separate-remotes.
>
> Is there any good reason, btw for --bare not to do separate-remotes ?

Traditinoally, the purpose of --bare is to set up distribution
points, either CVS style "shared public repository that
everybody pushes into and fetches from", or "owned by me and I
push my change into it to publish".  In either case, there is no
point of even having an upstream in such a repository, let
alone refs/remotes/origin/ hierarchy.

So that's the "good reason" part.

Having said that, last night I was thinking about making the
layout after a clone a bit more flexible and orthogonal to
existence of working tree.  What got me thinking about it was
different from your motivation, though.

If you have an ordinary working tree, and if you want to have a
clone of that working tree that you can work in, independently
(i.e. not treating the repository you cloned from any specially
from other repositories by marking it "origin"), currently there
is no straightforward way to do so, other than doing 
something like:

 (1) do a --bare clone, which would create "project.git";

 (2) mkdir "project" && mv "project.git" "project/.git";

 (3) edit "project/.git/config" and mark it as a non-bare
     repository.

 (4) "git checkout -f HEAD".

I'd call this a "pure" clone.  It is as close as the original
repository you would get without copying other per-repository
data such as .git/info, .git/config, so that you can start using
it as an _independent _repository.  It is like a --bare clone
but with a working tree associated with it.

What you want is probably the opposite.  A bare, but non-pure
clone.  By non-pure, what I mean is:

 (1) it has an upstream ("origin"), hence
 (2) it has refs/remotes/origin, and "remotes.origin.*"

If the word "non-pure" has a negative connotation, you could
call it "a follower repository".

In short, the current behaviour of --bare is "bare and pure",
and clone without --bare is "not bare, and a follower".  What I
wanted was "not bare, but pure", and I think what you want is
"bare, but a follower".

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Jan Hudec]

[-- Attachment #1: Type: text/plain, Size: 1188 bytes --]

On Mon, May 21, 2007 at 20:05:06 +0200, Sven Verdoolaege wrote:
> On Mon, May 21, 2007 at 06:59:38PM +0200, Jan Hudec wrote:
> > We could store the GIT_DIR of submodule within the GIT_DIR of the
> > superproject instead of the submodule directory itself. So instead of:
> >  /
> >  /.git
> >  /subdir
> >  /subdir/.git
> > 
> > There would be:
> >  /
> >  /.git
> >  /subdir
> >  /.git/submodules/submodule-name.git
> 
> I have this already, except that I use /.git/submodules/submodule-name/.git
> because I was too lazy to figure out how to get clone to use the above
> without using --bare, because --bare disables separate-remotes.
> 
> Is there any good reason, btw for --bare not to do separate-remotes ?
> We could throw out a lot of speical cases, especially the --bare http
> fetch if we would simply always do a separate-remotes.

Glad to hear it. I didn't really have time to read through all the patches.

> > This would require changes to the logic how git finds GIT_DIR (which would be
> > really deep change),
> 
> Euhm.... I just add a symlink...

Yes. Except I am not sure mingw supports that.

-- 
						 Jan 'Bulb' Hudec <bulb@ucw.cz>

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Martin Waitz]

[-- Attachment #1: Type: text/plain, Size: 2819 bytes --]

hoi :)

On Mon, May 21, 2007 at 06:59:38PM +0200, Jan Hudec wrote:
> Here is one possible idea:
> 
> We could store the GIT_DIR of submodule within the GIT_DIR of the
> superproject instead of the submodule directory itself. So instead of:
>  /
>  /.git
>  /subdir
>  /subdir/.git
> 
> There would be:
>  /
>  /.git
>  /subdir
>  /.git/submodules/submodule-name.git
> 
> This would require changes to the logic how git finds GIT_DIR (which would be
> really deep change), but it would provide place to store the submodule data
> while the submodule is not being checked out. 

I agree that we need something like that.

We don't have to move the entire subproject.git into the superproject,
but we need to have all _referenced_ objects in the .git dir of the
superproject.

There are several possibilities to do so:

 * move the entire .git dir
 * move .git/objects
 * explicitly copy all referenced objects

I have some experimental code to configure a per-subproject directory
in the superproject/.git as alternate object store for the submodule
to make the last two solutions possible.  Perhaps I should dig it out again
and adapt it to current git.

If there is a 1:1 relationship between subproject and object store then
even efficient fsck and repack/prune are possible for the submodule without
loosing objects.
But such a 1:1 relationship is bad when you move subprojects to another
location (or include the same subproject several times in different
locations of the tree).
Perhaps the user should be able to choose which one he wants.

> > Not at all.  There is no reason to believe that the case that
> > superproject and subproject come from related URLs is more
> > common.  One of the reasons to do a separated project
> 
> I definitely don't think it's more common. But it's the harder case and it
> might happen. Generally it will happen if some people work on both the
> superproject and the subproject. Of course the argument is that than it
> should not be separate projects, but maybe the teams just partly overlap.

I think it will be _very_ common to store super and subprojects in
related locations.  First to be independent from third-party servers
while working on the superproject.
Second (and I think more important) because many times there will
be superproject related adaptations in the subproject.  Yes they
are independent, and exactly for that reason the subproject upstream
maintainers may not take every change which is needed to satisfy the
superproject.  We _now_ see that in all Linux distributions already.
So when you use superprojects to integrate several independent projects,
then the superproject maintainer/administrator should really keep a
clone of all subprojects handy on his site.

-- 
Martin Waitz

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Jan Hudec]

[-- Attachment #1: Type: text/plain, Size: 3604 bytes --]

On Mon, May 21, 2007 at 23:11:34 +0200, Martin Waitz wrote:
> On Mon, May 21, 2007 at 06:59:38PM +0200, Jan Hudec wrote:
> > There would be:
> >  /
> >  /.git
> >  /subdir
> >  /.git/submodules/submodule-name.git
> > 
> > This would require changes to the logic how git finds GIT_DIR (which would be
> > really deep change), but it would provide place to store the submodule data
> > while the submodule is not being checked out. 
> 
> I agree that we need something like that.
> 
> We don't have to move the entire subproject.git into the superproject,
> but we need to have all _referenced_ objects in the .git dir of the
> superproject.
> 
> There are several possibilities to do so:
> 
>  * move the entire .git dir
>  * move .git/objects
>  * explicitly copy all referenced objects

I believe we really need entire .git dir. When the superporject checks out
revision which does not reference that subproject, we still need to preserve
not only the objects of subproject, but also the refs and config.

> I have some experimental code to configure a per-subproject directory
> in the superproject/.git as alternate object store for the submodule
> to make the last two solutions possible.  Perhaps I should dig it out again
> and adapt it to current git.
> 
> If there is a 1:1 relationship between subproject and object store then
> even efficient fsck and repack/prune are possible for the submodule without
> loosing objects.
> But such a 1:1 relationship is bad when you move subprojects to another
> location (or include the same subproject several times in different
> locations of the tree).
> Perhaps the user should be able to choose which one he wants.

That's why there should be the extra level of indirection using .gitmodules.
It should map the directory name to the object store name, so you can
relocate the subproject.

Including the same project several times is indeed interesting. Maybe the
subprojects should be "light checkouts" (I believe something like this was
already discussed on the list sometime). Those would be .git dirs, that would
only have HEAD and pointer to another .git dir with everything else.

> > > Not at all.  There is no reason to believe that the case that
> > > superproject and subproject come from related URLs is more
> > > common.  One of the reasons to do a separated project
> > 
> > I definitely don't think it's more common. But it's the harder case and it
> > might happen. Generally it will happen if some people work on both the
> > superproject and the subproject. Of course the argument is that than it
> > should not be separate projects, but maybe the teams just partly overlap.
> 
> I think it will be _very_ common to store super and subprojects in
> related locations.  First to be independent from third-party servers
> while working on the superproject.
> Second (and I think more important) because many times there will
> be superproject related adaptations in the subproject.  Yes they
> are independent, and exactly for that reason the subproject upstream
> maintainers may not take every change which is needed to satisfy the
> superproject.  We _now_ see that in all Linux distributions already.
> So when you use superprojects to integrate several independent projects,
> then the superproject maintainer/administrator should really keep a
> clone of all subprojects handy on his site.

Yes, repositories with distribution-specific patches will add a large class
of cases requiring multiple sources support.

-- 
						 Jan 'Bulb' Hudec <bulb@ucw.cz>

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Martin Waitz]

[-- Attachment #1: Type: text/plain, Size: 4232 bytes --]

On Tue, May 22, 2007 at 09:37:06PM +0200, Jan Hudec wrote:
> > We don't have to move the entire subproject.git into the superproject,
> > but we need to have all _referenced_ objects in the .git dir of the
> > superproject.
> > 
> > There are several possibilities to do so:
> > 
> >  * move the entire .git dir
> >  * move .git/objects
> >  * explicitly copy all referenced objects
> 
> I believe we really need entire .git dir. When the superporject checks out
> revision which does not reference that subproject, we still need to preserve
> not only the objects of subproject, but also the refs and config.

but all the other refs do not belong to the superproject.
For those who are working on the subproject there are of course a lot
of refs which they have to work with, but that can be dealt with
outside of the superproject scope.  The subproject is still a normal
Git repository, after all.
That is, you can have remote entries, branches and what not.
But all that is not interesting in the superproject scope.

So I thing moving the entire subproject.git into the superproject.git is too
much.  The superproject is only interested in the objects and in one
ref -- the one stored inside its tree.

> > I have some experimental code to configure a per-subproject directory
> > in the superproject/.git as alternate object store for the submodule
> > to make the last two solutions possible.  Perhaps I should dig it out again
> > and adapt it to current git.
> > 
> > If there is a 1:1 relationship between subproject and object store then
> > even efficient fsck and repack/prune are possible for the submodule without
> > loosing objects.
> > But such a 1:1 relationship is bad when you move subprojects to another
> > location (or include the same subproject several times in different
> > locations of the tree).
> > Perhaps the user should be able to choose which one he wants.
> 
> That's why there should be the extra level of indirection using .gitmodules.
> It should map the directory name to the object store name, so you can
> relocate the subproject.
> 
> Including the same project several times is indeed interesting. Maybe the
> subprojects should be "light checkouts" (I believe something like this was
> already discussed on the list sometime). Those would be .git dirs, that would
> only have HEAD and pointer to another .git dir with everything else.

Well, even if they might share a lot of objects they might be included
for completely different reasons and so might need to work with
different communities (remote entries, branches, etc.).

So sharing objects makes sense, sharing the rest of .git is not
neccessary.

> > I think it will be _very_ common to store super and subprojects in
> > related locations.  First to be independent from third-party servers
> > while working on the superproject.
> > Second (and I think more important) because many times there will
> > be superproject related adaptations in the subproject.  Yes they
> > are independent, and exactly for that reason the subproject upstream
> > maintainers may not take every change which is needed to satisfy the
> > superproject.  We _now_ see that in all Linux distributions already.
> > So when you use superprojects to integrate several independent projects,
> > then the superproject maintainer/administrator should really keep a
> > clone of all subprojects handy on his site.
> 
> Yes, repositories with distribution-specific patches will add a large class
> of cases requiring multiple sources support.

You don't really need multiple sources for it.
The subproject contains both upstream and local changes, but I think
it makes sense to keep the entire object store local (the same way
to keep all the entire history local even if you only want to add to it
in a normal repository).  Those people who work on the subproject and
communicate with its upstream developers of course need remote entries
and have to synchronize the subproject with upstream.  But that is
not related to the superproject at all.

So yes, you have different sources but you don't need extra support
in the subproject implementation for it.

-- 
Martin Waitz

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Jakub Narebski]

Martin Waitz wrote:

> On Tue, May 22, 2007 at 09:37:06PM +0200, Jan Hudec wrote:

>> Including the same project several times is indeed interesting. Maybe the
>> subprojects should be "light checkouts" (I believe something like this was
>> already discussed on the list sometime). Those would be .git dirs, that would
>> only have HEAD and pointer to another .git dir with everything else.
> 
> Well, even if they might share a lot of objects they might be included
> for completely different reasons and so might need to work with
> different communities (remote entries, branches, etc.).
> 
> So sharing objects makes sense, sharing the rest of .git is not
> neccessary.

One of the final ideas for "lightweight checkout" was having in
.git/config the location of "true" $GIT_DIR (or parts of it: 
GIT_INDEX_FILE, GIT_OBJECT_DIRECTORY), and "shadowing" the rest
of "true $GIT_DIR" with what is present in .git. It means that
you can have .git/index and .git/HEAD, and if you don't find
appropriate .git/refs/heads/master file you look to "true $GIT_DIR".

-- 
Jakub Narebski
Warsaw, Poland
ShadeHawk on #git

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Jan Hudec]

[-- Attachment #1: Type: text/plain, Size: 5640 bytes --]

On Thu, May 24, 2007 at 17:48:33 +0200, Martin Waitz wrote:
> On Tue, May 22, 2007 at 09:37:06PM +0200, Jan Hudec wrote:
> > > We don't have to move the entire subproject.git into the superproject,
> > > but we need to have all _referenced_ objects in the .git dir of the
> > > superproject.
> > > 
> > > There are several possibilities to do so:
> > > 
> > >  * move the entire .git dir
> > >  * move .git/objects
> > >  * explicitly copy all referenced objects
> > 
> > I believe we really need entire .git dir. When the superporject checks out
> > revision which does not reference that subproject, we still need to preserve
> > not only the objects of subproject, but also the refs and config.
> 
> but all the other refs do not belong to the superproject.
> For those who are working on the subproject there are of course a lot
> of refs which they have to work with, but that can be dealt with
> outside of the superproject scope.  The subproject is still a normal
> Git repository, after all.
> That is, you can have remote entries, branches and what not.
> But all that is not interesting in the superproject scope.
> 
> So I thing moving the entire subproject.git into the superproject.git is too
> much.  The superproject is only interested in the objects and in one
> ref -- the one stored inside its tree.

No, the way I mean it the subproject and superproject don't share a single
bit. The subproject.git dir is subdirectory of superproject.git, but has no
thing in common with it.

> > > I have some experimental code to configure a per-subproject directory
> > > in the superproject/.git as alternate object store for the submodule
> > > to make the last two solutions possible.  Perhaps I should dig it out again
> > > and adapt it to current git.

Ah. My bad. Didn't notice this. I do NOT want to share any objects between
subproject and superproject. At least not unless the user explicitely asks
for that, which might make sense if the subproject was carved out of the
superproject.

> > > If there is a 1:1 relationship between subproject and object store then
> > > even efficient fsck and repack/prune are possible for the submodule without
> > > loosing objects.
> > > But such a 1:1 relationship is bad when you move subprojects to another
> > > location (or include the same subproject several times in different
> > > locations of the tree).
> > > Perhaps the user should be able to choose which one he wants.
> > 
> > That's why there should be the extra level of indirection using .gitmodules.
> > It should map the directory name to the object store name, so you can
> > relocate the subproject.
> > 
> > Including the same project several times is indeed interesting. Maybe the
> > subprojects should be "light checkouts" (I believe something like this was
> > already discussed on the list sometime). Those would be .git dirs, that would
> > only have HEAD and pointer to another .git dir with everything else.
> 
> Well, even if they might share a lot of objects they might be included
> for completely different reasons and so might need to work with
> different communities (remote entries, branches, etc.).
> 
> So sharing objects makes sense, sharing the rest of .git is not
> neccessary.

No, I didn't mean the subproject and superproject would share anything.
The case I talk about is if project foo has subdirs A and B and they both
contain (different revisions of) the same subproject. The .gitmodules
definition is:
    [submodule "A"]
    name=bar
    [submodule "B"]
    name=bar

In such case A/.git and B/.git can't both be symlinks to
.git/subprojects/bar.git, because they have different HEAD, but everything
else should be defined by .git/subprojects/bar.git

> > > I think it will be _very_ common to store super and subprojects in
> > > related locations.  First to be independent from third-party servers
> > > while working on the superproject.
> > > Second (and I think more important) because many times there will
> > > be superproject related adaptations in the subproject.  Yes they
> > > are independent, and exactly for that reason the subproject upstream
> > > maintainers may not take every change which is needed to satisfy the
> > > superproject.  We _now_ see that in all Linux distributions already.
> > > So when you use superprojects to integrate several independent projects,
> > > then the superproject maintainer/administrator should really keep a
> > > clone of all subprojects handy on his site.
> > 
> > Yes, repositories with distribution-specific patches will add a large class
> > of cases requiring multiple sources support.
> 
> You don't really need multiple sources for it.
> The subproject contains both upstream and local changes, but I think

Upstream + local is not the interesting case. Multiple upstreams is.

> it makes sense to keep the entire object store local (the same way
> to keep all the entire history local even if you only want to add to it
> in a normal repository).  Those people who work on the subproject and
> communicate with its upstream developers of course need remote entries
> and have to synchronize the subproject with upstream.  But that is
> not related to the superproject at all.
> 
> So yes, you have different sources but you don't need extra support
> in the subproject implementation for it.

Well, if the subproject is not auto-fetched, there's no need for extra
support. But if there is auto-fetch, it should be aware of possibility to
have multiple upstreams.

-- 
						 Jan 'Bulb' Hudec <bulb@ucw.cz>

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Martin Waitz <tali@admingilde.org> writes:

> On Mon, May 21, 2007 at 06:59:38PM +0200, Jan Hudec wrote:
>> Here is one possible idea:
>> 
>> We could store the GIT_DIR of submodule within the GIT_DIR of the
>> superproject instead of the submodule directory itself. So instead of:
>>  /
>>  /.git
>>  /subdir
>>  /subdir/.git
>> 
>> There would be:
>>  /
>>  /.git
>>  /subdir
>>  /.git/submodules/submodule-name.git
>> 
>> This would require changes to the logic how git finds GIT_DIR (which would be
>> really deep change), but it would provide place to store the submodule data
>> while the submodule is not being checked out. 
>
> I agree that we need something like that.
>
> We don't have to move the entire subproject.git into the superproject,
> but we need to have all _referenced_ objects in the .git dir of the
> superproject.
>
> There are several possibilities to do so:
>
>  * move the entire .git dir
>  * move .git/objects
>  * explicitly copy all referenced objects

I was hoping that we can start from an initial cut that supports
only a superproject that had its subprojects in their places
from its initial commit, and did not have to worry about this
from day one, and deal with this kind of "more advanced" stuff
incrementally.  Unfortunately it's more fun to talk about more
advanced stuff than starting with small but solid stuff.

And we would need to make sure whatever we do as the "small but
solid" initial round can later support more advanced
arrangements later, so we would need to think about the issues
now anyway to a certain degree.

How about doing something like this, instead?

 (1) superproject .gitmodules (in-tree) and .git/config (local
     repository) use the three-level naming in $gmane/47567.
     Namely, (1a) .gitmodules says which subdirectory has a
     checkout of what project, and names the project in
     logical/abstract terms, not with a URL (e.g. "kernel26");
     (1b) .gitmodules also associates a set of suggested URLs
     for each of the logical/abstract project name; (1c)
     .git/config records which project are of interest.

 (2) In superproject .git/, we would have a bare repository for
     each project used by the superproject.

	.git/subproject/kernel26/{objects,refs,...}

     This is created by making a bare clone from the upstream
     URL, decided by the user with the help from suggested URL
     described in the superproject .gitmodules.

     The idea is to use this repository as a long-term
     subproject state across branch switching.

 (3) When we need to check out a revision of superproject whose
     .gitmodules has "kernel-src/ -> kernel26", and when we
     haven't done so (perhaps we are doing an initial checkout,
     perhaps we are switching from a different revision of the
     superproject that did not have "kernel26" project at
     kernel-src/ directory), we rm -f kernel-src/ and then
     "git-clone -l -s" from the repository we keep in (2) to
     populate kernel-src/ directory.

 (4) Before performing the above step (3), we need to make sure
     we are not losing anything in kernel-src/ if exists.  Three
     cases plus one:

     (4a) The path kernel-src/ in the old checkout was not a
     subproject (either it did not exist, it was a blob, or it
     was a directory with files that are tracked as part of the
     superproject).  The usual "don't lose local modification"
     rule we use try to carry local changes forward across
     branch switching, but in this case we shouldn't do so.

     (4b) It has the same logical/abstract project checked out;
     the commit recorded in the superproject tree may or may not
     be the same as what its HEAD points at.  In this case we do
     not have to worry about swapping the git repository at
     kernel-src/ directory, although we would need to check out
     the correct revision, and worry about what to do with any
     local modification (I think the usual "don't lose local
     modification but carry them forward" rule would be Ok in
     this case).

     (4c) It has a different project checked out; we need to be
     careful to keep local changes, and also we need to make
     sure the local changes in this subproject repository are
     pushed back to (2).  It could be that automated "git push"
     after making sure everything is committed is sufficient and
     have the user handle failure cases.

     (4d) This applies not just "before step (3)", but in cases
     where we need to replace a checked out subproject directory
     with something else (e.g. blob or directory that belong to
     the superproject, or noneness).  We would need to make sure
     no local change is lost, and the repository is synched up
     with (2).

I think an arrangement like this would solve "symlink is a bitch
for MinGW" problem Johannes Sixt brought up today with Sven's
RFC as well.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Sven Verdoolaege]

On Thu, May 24, 2007 at 11:26:01AM -0700, Junio C Hamano wrote:
> How about doing something like this, instead?
> 
>  (1) superproject .gitmodules (in-tree) and .git/config (local
>      repository) use the three-level naming in $gmane/47567.
>      Namely, (1a) .gitmodules says which subdirectory has a
>      checkout of what project, and names the project in
>      logical/abstract terms, not with a URL (e.g. "kernel26");
>      (1b) .gitmodules also associates a set of suggested URLs
>      for each of the logical/abstract project name; (1c)
>      .git/config records which project are of interest.

What about the idea of considering any project that is already
present to be of interest by default ?

>  (2) In superproject .git/, we would have a bare repository for
>      each project used by the superproject.
> 
> 	.git/subproject/kernel26/{objects,refs,...}
> 
>      This is created by making a bare clone from the upstream
>      URL, decided by the user with the help from suggested URL
>      described in the superproject .gitmodules.

Do you mean a "pure" clone, i.e., without a working tree,
but with separate-remotes?

>      (4b) It has the same logical/abstract project checked out;
>      the commit recorded in the superproject tree may or may not
>      be the same as what its HEAD points at.  In this case we do
>      not have to worry about swapping the git repository at
>      kernel-src/ directory, although we would need to check out
>      the correct revision, and worry about what to do with any
>      local modification (I think the usual "don't lose local
>      modification but carry them forward" rule would be Ok in
>      this case).

We may also need to fetch from the remote subproject.
Should I do this with a "git fetch" during the checkout as I do now?

skimo

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Sven Verdoolaege <skimo@kotnet.org> writes:

> On Thu, May 24, 2007 at 11:26:01AM -0700, Junio C Hamano wrote:
>> How about doing something like this, instead?
>> 
>>  (1) superproject .gitmodules (in-tree) and .git/config (local
>>      repository) use the three-level naming in $gmane/47567.
>>      Namely, (1a) .gitmodules says which subdirectory has a
>>      checkout of what project, and names the project in
>>      logical/abstract terms, not with a URL (e.g. "kernel26");
>>      (1b) .gitmodules also associates a set of suggested URLs
>>      for each of the logical/abstract project name; (1c)
>>      .git/config records which project are of interest.
>
> What about the idea of considering any project that is already
> present to be of interest by default ?

You could do that -- I consider that a minor detail in the
implementation.

>>  (2) In superproject .git/, we would have a bare repository for
>>      each project used by the superproject.
>> 
>> 	.git/subproject/kernel26/{objects,refs,...}
>> 
>>      This is created by making a bare clone from the upstream
>>      URL, decided by the user with the help from suggested URL
>>      described in the superproject .gitmodules.
>
> Do you mean a "pure" clone, i.e., without a working tree,
> but with separate-remotes?

I meant a bare clone without separate remotes.

The counter-proposal outline essentially says, for the sake of
simplicity, "nuke existing subproject directory whenever we need
to replace it with something else, and reclone a new/replacement
subproject directory every time we need to check it out, after
making sure nothing is lost".

Except that having this intermediate repository (a) helps making
such a re-clone extremely cheap "git clone -l -s -n", and (b)
provides with a place to hold locally committed changes that are
not pushed back to the true upstream (you may _never_ push it
back in the first place).

>>      (4b) It has the same logical/abstract project checked out;
>>      the commit recorded in the superproject tree may or may not
>>      be the same as what its HEAD points at.  In this case we do
>>      not have to worry about swapping the git repository at
>>      kernel-src/ directory, although we would need to check out
>>      the correct revision, and worry about what to do with any
>>      local modification (I think the usual "don't lose local
>>      modification but carry them forward" rule would be Ok in
>>      this case).
>
> We may also need to fetch from the remote subproject.
> Should I do this with a "git fetch" during the checkout as I do now?

If you are disconnected, you obviously cannot afford to update
(2) every time you switch branches in the superproject, and even
if you are connected, updating (2) would not be needed most of
the time if what you are doing is switching superproject
branches, as long as your last superproject fetch is not very
much more recent than your last subproject fetch.

If we were to follow the outline in the counter-proposal, I'd
imagine that update of (2) can happen at any time.  It could be
part of "git fetch" in superprojects, of lazily done when we
need to checkout a new revision for a particular subproject, but
only if the last time you fetched superproject is more recent
than the time you updated (2) for the subproject last time.

Or something like that.  I consider that also a minor detail in
the implementation.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Sven Verdoolaege]

On Thu, May 24, 2007 at 11:58:26AM -0700, Junio C Hamano wrote:
> Sven Verdoolaege <skimo@kotnet.org> writes:
> > On Thu, May 24, 2007 at 11:26:01AM -0700, Junio C Hamano wrote:
> >>  (2) In superproject .git/, we would have a bare repository for
> >>      each project used by the superproject.
> >> 
> >> 	.git/subproject/kernel26/{objects,refs,...}
> >> 
> >>      This is created by making a bare clone from the upstream
> >>      URL, decided by the user with the help from suggested URL
> >>      described in the superproject .gitmodules.
> >
> > Do you mean a "pure" clone, i.e., without a working tree,
> > but with separate-remotes?
> 
> I meant a bare clone without separate remotes.

Why without separate remotes?
It has been argued before that changes in the subproject
may come from different remotes, so the user may want
to configure extra remotes from which to fetch.

> The counter-proposal outline essentially says, for the sake of
> simplicity, "nuke existing subproject directory whenever we need
> to replace it with something else, and reclone a new/replacement
> subproject directory every time we need to check it out, after
> making sure nothing is lost".

And she can't do it in the clone in his working tree if that's
going to get nuked from time to time.

> If we were to follow the outline in the counter-proposal, I'd
> imagine that update of (2) can happen at any time.  It could be
> part of "git fetch" in superprojects, of lazily done when we
> need to checkout a new revision for a particular subproject, but
> only if the last time you fetched superproject is more recent
> than the time you updated (2) for the subproject last time.
> 
> Or something like that.  I consider that also a minor detail in
> the implementation.

But you still need figure out _what_ to fetch.
Before you suggested to just use the default set up by
clone with separate remotes, but you no longer have that
in your new proposal.

skimo

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Sven Verdoolaege <skimo@kotnet.org> writes:

[side note: I am ignoring your reply-to: liacs.nl as its MTA
seems to use sorbs that has my ISP's outgoing sender identified
as spam source; I'll send the bounce to you privately in a
separate message.]

> On Thu, May 24, 2007 at 11:58:26AM -0700, Junio C Hamano wrote:
>> Sven Verdoolaege <skimo@kotnet.org> writes:
>> > On Thu, May 24, 2007 at 11:26:01AM -0700, Junio C Hamano wrote:
>> >>  (2) In superproject .git/, we would have a bare repository for
>> >>      each project used by the superproject.
>> >> 
>> >> 	.git/subproject/kernel26/{objects,refs,...}
>> >> 
>> >>      This is created by making a bare clone from the upstream
>> >>      URL, decided by the user with the help from suggested URL
>> >>      described in the superproject .gitmodules.
>> >
>> > Do you mean a "pure" clone, i.e., without a working tree,
>> > but with separate-remotes?
>> 
>> I meant a bare clone without separate remotes.
>
> Why without separate remotes?
> It has been argued before that changes in the subproject
> may come from different remotes, so the user may want
> to configure extra remotes from which to fetch.

By different remotes, which do you mean?

 (1) .git/subprojects/kernel26/ repository has 'origin'
     different from any of the suggested URL in .gitmodules, but
     as far as it is concerned there is one 'origin';

or

 (2) it has 'origin' that is what the superproject suggests, but
     the user locally uses additional repositories to pull and
     merge from;

If the former that is not an argument, so I'd assume the
latter.  I would say in such a case, you are better off having a
usual repository to manage the development of subproject part,
not grafted to any superproject repository, and handle such
merges there (after all, a "subproject" should stand on its own
without having any of the superproject stuff).  And treat THAT
repository as the 'origin' used in (1) above.  It might be
easier to use non separate-remote layout in the standalone
repository for the subproject, but that is a separate issue.

>> The counter-proposal outline essentially says, for the sake of
>> simplicity, "nuke existing subproject directory whenever we need
>> to replace it with something else, and reclone a new/replacement
>> subproject directory every time we need to check it out, after
>> making sure nothing is lost".
>
> And she can't do it in the clone in his working tree if that's
> going to get nuked from time to time.

And she does not have to.  She can do the development/fixes in
(temporarily) checked out subproject tree, and push it back to
the .git/subproject/kernel26/ repository in the superproject
before she leaves (i.e. before branch switching at superproject
level needs to obliterate it).  The change stored in the
.git/subproject/kernel26/ repository in the superproject can
further be pushed back to its 'origin', be it the true
"upstream", or "the standalone repository for the subproject" I
mentioned above.

> But you still need figure out _what_ to fetch.
> Before you suggested to just use the default set up by
> clone with separate remotes, but you no longer have that
> in your new proposal.

I do remember saying the "default set up by clone" but I did not
mean separate remotes.

What is fetched by a bare and non-separate-remote repository vs
a repository that uses separate-remote layout from 'origin' is
exactly the same -- the difference is only 'pure/bare' layout
would use

	fetch = refs/heads/*:refs/heads/*

while separate-remotes would use

	fetch = refs/heads/*:/refs/remotes/origin/*

So I do not think the difference matters for our purpose of
being able to check out commits that are referenced in
superproject trees.  As long as we require that the 'origin' for
"longer term repository to keep track of the subproject in
superproject" (aka repository (2) in my message you are
responding to) always contain the commit referenced by tree
objects in the superproject, which I think is a sensible thing
to require (otherwise you cannot even clone and checkout the
whole superproject), both layout would work equally well.  It's
just bare/pure layout is easier to understand because it is
essentially a "mirror" of the upstream.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Petr Baudis]

On Thu, May 24, 2007 at 10:32:32PM CEST, Junio C Hamano wrote:
> And she does not have to.  She can do the development/fixes in
> (temporarily) checked out subproject tree, and push it back to
> the .git/subproject/kernel26/ repository in the superproject
> before she leaves (i.e. before branch switching at superproject
> level needs to obliterate it).  The change stored in the
> .git/subproject/kernel26/ repository in the superproject can
> further be pushed back to its 'origin', be it the true
> "upstream", or "the standalone repository for the subproject" I
> mentioned above.
..snip..
> It's just bare/pure layout is easier to understand because it is
> essentially a "mirror" of the upstream.

I OTOH think that it's less confusing to just keep it the same for all
the normal repositories - and you yourself did not rule out the
possibility to have local changes in the repository, in which case we
certainly should use the separate-remotes layout or we go back to last
year's refs mess.

-- 
				Petr "Pasky" Baudis
Stuff: http://pasky.or.cz/
Ever try. Ever fail. No matter. // Try again. Fail again. Fail better.
		-- Samuel Beckett

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Petr Baudis <pasky@suse.cz> writes:

> On Thu, May 24, 2007 at 10:32:32PM CEST, Junio C Hamano wrote:
> ...
>> It's just bare/pure layout is easier to understand because it is
>> essentially a "mirror" of the upstream.
>
> I OTOH think that it's less confusing to just keep it the same for all
> the normal repositories - and you yourself did not rule out the
> possibility to have local changes in the repository, in which case we
> certainly should use the separate-remotes layout or we go back to last
> year's refs mess.

That "local changes in the repository" will happen in the clone
created in kernel-src/ directory and I do not have any problem
using separate remotes there.  I was talking about the refs
layout used in .git/subproject/kernel26/, which is a mirror of
the upstream and the clone source for that (temporarily checked
out) repository at kernel-src/ directory.  After all, the
"mirror" is where you would push your local changes back into,
and it is more common to use non-separate-remote layout (without
even 'master'->'origin' mapping, hence I had to come up with a
new name 'pure', but I realize that what I meant really is a
'mirror') there.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Junio C Hamano <junkio@cox.net> writes:

Sorry, I hate to send "amend" message like this, but there are
some places I was not clear enough...

> How about doing something like this, instead?
>
>  (1) superproject .gitmodules (in-tree) and .git/config (local
>      repository) use the three-level naming in $gmane/47567.
>      Namely, (1a) .gitmodules says which subdirectory has a
>      checkout of what project, and names the project in
>      logical/abstract terms, not with a URL (e.g. "kernel26");
>      (1b) .gitmodules also associates a set of suggested URLs
>      for each of the logical/abstract project name; (1c)
>      .git/config records which project are of interest.

... are of interest, and perhaps which URL to use but that is
not necessary, as we can record that in repositories we create
in (2).

>  (2) In superproject .git/, we would have a bare repository for
>      each project used by the superproject.
>
> 	.git/subproject/kernel26/{objects,refs,...}
>
>      This is created by making a bare clone from the upstream
>      URL, decided by the user with the help from suggested URL
>      described in the superproject .gitmodules.

This bare clone probably have its own config that records
remote.origin.url, which would probably be the same as one of
the URL suggested in .gitmodules for many people, but could be
different.  As to what should happen when the .gitmodules that
comes from the upstream changes the set of suggested URLs
recorded in there, see $gmane/47621,

>      The idea is to use this repository as a long-term
>      subproject state across branch switching.
>
>  (3) When we need to check out a revision of superproject whose
>      .gitmodules has "kernel-src/ -> kernel26", and when we
>      haven't done so (perhaps we are doing an initial checkout,
>      perhaps we are switching from a different revision of the
>      superproject that did not have "kernel26" project at
>      kernel-src/ directory), we rm -f kernel-src/ and then
>      "git-clone -l -s" from the repository we keep in (2) to
>      populate kernel-src/ directory.

The last part of the above should have been:

       we "rm -fr kernel-src/" and then "git-clone -l -s -n"
       from the repository we keep in (2).  And then check out
       the commit that is pointed at by the superproject tree
       (rather, "index").

>  (4) Before performing the above step (3), we need to make sure
>      we are not losing anything in kernel-src/ if exists.  Three
>      cases plus one:
>
>      (4a) The path kernel-src/ in the old checkout was not a
>      subproject (either it did not exist, it was a blob, or it
>      was a directory with files that are tracked as part of the
>      superproject).  The usual "don't lose local modification"
>      rule we use try to carry local changes forward across
>      branch switching, but in this case we shouldn't do so.

The last part of the above should have been:

       ... but in this case we shouldn't even try to carry local
       changes forward, as there is nowhere to carry them to.
       Instead we should error out and have the user clean-up
       the subproject repository (either "git reset", or "git
       commit && git push" to stash the change back to
       repository we prepared in (2)).  We might also want to
       allow the user to say "local changes in this subproject
       checkout does not matter" by saying "git checkout -f
       $commit" at the superproject level, but I think that is
       rather dangerous.

>      (4b) It has the same logical/abstract project checked out;
> ...
>      (4c) It has a different project checked out; we need to be
>      careful to keep local changes, and also we need to make
>      sure the local changes in this subproject repository are
>      pushed back to (2).  It could be that automated "git push"
>      after making sure everything is committed is sufficient and
>      have the user handle failure cases.
>
>      (4d) This applies not just "before step (3)", but in cases
>      where we need to replace a checked out subproject directory
>      with something else (e.g. blob or directory that belong to
>      the superproject, or noneness).  We would need to make sure
>      no local change is lost, and the repository is synched up
>      with (2).

... which is the same check and action we would do in case (4c).

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Petr Baudis]

On Thu, May 24, 2007 at 08:26:01PM CEST, Junio C Hamano wrote:
> How about doing something like this, instead?

These discussions are spread over so many posts (and especially threads)
that it's far beyond me to track it all - I hope I won't repeat
something already debunked.

>  (1) superproject .gitmodules (in-tree) and .git/config (local
>      repository) use the three-level naming in $gmane/47567.
>      Namely, (1a) .gitmodules says which subdirectory has a
>      checkout of what project, and names the project in
>      logical/abstract terms, not with a URL (e.g. "kernel26");
>      (1b) .gitmodules also associates a set of suggested URLs
>      for each of the logical/abstract project name; (1c)
>      .git/config records which project are of interest.

How do you deal with clashes in subproject names? Until now, several
independent projects might live happily in a single repository, this
breaks that. When merging disparate projects, you can resolve name
clashes in .gitmodules, but going back to one of the merged trunks just
won't work. It's ugly.

Now, we can just declare that we don't care about this case. This stance
wouldn't make me comfortable at all, but at least we should make it
clear that we know about this problem and the users are on their own
when it happens.

>  (2) In superproject .git/, we would have a bare repository for
>      each project used by the superproject.
> 
> 	.git/subproject/kernel26/{objects,refs,...}
> 
>      This is created by making a bare clone from the upstream
>      URL, decided by the user with the help from suggested URL
>      described in the superproject .gitmodules.
> 
>      The idea is to use this repository as a long-term
>      subproject state across branch switching.

When not using subproject aliases, you could just name this after some
normalized form of the URL (anything suitable up to an sha1sum of the
URL :).

-- 
				Petr "Pasky" Baudis
Stuff: http://pasky.or.cz/
Ever try. Ever fail. No matter. // Try again. Fail again. Fail better.
		-- Samuel Beckett

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Jan Hudec]

[-- Attachment #1: Type: text/plain, Size: 5170 bytes --]

On Thu, May 24, 2007 at 11:26:01 -0700, Junio C Hamano wrote:
> Martin Waitz <tali@admingilde.org> writes:
> > On Mon, May 21, 2007 at 06:59:38PM +0200, Jan Hudec wrote:
> [...]
> 
> I was hoping that we can start from an initial cut that supports
> only a superproject that had its subprojects in their places
> from its initial commit, and did not have to worry about this
> from day one, and deal with this kind of "more advanced" stuff
> incrementally.  Unfortunately it's more fun to talk about more
> advanced stuff than starting with small but solid stuff.
> 
> And we would need to make sure whatever we do as the "small but
> solid" initial round can later support more advanced
> arrangements later, so we would need to think about the issues
> now anyway to a certain degree.
> 
> How about doing something like this, instead?

It's almost exactly what I had in mind, except much better described, so
I definitely support this. It seems that it can work.

>  (1) superproject .gitmodules (in-tree) and .git/config (local
>      repository) use the three-level naming in $gmane/47567.
>      Namely, (1a) .gitmodules says which subdirectory has a
>      checkout of what project, and names the project in
>      logical/abstract terms, not with a URL (e.g. "kernel26");
>      (1b) .gitmodules also associates a set of suggested URLs
>      for each of the logical/abstract project name; (1c)
>      .git/config records which project are of interest.
> 
>  (2) In superproject .git/, we would have a bare repository for
>      each project used by the superproject.
> 
> 	.git/subproject/kernel26/{objects,refs,...}
> 
>      This is created by making a bare clone from the upstream
>      URL, decided by the user with the help from suggested URL
>      described in the superproject .gitmodules.
> 
>      The idea is to use this repository as a long-term
>      subproject state across branch switching.
> 
>  (3) When we need to check out a revision of superproject whose
>      .gitmodules has "kernel-src/ -> kernel26", and when we
>      haven't done so (perhaps we are doing an initial checkout,
>      perhaps we are switching from a different revision of the
>      superproject that did not have "kernel26" project at
>      kernel-src/ directory), we rm -f kernel-src/ and then
>      "git-clone -l -s" from the repository we keep in (2) to
>      populate kernel-src/ directory.

If the "clone" could also share the refs, config and everything except HEAD,
it would make it completely (or almost so) transparent to the user. Making it
non-transparent will work well enough though and should not require any new
changes.

The problem is, that than the HEAD could get out of sync with the refs
(because they are linked from other repo), so it would have to remember both
the commit name and the symbolic ref. I would behave as symref only if the
commit name in it and the target are the same and as hard ref otherwise.

>  (4) Before performing the above step (3), we need to make sure
>      we are not losing anything in kernel-src/ if exists.  Three
>      cases plus one:
> 
>      (4a) The path kernel-src/ in the old checkout was not a
>      subproject (either it did not exist, it was a blob, or it
>      was a directory with files that are tracked as part of the
>      superproject).  The usual "don't lose local modification"
>      rule we use try to carry local changes forward across
>      branch switching, but in this case we shouldn't do so.
> 
>      (4b) It has the same logical/abstract project checked out;
>      the commit recorded in the superproject tree may or may not
>      be the same as what its HEAD points at.  In this case we do
>      not have to worry about swapping the git repository at
>      kernel-src/ directory, although we would need to check out
>      the correct revision, and worry about what to do with any
>      local modification (I think the usual "don't lose local
>      modification but carry them forward" rule would be Ok in
>      this case).
> 
>      (4c) It has a different project checked out; we need to be
>      careful to keep local changes, and also we need to make
>      sure the local changes in this subproject repository are
>      pushed back to (2).  It could be that automated "git push"
>      after making sure everything is committed is sufficient and
>      have the user handle failure cases.
> 
>      (4d) This applies not just "before step (3)", but in cases
>      where we need to replace a checked out subproject directory
>      with something else (e.g. blob or directory that belong to
>      the superproject, or noneness).  We would need to make sure
>      no local change is lost, and the repository is synched up
>      with (2).
> 
> I think an arrangement like this would solve "symlink is a bitch
> for MinGW" problem Johannes Sixt brought up today with Sven's
> RFC as well.

It would also solve (rare) case when for some reason the same subproject
should be checked out twice (different revisions).

-- 
						 Jan 'Bulb' Hudec <bulb@ucw.cz>

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Jan Hudec <bulb@ucw.cz> writes:

> It would also solve (rare) case when for some reason the same subproject
> should be checked out twice (different revisions).

I have a feeling that you are equating two subprojects that
happens to have the same upstream URL the same, and if that is
the case I think that is a mistake.

If you were doing an efficient cgi script that renders history
of git managed projects, binding git as its subproject, and that
system can be built with either 'maint' (i.e. 1.5.2 series) or
'master' (i.e. ultrastable WIP towards 1.5.3), even though they
both might come from git://git.kernel.org/pub/scm/git/git.git/,
I think they should be registered as two separate logical
subprojects.

The .gitmodules file might have:

	[module "git-maint"]
        	path = git-stale/
        	url = git://git.kernel.org/pub/scm/git/git.git/
                ;; branch = maint

	[module "git-master"]
        	path = git-stable/
        	url = git://git.kernel.org/pub/scm/git/git.git/
                ;; branch = master

and two paths (git-stale/ and git-stable/) in the superproject
tree would have commit object names from the named branches.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Steven Grimm]

Junio C Hamano wrote:
> If you were doing an efficient cgi script that renders history
> of git managed projects, binding git as its subproject, and that
> system can be built with either 'maint' (i.e. 1.5.2 series) or
> 'master' (i.e. ultrastable WIP towards 1.5.3), even though they
> both might come from git://git.kernel.org/pub/scm/git/git.git/,
> I think they should be registered as two separate logical
> subprojects.
>   

I agree strongly with this, and it's another good reason that we have to 
be able to use something other than the URL as the key to look up a 
subproject's repository location. If you use the URL it is impossible to 
differentiate the two subprojects in this case.

-Steve

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Junio C Hamano]

Steven Grimm <koreth@midwinter.com> writes:

> Junio C Hamano wrote:
>> If you were doing an efficient cgi script that renders history
>> of git managed projects, binding git as its subproject, and that
>> system can be built with either 'maint' (i.e. 1.5.2 series) or
>> 'master' (i.e. ultrastable WIP towards 1.5.3), even though they
>> both might come from git://git.kernel.org/pub/scm/git/git.git/,
>> I think they should be registered as two separate logical
>> subprojects.
>
> I agree strongly with this, and it's another good reason that we have
> to be able to use something other than the URL as the key to look up a
> subproject's repository location. If you use the URL it is impossible
> to differentiate the two subprojects in this case.

Heh, you do not have to rub it in ;-)  I already agreed that
your three-level arrangement is easier to work with.

Re: [PATCH 07/16] git-read-tree: take --submodules option

[Petr Baudis]

On Fri, May 18, 2007 at 09:24:56PM CEST, skimo@liacs.nl wrote:
> From: Sven Verdoolaege <skimo@kotnet.org>
> 
> This option currently has no effect.
> 
> Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>

Nacked-by: Petr Baudis <pasky@suse.cz>

Please do not add more undocumented parameters - include documentation
in the patch adding the parameter.

-- 
				Petr "Pasky" Baudis
Stuff: http://pasky.or.cz/
Ever try. Ever fail. No matter. // Try again. Fail again. Fail better.
		-- Samuel Beckett

[PATCH 08/16] unpack-trees.c: assume submodules are clean

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

If the submodules are not clean, then we will get an error
when we actally do the checkout.

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 unpack-trees.c |   43 ++++++++++++++++++++++++++++++++++---------
 1 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/unpack-trees.c b/unpack-trees.c
index 5fa637a..e979bc5 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -5,6 +5,7 @@
 #include "cache-tree.h"
 #include "unpack-trees.h"
 #include "progress.h"
+#include "refs.h"
 
 #define DBRT_DEBUG 1
 
@@ -426,11 +427,24 @@ static void invalidate_ce_path(struct cache_entry *ce)
 		cache_tree_invalidate_path(active_cache_tree, ce->name);
 }
 
-static int verify_clean_subdirectory(const char *path, const char *action,
+/* Check that checking out ce->sha1 in subdir ce->name is not
+ * going to overwrite any working files.
+ *
+ * FIXME: implement this function, so we can detect problems
+ *        early, rather than waiting until we actually try to checkout
+ *        the submodules.
+ */
+static int verify_clean_submodule(struct cache_entry *ce, const char *action,
+				      struct unpack_trees_options *o)
+{
+	return 0;
+}
+
+static int verify_clean_subdirectory(struct cache_entry *ce, const char *action,
 				      struct unpack_trees_options *o)
 {
 	/*
-	 * we are about to extract "path"; we would not want to lose
+	 * we are about to extract "ce->name"; we would not want to lose
 	 * anything in the existing directory there.
 	 */
 	int namelen;
@@ -438,13 +452,24 @@ static int verify_clean_subdirectory(const char *path, const char *action,
 	struct dir_struct d;
 	char *pathbuf;
 	int cnt = 0;
+	unsigned char sha1[20];
+
+	if (S_ISDIRLNK(ntohl(ce->ce_mode)) &&
+	    resolve_gitlink_ref(ce->name, "HEAD", sha1) == 0) {
+		/* If we are not going to update the submodule, then
+		 * we don't care.
+		 */
+		if (!o->submodules || !hashcmp(sha1, ce->sha1))
+			return 0;
+		verify_clean_submodule(ce, action, o);
+	}
 
 	/*
 	 * First let's make sure we do not have a local modification
 	 * in that directory.
 	 */
-	namelen = strlen(path);
-	pos = cache_name_pos(path, namelen);
+	namelen = strlen(ce->name);
+	pos = cache_name_pos(ce->name, namelen);
 	if (0 <= pos)
 		return cnt; /* we have it as nondirectory */
 	pos = -pos - 1;
@@ -452,7 +477,7 @@ static int verify_clean_subdirectory(const char *path, const char *action,
 		struct cache_entry *ce = active_cache[i];
 		int len = ce_namelen(ce);
 		if (len < namelen ||
-		    strncmp(path, ce->name, namelen) ||
+		    strncmp(ce->name, ce->name, namelen) ||
 		    ce->name[namelen] != '/')
 			break;
 		/*
@@ -470,16 +495,16 @@ static int verify_clean_subdirectory(const char *path, const char *action,
 	 * present file that is not ignored.
 	 */
 	pathbuf = xmalloc(namelen + 2);
-	memcpy(pathbuf, path, namelen);
+	memcpy(pathbuf, ce->name, namelen);
 	strcpy(pathbuf+namelen, "/");
 
 	memset(&d, 0, sizeof(d));
 	if (o->dir)
 		d.exclude_per_dir = o->dir->exclude_per_dir;
-	i = read_directory(&d, path, pathbuf, namelen+1, NULL);
+	i = read_directory(&d, ce->name, pathbuf, namelen+1, NULL);
 	if (i)
 		die("Updating '%s' would lose untracked files in it",
-		    path);
+		    ce->name);
 	free(pathbuf);
 	return cnt;
 }
@@ -513,7 +538,7 @@ static void verify_absent(struct cache_entry *ce, const char *action,
 			 * files that are in "foo/" we would lose
 			 * it.
 			 */
-			cnt = verify_clean_subdirectory(ce->name, action, o);
+			cnt = verify_clean_subdirectory(ce, action, o);
 
 			/*
 			 * If this removed entries from the index,
-- 
1.5.2.rc3.783.gc7476-dirty

[PATCH 09/16] entry.c: optionally checkout submodules

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 entry.c |   42 ++++++++++++++++++++++++++++++++++++++++--
 1 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/entry.c b/entry.c
index 82bf725..96a4a60 100644
--- a/entry.c
+++ b/entry.c
@@ -1,5 +1,6 @@
 #include "cache.h"
 #include "blob.h"
+#include "run-command.h"
 
 static void create_directories(const char *path, const struct checkout *state)
 {
@@ -163,6 +164,44 @@ static int write_entry(struct cache_entry *ce, char *path, const struct checkout
 	return 0;
 }
 
+static int checkout_submodule(const char *path, struct cache_entry *ce, const struct checkout *state)
+{
+	static char cwd[PATH_MAX];
+	const char *gitdirenv;
+	const char *args[10];
+	int argc;
+	int err;
+
+	if (!state->submodules)
+		return 0;
+
+	if (!getcwd(cwd, sizeof(cwd)) || cwd[0] != '/')
+		die("Unable to read current working directory");
+
+	if (chdir(path))
+		die("Cannot move to '%s'", path);
+
+	argc = 0;
+	args[argc++] = "checkout";
+	if (state->force)
+	    args[argc++] = "-f";
+	args[argc++] = sha1_to_hex(ce->sha1);
+	args[argc] = NULL;
+
+	gitdirenv = getenv(GIT_DIR_ENVIRONMENT);
+	unsetenv(GIT_DIR_ENVIRONMENT);
+	err = run_command_v_opt(args, RUN_GIT_CMD);
+	setenv(GIT_DIR_ENVIRONMENT, gitdirenv, 1);
+
+	if (chdir(cwd))
+		die("Cannot come back to cwd");
+
+	if (err)
+		return error("failed to run git-checkout in submodule '%s'", path);
+
+	return 0;
+}
+
 int checkout_entry(struct cache_entry *ce, const struct checkout *state, char *topath)
 {
 	static char path[PATH_MAX + 1];
@@ -193,9 +232,8 @@ int checkout_entry(struct cache_entry *ce, const struct checkout *state, char *t
 		 */
 		unlink(path);
 		if (S_ISDIR(st.st_mode)) {
-			/* If it is a gitlink, leave it alone! */
 			if (S_ISDIRLNK(ntohl(ce->ce_mode)))
-				return 0;
+				return checkout_submodule(path, ce, state);
 			if (!state->force)
 				return error("%s is a directory", path);
 			remove_subtree(path);
-- 
1.5.2.rc3.783.gc7476-dirty

Re: [PATCH 09/16] entry.c: optionally checkout submodules

[Alex Riesen]

skimo@liacs.nl, Fri, May 18, 2007 21:24:58 +0200:
> +	if (err)
> +		return error("failed to run git-checkout in submodule '%s'", path);

We may need an option to ignore these failures. Maybe even active by
default. Imagine a superproject with _optional_ submodules, where it
is just nice to know that some submodules weren't checked out. BTW,
doesn't git-checkout already prints an error?

Re: [PATCH 09/16] entry.c: optionally checkout submodules

[Sven Verdoolaege]

On Fri, May 18, 2007 at 11:56:42PM +0200, Alex Riesen wrote:
> skimo@liacs.nl, Fri, May 18, 2007 21:24:58 +0200:
> > +	if (err)
> > +		return error("failed to run git-checkout in submodule '%s'", path);
> 
> We may need an option to ignore these failures. Maybe even active by
> default. Imagine a superproject with _optional_ submodules, where it
> is just nice to know that some submodules weren't checked out. BTW,
> doesn't git-checkout already prints an error?

Probably.  You probably noticed that I haven't written any tests yet...

Still, the error that git-checkout prints may not give enough of a clue
that something was wrong with a submodule.

skimo

Re: [PATCH 09/16] entry.c: optionally checkout submodules

[Alex Riesen]

Sven Verdoolaege, Sat, May 19, 2007 00:03:23 +0200:
> On Fri, May 18, 2007 at 11:56:42PM +0200, Alex Riesen wrote:
> > skimo@liacs.nl, Fri, May 18, 2007 21:24:58 +0200:
> > > +	if (err)
> > > +		return error("failed to run git-checkout in submodule '%s'", path);
> > 
> > We may need an option to ignore these failures. Maybe even active by
> > default. Imagine a superproject with _optional_ submodules, where it
> > is just nice to know that some submodules weren't checked out. BTW,
> > doesn't git-checkout already prints an error?
> 
> Probably.  You probably noticed that I haven't written any tests yet...
> 

I see. It was a very ... provocative patch series :)

> Still, the error that git-checkout prints may not give enough of a clue
> that something was wrong with a submodule.

Like, for example, it failed because the directory is not a git repo
yet, because the previous git-checkout was called _without_
--submodule and the directories created are just empty.

Anyway, just a "failed to run git-checkout" is not very helpful
either. Come to think about it, there is not very much you can tell
out of super-project context. git-checkout will always know better.

Re: [PATCH 09/16] entry.c: optionally checkout submodules

[Alex Riesen]

skimo@liacs.nl, Fri, May 18, 2007 21:24:58 +0200:
> +	if (!getcwd(cwd, sizeof(cwd)) || cwd[0] != '/')
> +		die("Unable to read current working directory");
> +
> +	if (chdir(path))
> +		die("Cannot move to '%s'", path);
> +

How about modifying run_command to chdir after fork?

You'd save the hassle of save/restoring cwd and don't mess up process'
context (which is always a good idea to preserve). The code'd be
simplier, too.

[PATCH] Add run_command_v_opt_cd: chdir into a directory before exec

[Alex Riesen]

It can make code simplier (no need to preserve cwd) and safer
(no chance the cwd of the current process is accidentally forgotten).

Signed-off-by: Alex Riesen <raa.lkml@gmail.com>
---
Alex Riesen, Sat, May 19, 2007 00:00:14 +0200:
> skimo@liacs.nl, Fri, May 18, 2007 21:24:58 +0200:
> > +	if (!getcwd(cwd, sizeof(cwd)) || cwd[0] != '/')
> > +		die("Unable to read current working directory");
> > +
> > +	if (chdir(path))
> > +		die("Cannot move to '%s'", path);
> > +
> 
> How about modifying run_command to chdir after fork?
> 
> You'd save the hassle of save/restoring cwd and don't mess up process'
> context (which is always a good idea to preserve). The code'd be
> simplier, too.
> 

something like this

 run-command.c |   27 ++++++++++++++++++++++-----
 run-command.h |    2 ++
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/run-command.c b/run-command.c
index eff523e..043b570 100644
--- a/run-command.c
+++ b/run-command.c
@@ -73,6 +73,9 @@ int start_command(struct child_process *cmd)
 			close(cmd->out);
 		}
 
+		if (cmd->dir && chdir(cmd->dir))
+			die("exec %s: cd to %s failed (%s)", cmd->argv[0],
+			    cmd->dir, strerror(errno));
 		if (cmd->git_cmd) {
 			execv_git_cmd(cmd->argv);
 		} else {
@@ -133,13 +136,27 @@ int run_command(struct child_process *cmd)
 	return finish_command(cmd);
 }
 
+static void prepare_run_command_v_opt(struct child_process *cmd,
+				      const char **argv, int opt)
+{
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->argv = argv;
+	cmd->no_stdin = opt & RUN_COMMAND_NO_STDIN ? 1 : 0;
+	cmd->git_cmd = opt & RUN_GIT_CMD ? 1 : 0;
+	cmd->stdout_to_stderr = opt & RUN_COMMAND_STDOUT_TO_STDERR ? 1 : 0;
+}
+
 int run_command_v_opt(const char **argv, int opt)
 {
 	struct child_process cmd;
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.argv = argv;
-	cmd.no_stdin = opt & RUN_COMMAND_NO_STDIN ? 1 : 0;
-	cmd.git_cmd = opt & RUN_GIT_CMD ? 1 : 0;
-	cmd.stdout_to_stderr = opt & RUN_COMMAND_STDOUT_TO_STDERR ? 1 : 0;
+	prepare_run_command_v_opt(&cmd, argv, opt);
+	return run_command(&cmd);
+}
+
+int run_command_v_opt_cd(const char **argv, int opt, const char *dir)
+{
+	struct child_process cmd;
+	prepare_run_command_v_opt(&cmd, argv, opt);
+	cmd.dir = dir;
 	return run_command(&cmd);
 }
diff --git a/run-command.h b/run-command.h
index 3680ef9..cbd7484 100644
--- a/run-command.h
+++ b/run-command.h
@@ -16,6 +16,7 @@ struct child_process {
 	pid_t pid;
 	int in;
 	int out;
+	const char *dir;
 	unsigned close_in:1;
 	unsigned close_out:1;
 	unsigned no_stdin:1;
@@ -32,5 +33,6 @@ int run_command(struct child_process *);
 #define RUN_GIT_CMD	     2	/*If this is to be git sub-command */
 #define RUN_COMMAND_STDOUT_TO_STDERR 4
 int run_command_v_opt(const char **argv, int opt);
+int run_command_v_opt_cd(const char **argv, int opt, const char *dir);
 
 #endif
-- 
1.5.2.rc3.83.gbbb0

[PATCH] Use run_command_v_opt_cd when checking out a submodule

[Alex Riesen]

Signed-off-by: Alex Riesen <raa.lkml@gmail.com>
---
 entry.c |   12 +-----------
 1 files changed, 1 insertions(+), 11 deletions(-)

diff --git a/entry.c b/entry.c
index 96a4a60..0316c74 100644
--- a/entry.c
+++ b/entry.c
@@ -166,7 +166,6 @@ static int write_entry(struct cache_entry *ce, char *path, const struct checkout
 
 static int checkout_submodule(const char *path, struct cache_entry *ce, const struct checkout *state)
 {
-	static char cwd[PATH_MAX];
 	const char *gitdirenv;
 	const char *args[10];
 	int argc;
@@ -175,12 +174,6 @@ static int checkout_submodule(const char *path, struct cache_entry *ce, const st
 	if (!state->submodules)
 		return 0;
 
-	if (!getcwd(cwd, sizeof(cwd)) || cwd[0] != '/')
-		die("Unable to read current working directory");
-
-	if (chdir(path))
-		die("Cannot move to '%s'", path);
-
 	argc = 0;
 	args[argc++] = "checkout";
 	if (state->force)
@@ -190,12 +183,9 @@ static int checkout_submodule(const char *path, struct cache_entry *ce, const st
 
 	gitdirenv = getenv(GIT_DIR_ENVIRONMENT);
 	unsetenv(GIT_DIR_ENVIRONMENT);
-	err = run_command_v_opt(args, RUN_GIT_CMD);
+	err = run_command_v_opt_cd(args, RUN_GIT_CMD, path);
 	setenv(GIT_DIR_ENVIRONMENT, gitdirenv, 1);
 
-	if (chdir(cwd))
-		die("Cannot come back to cwd");
-
 	if (err)
 		return error("failed to run git-checkout in submodule '%s'", path);
 
-- 
1.5.2.rc3.83.gbbb0

[PATCH 10/16] git-checkout: pass --submodules option to git-read-tree

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 git-checkout.sh |   20 +++++++++++++++-----
 1 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/git-checkout.sh b/git-checkout.sh
index 6b6facf..cbb1f00 100755
--- a/git-checkout.sh
+++ b/git-checkout.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-USAGE='[-q] [-f] [-b <new_branch>] [-m] [<branch>] [<paths>...]'
+USAGE='[-q] [-f] [--submodules] [--no-submodules] [-b <new_branch>] [-m] [<branch>] [<paths>...]'
 SUBDIRECTORY_OK=Sometimes
 . git-sh-setup
 require_work_tree
@@ -16,6 +16,7 @@ track=
 newbranch=
 newbranch_log=
 merge=
+submodules=
 quiet=
 v=-v
 LF='
@@ -46,6 +47,15 @@ while [ "$#" != "0" ]; do
 	-m)
 		merge=1
 		;;
+	*,--su|*,--sub|*,--subm|*,--submo|*,--submod|*,--submodu|*,--submodul|\
+	*,--submodule|*,--submodules)
+		submodules="--submodules"
+		;;
+	*,--no-su|*,--no-sub|*,--no-subm|*,--no-submo|*,--no-submod|\
+	*,--no-submodu|*,--no-submodul|\
+	*,--no-submodule|*,--no-submodules)
+		submodules="--no-submodules"
+		;;
 	"-q")
 		quiet=1
 		v=
@@ -199,10 +209,10 @@ fi
 
 if [ "$force" ]
 then
-    git-read-tree $v --reset -u $new
+    git-read-tree $v $submodules --reset -u $new
 else
     git-update-index --refresh >/dev/null
-    merge_error=$(git-read-tree -m -u --exclude-per-directory=.gitignore $old $new 2>&1) || (
+    merge_error=$(git-read-tree $submodules -m -u --exclude-per-directory=.gitignore $old $new 2>&1) || (
 	case "$merge" in
 	'')
 		echo >&2 "$merge_error"
@@ -212,7 +222,7 @@ else
 	# Match the index to the working tree, and do a three-way.
     	git diff-files --name-only | git update-index --remove --stdin &&
 	work=`git write-tree` &&
-	git read-tree $v --reset -u $new || exit
+	git read-tree $v $submodules --reset -u $new || exit
 
 	eval GITHEAD_$new='${new_name:-${branch:-$new}}' &&
 	eval GITHEAD_$work=local &&
@@ -223,7 +233,7 @@ else
 	# this is not a real merge before committing, but just carrying
 	# the working tree changes along.
 	unmerged=`git ls-files -u`
-	git read-tree $v --reset $new
+	git read-tree $v $submodules --reset $new
 	case "$unmerged" in
 	'')	;;
 	*)
-- 
1.5.2.rc3.783.gc7476-dirty

Re: [PATCH 10/16] git-checkout: pass --submodules option to git-read-tree

[Petr Baudis]

On Fri, May 18, 2007 at 09:24:59PM CEST, skimo@liacs.nl wrote:
> From: Sven Verdoolaege <skimo@kotnet.org>
> 
> Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
> ---
>  git-checkout.sh |   20 +++++++++++++++-----
>  1 files changed, 15 insertions(+), 5 deletions(-)
> 
> diff --git a/git-checkout.sh b/git-checkout.sh
> index 6b6facf..cbb1f00 100755
> --- a/git-checkout.sh
> +++ b/git-checkout.sh
> @@ -1,6 +1,6 @@
>  #!/bin/sh
>  
> -USAGE='[-q] [-f] [-b <new_branch>] [-m] [<branch>] [<paths>...]'
> +USAGE='[-q] [-f] [--submodules] [--no-submodules] [-b <new_branch>] [-m] [<branch>] [<paths>...]'
>  SUBDIRECTORY_OK=Sometimes
>  . git-sh-setup
>  require_work_tree

Thus Documentation/git-checkout.txt needs to be updated as well.

-- 
				Petr "Pasky" Baudis
Stuff: http://pasky.or.cz/
Ever try. Ever fail. No matter. // Try again. Fail again. Fail better.
		-- Samuel Beckett

[PATCH 11/16] git-fetch: skip empty arguments

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

This makes it easier for scripts to call git-fetch with options
that may or may not be set.

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 git-fetch.sh |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/git-fetch.sh b/git-fetch.sh
index 0e05cf1..dbeca14 100755
--- a/git-fetch.sh
+++ b/git-fetch.sh
@@ -76,6 +76,8 @@ do
 	-*)
 		usage
 		;;
+	'')
+		;;
 	*)
 		break
 		;;
-- 
1.5.2.rc3.783.gc7476-dirty

Re: [PATCH 11/16] git-fetch: skip empty arguments

[Junio C Hamano]

skimo@liacs.nl writes:

> From: Sven Verdoolaege <skimo@kotnet.org>
>
> This makes it easier for scripts to call git-fetch with options
> that may or may not be set.

For git-fetch it does not matter as I do not think there is any
valid case to pass an empty string as a parameter to it (even
"fetch from our own repository" requires a single dot).  But
from discipline point of view, I am not happy about this.

If you are talking about shell scripts, the standard way to do
that is to say ${1+"$1"}.

[PATCH 12/16] builtin-fetch--tool: extend "native-store" for use in cloning

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 builtin-fetch--tool.c |   48 ++++++++++++++++++++++++++++++++++++++++++++----
 1 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c
index ed4d5de..3441a4a 100644
--- a/builtin-fetch--tool.c
+++ b/builtin-fetch--tool.c
@@ -207,6 +207,32 @@ static void remove_keep_on_signal(int signo)
 	raise(signo);
 }
 
+static char *construct_local_name(const char *remote_ref, const char *remote_nick,
+				  int use_separate_remote)
+{
+	static char local_ref[PATH_MAX];
+	int len = strlen(remote_ref);
+
+	if (len >= 3 && !memcmp(remote_ref+len-3, "^{}", 3))
+		return NULL;
+	if (!strcmp(remote_ref, "HEAD"))
+		return "REMOTE_HEAD";
+	if (!prefixcmp(remote_ref, "refs/heads/")) {
+		if (snprintf(local_ref, sizeof(local_ref), "refs/%s%s/%s",
+			    use_separate_remote ? "remotes/" : "heads",
+			    use_separate_remote ? remote_nick : "",
+			    remote_ref+11) > sizeof(local_ref))
+			die("Local branchname too long");
+	} else if (!prefixcmp(remote_ref, "refs/tags/")) {
+		if (snprintf(local_ref, sizeof(local_ref), "refs/tags/%s",
+			    remote_ref+10) > sizeof(local_ref))
+			die("Local branchname too long");
+	} else
+		return NULL;
+
+	return local_ref;
+}
+
 static char *find_local_name(const char *remote_name, const char *refs,
 			     int *force_p, int *not_for_merge_p)
 {
@@ -261,7 +287,8 @@ static int fetch_native_store(FILE *fp,
 			      const char *remote,
 			      const char *remote_nick,
 			      const char *refs,
-			      int verbose, int force)
+			      int verbose, int force,
+			      int all, int use_separate_remote)
 {
 	char buffer[1024];
 	int err = 0;
@@ -294,8 +321,12 @@ static int fetch_native_store(FILE *fp,
 			continue;
 		}
 
-		local_name = find_local_name(cp, refs,
-					     &single_force, &not_for_merge);
+		if (all)
+			local_name = construct_local_name(cp, remote_nick,
+							  use_separate_remote);
+		else
+			local_name = find_local_name(cp, refs,
+						     &single_force, &not_for_merge);
 		if (!local_name)
 			continue;
 		err |= append_fetch_head(fp,
@@ -514,6 +545,8 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix)
 	int verbose = 0;
 	int force = 0;
 	int sopt = 0;
+	int all = 0;
+	int use_separate_remote = 1;
 
 	while (1 < argc) {
 		const char *arg = argv[1];
@@ -523,6 +556,12 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix)
 			force = 1;
 		else if (!strcmp("-s", arg))
 			sopt = 1;
+		else if (!strcmp("--all", arg))
+			all = 1;
+		else if (!strcmp("--use-separate-remote", arg))
+			use_separate_remote = 1;
+		else if (!strcmp("--no-separate-remote", arg))
+			use_separate_remote = 0;
 		else
 			break;
 		argc--;
@@ -554,7 +593,8 @@ int cmd_fetch__tool(int argc, const char **argv, const char *prefix)
 			return error("fetch-native-store takes 3 args");
 		fp = fopen(git_path("FETCH_HEAD"), "a");
 		result = fetch_native_store(fp, argv[2], argv[3], argv[4],
-					    verbose, force);
+					    verbose, force, all,
+					    use_separate_remote);
 		fclose(fp);
 		return result;
 	}
-- 
1.5.2.rc3.783.gc7476-dirty

Re: [PATCH 12/16] builtin-fetch--tool: extend "native-store" for use in cloning

[Alex Riesen]

skimo@liacs.nl, Fri, May 18, 2007 21:25:01 +0200:
> @@ -261,7 +287,8 @@ static int fetch_native_store(FILE *fp,
>  			      const char *remote,
>  			      const char *remote_nick,
>  			      const char *refs,
> -			      int verbose, int force)
> +			      int verbose, int force,
> +			      int all, int use_separate_remote)
>  {
>  	char buffer[1024];
>  	int err = 0;
> @@ -294,8 +321,12 @@ static int fetch_native_store(FILE *fp,
>  			continue;
>  		}
>  
> -		local_name = find_local_name(cp, refs,
> -					     &single_force, &not_for_merge);
> +		if (all)
> +			local_name = construct_local_name(cp, remote_nick,
> +							  use_separate_remote);
> +		else
> +			local_name = find_local_name(cp, refs,
> +						     &single_force, &not_for_merge);

This code produces warning about possible uninitialized used of
single_force and not_for_merge. I used the patch below, but didn't
look into what the "all" does.

---
 builtin-fetch--tool.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/builtin-fetch--tool.c b/builtin-fetch--tool.c
index 12adb38..ebb49d9 100644
--- a/builtin-fetch--tool.c
+++ b/builtin-fetch--tool.c
@@ -273,7 +273,7 @@ static int fetch_native_store(FILE *fp,
 		int len;
 		char *cp;
 		char *local_name;
-		int single_force, not_for_merge;
+		int single_force = force, not_for_merge = 0;
 
 		for (cp = buffer; *cp && !isspace(*cp); cp++)
 			;
@@ -301,7 +301,7 @@ static int fetch_native_store(FILE *fp,
 		err |= append_fetch_head(fp,
 					 buffer, remote, cp, remote_nick,
 					 local_name, not_for_merge,
-					 verbose, force || single_force);
+					 verbose, single_force);
 	}
 	return err;
 }
-- 
1.5.2.rc3.83.gbbb0

Re: [PATCH 12/16] builtin-fetch--tool: extend "native-store" for use in cloning

[Sven Verdoolaege]

On Sat, May 19, 2007 at 12:52:34AM +0200, Alex Riesen wrote:
> This code produces warning about possible uninitialized used of
> single_force and not_for_merge. I used the patch below, but didn't
> look into what the "all" does.

Sorry for being sloppy.  It is assumed that "all" is only used
for cloning and then the other two flags don't really matter,
but I should've set some defaults.
Yours look fine, thanks.

A fixed-up version is available in the "submodules" branch
of http://www.liacs.nl/~sverdool/git.git or
www.liacs.nl/~sverdool/gitweb.cgi?p=git.git;a=shortlog;h=submodules

skimo

[PATCH 13/16] git-clone: rely on git-fetch for fetching for most protocols

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 git-clone.sh |   20 ++++++++++++--------
 git-fetch.sh |   28 ++++++++++++++++++++++------
 2 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/git-clone.sh b/git-clone.sh
index fdd354f..44127c5 100755
--- a/git-clone.sh
+++ b/git-clone.sh
@@ -159,6 +159,11 @@ then
 	no_checkout=yes
 	use_separate_remote=
 fi
+if test t = "$use_separate_remote"; then
+	separate_remote_flag="--use-separate-remote"
+else
+	separate_remote_flag="--no-separate-remote"
+fi
 
 if test -z "$origin"
 then
@@ -219,6 +224,10 @@ then
 	fi
 fi
 
+# Write out $origin URL
+GIT_CONFIG="$GIT_DIR/config"
+git-config remote."$origin".url "$repo" || exit
+
 rm -f "$GIT_DIR/CLONE_HEAD"
 
 # We do local magic only when the user tells us to.
@@ -299,11 +308,9 @@ yes,yes)
 		fi
 		;;
 	*)
-		case "$upload_pack" in
-		'') git-fetch-pack --all -k $quiet $depth $no_progress "$repo";;
-		*) git-fetch-pack --all -k $quiet "$upload_pack" $depth $no_progress "$repo" ;;
-		esac >"$GIT_DIR/CLONE_HEAD" ||
-			die "fetch-pack from '$repo' failed."
+		git-fetch --all -k $quiet "$upload_pack" $depth \
+			$separate_remote_flag "$origin" ||
+			die "fetch from '$repo' failed."
 		;;
 	esac
 	;;
@@ -387,9 +394,6 @@ then
 		origin_track="$remote_top/$head_points_at" &&
 		git-update-ref HEAD "$head_sha1" &&
 
-		# Upstream URL
-		git-config remote."$origin".url "$repo" &&
-
 		# Set up the mappings to track the remote branches.
 		git-config remote."$origin".fetch \
 			"+refs/heads/*:$remote_top/*" '^$' &&
diff --git a/git-fetch.sh b/git-fetch.sh
index dbeca14..e169848 100755
--- a/git-fetch.sh
+++ b/git-fetch.sh
@@ -15,6 +15,7 @@ LF='
 '
 IFS="$LF"
 
+all=
 no_tags=
 tags=
 append=
@@ -25,6 +26,7 @@ exec=
 keep=
 shallow_depth=
 no_progress=
+use_separate_remote=
 test -t 1 || no_progress=--no-progress
 quiet=
 while case "$#" in 0) break ;; esac
@@ -33,6 +35,9 @@ do
 	-a|--a|--ap|--app|--appe|--appen|--append)
 		append=t
 		;;
+	--al|--all)
+		all=--all
+		;;
 	--upl|--uplo|--uploa|--upload|--upload-|--upload-p|\
 	--upload-pa|--upload-pac|--upload-pack)
 		shift
@@ -63,6 +68,12 @@ do
 	-v|--verbose)
 		verbose=Yes
 		;;
+	--use-separate-remote)
+		use_separate_remote="--use-separate-remote"
+		;;
+	--no-separate-remote)
+		use_separate_remote="--no-separate-remote"
+		;;
 	-k|--k|--ke|--kee|--keep)
 		keep='-k -k'
 		;;
@@ -143,7 +154,9 @@ esac
 # branches file, and just fetch those and refspecs explicitly given.
 # Otherwise we do what we always did.
 
-reflist=$(get_remote_refs_for_fetch "$@")
+if test -z "$all"; then
+	reflist=$(get_remote_refs_for_fetch "$@")
+fi
 if test "$tags"
 then
 	taglist=`IFS='	' &&
@@ -165,8 +178,10 @@ fi
 
 fetch_all_at_once () {
 
-  eval=$(echo "$1" | git-fetch--tool parse-reflist "-")
-  eval "$eval"
+    if test -z "$all"; then
+	eval=$(echo "$1" | git-fetch--tool parse-reflist "-")
+	eval "$eval"
+    fi
 
     ( : subshell because we muck with IFS
       IFS=" 	$LF"
@@ -179,7 +194,8 @@ fetch_all_at_once () {
 	    git-bundle unbundle "$remote" $rref ||
 	    echo failed "$remote"
 	else
-		if	test -d "$remote" &&
+		if	test -z "$all" &&
+			test -d "$remote" &&
 
 			# The remote might be our alternate.  With
 			# this optimization we will bypass fetch-pack
@@ -203,7 +219,7 @@ fetch_all_at_once () {
 			echo "$ls_remote_result" | \
 				git-fetch--tool pick-rref "$rref" "-"
 		else
-			git-fetch-pack --thin $exec $keep $shallow_depth \
+			git-fetch-pack --thin $all $exec $keep $shallow_depth \
 				$quiet $no_progress "$remote" $rref ||
 			echo failed "$remote"
 		fi
@@ -214,7 +230,7 @@ fetch_all_at_once () {
 	test -n "$verbose" && flags="$flags -v"
 	test -n "$force" && flags="$flags -f"
 	GIT_REFLOG_ACTION="$GIT_REFLOG_ACTION" \
-		git-fetch--tool $flags native-store \
+		git-fetch--tool $flags $all $use_separate_remote native-store \
 			"$remote" "$remote_nick" "$refs"
       )
     ) || exit
-- 
1.5.2.rc3.783.gc7476-dirty

[PATCH 14/16] git-clone: rely on git-fetch for non-bare fetching over http

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 git-clone.sh |    6 +++---
 git-fetch.sh |   20 ++++++++++++++++++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/git-clone.sh b/git-clone.sh
index 44127c5..44387f4 100755
--- a/git-clone.sh
+++ b/git-clone.sh
@@ -262,8 +262,8 @@ yes,yes)
 	git-ls-remote "$repo" >"$GIT_DIR/CLONE_HEAD" || exit 1
 	;;
 *)
-	case "$repo" in
-	rsync://*)
+	case "$bare,$repo" in
+	*,rsync://*)
 		case "$depth" in
 		"") ;;
 		*) die "shallow over rsync not supported" ;;
@@ -295,7 +295,7 @@ yes,yes)
 		fi
 		git-ls-remote "$repo" >"$GIT_DIR/CLONE_HEAD" || exit 1
 		;;
-	https://*|http://*|ftp://*)
+	yes,https://*|yes,http://*|yes,ftp://*)
 		case "$depth" in
 		"") ;;
 		*) die "shallow over http or ftp not supported" ;;
diff --git a/git-fetch.sh b/git-fetch.sh
index e169848..84c2523 100755
--- a/git-fetch.sh
+++ b/git-fetch.sh
@@ -237,11 +237,31 @@ fetch_all_at_once () {
 
 }
 
+http_fetch () {
+	if [ -n "$GIT_SSL_NO_VERIFY" ]; then
+		curl_extra_args="-k"
+	fi
+	if [ -n "$GIT_CURL_FTP_NO_EPSV" -o \
+		"`git-config --bool http.noEPSV`" = true ]; then
+		curl_extra_args="${curl_extra_args} --disable-epsv"
+	fi
+
+	# $1 = Remote, $2 = Local
+	curl -nsfL $curl_extra_args "$1" >"$2"
+}
+
 fetch_per_ref () {
   reflist="$1"
   refs=
   rref=
 
+    if test -n "$all"; then
+	reflist=$(canon_refs_list_for_fetch -d "$remote_nick" \
+			"+refs/heads/*:refs/remotes/$remote_nick/*")
+	http_fetch "$remote/HEAD" "$GIT_DIR/REMOTE_HEAD" ||
+	rm -f "$GIT_DIR/REMOTE_HEAD"
+    fi
+
   for ref in $reflist
   do
       refs="$refs$LF$ref"
-- 
1.5.2.rc3.783.gc7476-dirty

[PATCH 15/16] git-read-tree: treat null commit as empty tree

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

---
 builtin-read-tree.c |    9 ++++++---
 unpack-trees.c      |    3 +++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/builtin-read-tree.c b/builtin-read-tree.c
index 929dd95..b9fcff7 100644
--- a/builtin-read-tree.c
+++ b/builtin-read-tree.c
@@ -17,9 +17,12 @@ static struct object_list *trees;
 
 static int list_tree(unsigned char *sha1)
 {
-	struct tree *tree = parse_tree_indirect(sha1);
-	if (!tree)
-		return -1;
+	struct tree *tree = NULL;
+	if (!is_null_sha1(sha1)) {
+		tree = parse_tree_indirect(sha1);
+		if (!tree)
+			return -1;
+	}
 	object_list_append(&tree->object, &trees);
 	return 0;
 }
diff --git a/unpack-trees.c b/unpack-trees.c
index e979bc5..30c2a49 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -26,6 +26,9 @@ static struct tree_entry_list *create_tree_entry_list(struct tree *tree)
 	struct tree_entry_list *ret = NULL;
 	struct tree_entry_list **list_p = &ret;
 
+	if (!tree)
+		return ret;
+
 	if (!tree->object.parsed)
 		parse_tree(tree);
 
-- 
1.5.2.rc3.783.gc7476-dirty

[PATCH 16/16] git-clone: add --submodules for cloning submodules

[skimo]

From: Sven Verdoolaege <skimo@kotnet.org>

When the --submodules option is specified, git-clone will search
for submodule.<submodule>.url options in the remote configuration
and clone each submodule using the first url that it can use from
the local site.

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 Documentation/config.txt    |    7 +++
 Documentation/git-clone.txt |    6 ++-
 git-clone.sh                |   18 +++++++-
 git-fetch.sh                |   90 +++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 117 insertions(+), 4 deletions(-)

diff --git a/Documentation/config.txt b/Documentation/config.txt
index ee1c35e..cee9e40 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -256,6 +256,10 @@ You probably do not need to adjust this value.
 +
 Common unit suffixes of 'k', 'm', or 'g' are supported.
 
+core.submodules
+	If true, gitlink:git-checkout[1] also checks out submodules.
+	False by default.
+
 alias.*::
 	Command aliases for the gitlink:git[1] command wrapper - e.g.
 	after defining "alias.last = cat-file commit HEAD", the invocation
@@ -606,6 +610,9 @@ showbranch.default::
 	The default set of branches for gitlink:git-show-branch[1].
 	See gitlink:git-show-branch[1].
 
+submodule.<submodule>.url
+	The URL of a submodule.  See gitlink:git-clone[1].
+
 tar.umask::
 	By default, gitlink:git-tar-tree[1] sets file and directories modes
 	to 0666 or 0777. While this is both useful and acceptable for projects
diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt
index 644bf12..565155b 100644
--- a/Documentation/git-clone.txt
+++ b/Documentation/git-clone.txt
@@ -11,7 +11,7 @@ SYNOPSIS
 [verse]
 'git-clone' [--template=<template_directory>] [-l [-s]] [-q] [-n] [--bare]
 	  [-o <name>] [-u <upload-pack>] [--reference <repository>]
-	  [--depth <depth>] <repository> [<directory>]
+	  [--depth <depth>] [--submodules] <repository> [<directory>]
 
 DESCRIPTION
 -----------
@@ -105,6 +105,10 @@ OPTIONS
 	with a long history, and would want to send in a fixes
 	as patches.
 
+--submodules::
+	Clone submodules specified in (remote) configuration parameters
+	submodule.<submodule>.url.
+
 <repository>::
 	The (possibly remote) repository to clone from.  It can
 	be any URL git-fetch supports.
diff --git a/git-clone.sh b/git-clone.sh
index 44387f4..f5a3548 100755
--- a/git-clone.sh
+++ b/git-clone.sh
@@ -14,7 +14,7 @@ die() {
 }
 
 usage() {
-	die "Usage: $0 [--template=<template_directory>] [--reference <reference-repo>] [--bare] [-l [-s]] [-q] [-u <upload-pack>] [--origin <name>] [--depth <n>] [-n] <repo> [<dir>]"
+	die "Usage: $0 [--template=<template_directory>] [--reference <reference-repo>] [--bare] [-l [-s]] [-q] [-u <upload-pack>] [--origin <name>] [--depth <n>] [-n] [--submodules] <repo> [<dir>]"
 }
 
 get_repo_base() {
@@ -88,6 +88,7 @@ origin_override=
 use_separate_remote=t
 depth=
 no_progress=
+submodules=
 test -t 1 || no_progress=--no-progress
 while
 	case "$#,$1" in
@@ -138,6 +139,8 @@ while
 	*,--depth)
 		shift
 		depth="--depth=$1";;
+	*,--su|*,--sub|*,--subm|*,--submo|*,--submod|*,--submodu|*,--submodul|\
+	*,--submodule|*,--submodules) submodules="--submodules" ;;
 	*,-*) usage ;;
 	*) break ;;
 	esac
@@ -156,6 +159,10 @@ then
 	then
 		die '--bare and --origin $origin options are incompatible.'
 	fi
+	if test -n "$submodules"
+	then
+		die '--bare and --submodules origin options are incompatible.'
+	fi
 	no_checkout=yes
 	use_separate_remote=
 fi
@@ -309,7 +316,7 @@ yes,yes)
 		;;
 	*)
 		git-fetch --all -k $quiet "$upload_pack" $depth \
-			$separate_remote_flag "$origin" ||
+			$separate_remote_flag $submodules "$origin" ||
 			die "fetch from '$repo' failed."
 		;;
 	esac
@@ -405,10 +412,15 @@ then
 		git-config branch."$head_points_at".merge "refs/heads/$head_points_at"
 	esac
 
+	if test -n "$submodules"
+	then
+		git-config core.submodules true
+	fi
+
 	case "$no_checkout" in
 	'')
 		test "z$quiet" = z -a "z$no_progress" = z && v=-v || v=
-		git-read-tree -m -u $v HEAD HEAD
+		git-read-tree -m -u $v $submodules HEAD HEAD
 	esac
 fi
 rm -f "$GIT_DIR/CLONE_HEAD" "$GIT_DIR/REMOTE_HEAD"
diff --git a/git-fetch.sh b/git-fetch.sh
index 84c2523..b7ef0c3 100755
--- a/git-fetch.sh
+++ b/git-fetch.sh
@@ -15,6 +15,70 @@ LF='
 '
 IFS="$LF"
 
+local_URL() {
+	# tranforms a "URL" on the remote to a URL that works on the local machine
+	# $1 - remote, $2 - URL on remote
+	echo "$1 $2" >&2
+	case "$1" in
+	https://*|http://*|ftp://*)
+		case "$2" in
+		https://*|http://*|ftp://*)
+			echo $2
+		esac
+		;;
+	ssh://*)
+		case "$2" in
+		https://*|http://*|ftp://*)
+			echo $2
+			;;
+		/*)
+			echo $(echo $1 | sed -e 's/\(ssh:\/\/[^\/]*\)\/.*/\1/')$2
+		esac
+		;;
+	/*)
+		echo $2
+		;;
+	*)
+		case "$2" in
+		https://*|http://*|ftp://*)
+			echo $2
+		esac
+	esac
+}
+
+clone_submodules () {
+	# $1 - remote
+	remote=$1
+	( : subshell because we muck with IFS
+        IFS=" 	$LF"
+	cd "$GIT_DIR/.."
+	git-config --remote="$remote" --get-regexp 'submodule\..*\.url' | \
+	sed -e 's/^submodule\.//' -e 's/\.url / /' |
+	while read submodule URL
+	do
+		previous=$(git-config "submodule.$submodule.url")
+		if test -n "$previous"
+		then
+			continue;
+		fi
+		URL=$(local_URL "$remote" "$URL")
+		if test -z "$URL"
+		then
+			continue;
+		fi
+		# At this point, we don't know if the submodule
+		# appears in the HEAD of the supermodule, so clone it
+		# without a checkout and overwrite HEAD so that a subsequent
+		# checkout won't assume the submodule has already been
+		# checked out.
+		git-clone --submodules -n "$URL" "$submodule"
+		z40=0000000000000000000000000000000000000000
+		GIT_DIR="$submodule/.git" git-update-ref --no-deref HEAD $z40
+		git-config "submodule.$submodule.url" "$URL"
+	done
+	)
+}
+
 all=
 no_tags=
 tags=
@@ -27,6 +91,7 @@ keep=
 shallow_depth=
 no_progress=
 use_separate_remote=
+submodules=
 test -t 1 || no_progress=--no-progress
 quiet=
 while case "$#" in 0) break ;; esac
@@ -84,6 +149,15 @@ do
 		shift
 		shallow_depth="--depth=$1"
 		;;
+	--su|--sub|--subm|--submo|--submod|--submodu|--submodul|\
+	--submodule|--submodules)
+		submodules="yes"
+		;;
+	--no-su|--no-sub|--no-subm|--no-submo|--no-submod|\
+	--no-submodu|--no-submodul|\
+	--no-submodule|--no-submodules)
+		submodules="no"
+		;;
 	-*)
 		usage
 		;;
@@ -149,6 +223,18 @@ case "$tags$no_tags" in
 	esac
 esac
 
+case "$submodules" in
+'')
+	case "$(git-config --bool core.submodules)" in
+	true)
+		submodues=yes
+		;;
+	*)
+		submodules=no
+		;;
+	esac
+esac
+
 # If --tags (and later --heads or --all) is specified, then we are
 # not talking about defaults stored in Pull: line of remotes or
 # branches file, and just fetch those and refspecs explicitly given.
@@ -407,3 +493,7 @@ case "$orig_head" in
 	fi
 	;;
 esac
+
+if test "$submodules" = yes; then
+	clone_submodules "$remote"
+fi
-- 
1.5.2.rc3.783.gc7476-dirty

Re: Second round of support for cloning submodules

[Sven Verdoolaege]

I forgot to mention that these changes go on top of next
(v1.5.2-rc3-762-g77e153b).

skimo

[PATCH] allow commands to be executed in submodules

[Martin Waitz]

Add an extra "submodule" field to struct child_process to be able to
easily start commands which are to be executed in a submodule
repository.

Signed-off-by: Martin Waitz <tali@admingilde.org>
---

 run-command.c     |   13 ++++++
 run-command.h     |    1 +
 2 files changed, 14 insertions(+), 0 deletions(-)

diff --git a/run-command.c b/run-command.c
index eff523e..c2475e4 100644
--- a/run-command.c
+++ b/run-command.c
@@ -73,6 +73,19 @@ int start_command(struct child_process *cmd)
 			close(cmd->out);
 		}
 
+		if (cmd->submodule) {
+			int err = chdir(cmd->submodule);
+			if (err) {
+				die("cannot exec %s in %s.",
+					cmd->argv[0], cmd->submodule);
+			}
+			/* don't inherit supermodule environment */
+			unsetenv(GIT_DIR_ENVIRONMENT);
+			unsetenv(DB_ENVIRONMENT);
+			unsetenv(INDEX_ENVIRONMENT);
+			unsetenv(GRAFT_ENVIRONMENT);
+		}
+
 		if (cmd->git_cmd) {
 			execv_git_cmd(cmd->argv);
 		} else {
diff --git a/run-command.h b/run-command.h
index 3680ef9..2940186 100644
--- a/run-command.h
+++ b/run-command.h
@@ -16,6 +16,7 @@ struct child_process {
 	pid_t pid;
 	int in;
 	int out;
+	const char *submodule;
 	unsigned close_in:1;
 	unsigned close_out:1;
 	unsigned no_stdin:1;
-- 
1.5.2.2.g081e


-- 
Martin Waitz

Re: [PATCH] allow commands to be executed in submodules

[Alex Riesen]

Martin Waitz, Sun, May 20, 2007 17:39:08 +0200:
> Add an extra "submodule" field to struct child_process to be able to
> easily start commands which are to be executed in a submodule
> repository.

How about making it more generic by allowing to specify the directory
to change to and environment for subprocess? You probably will be able
to convert even some of existing code to your new run_command then
(merge_recursive in builtin-revert.c, for example).

Something like this, perhaps:

diff --git a/run-command.c b/run-command.c
index eff523e..605aa1e 100644
--- a/run-command.c
+++ b/run-command.c
@@ -73,6 +73,13 @@ int start_command(struct child_process *cmd)
 			close(cmd->out);
 		}
 
+		if (cmd->dir && chdir(cmd->dir))
+			die("exec %s: cd to %s failed (%s)", cmd->argv[0],
+			    cmd->dir, strerror(errno));
+		if (cmd->env) {
+			for (; *cmd->env; cmd->env++)
+				putenv((char*)*cmd->env);
+		}
 		if (cmd->git_cmd) {
 			execv_git_cmd(cmd->argv);
 		} else {
@@ -133,13 +140,38 @@ int run_command(struct child_process *cmd)
 	return finish_command(cmd);
 }
 
+static void prepare_run_command_v_opt(struct child_process *cmd,
+				      const char **argv,
+				      int opt)
+{
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->argv = argv;
+	cmd->no_stdin = opt & RUN_COMMAND_NO_STDIN ? 1 : 0;
+	cmd->git_cmd = opt & RUN_GIT_CMD ? 1 : 0;
+	cmd->stdout_to_stderr = opt & RUN_COMMAND_STDOUT_TO_STDERR ? 1 : 0;
+}
+
 int run_command_v_opt(const char **argv, int opt)
 {
 	struct child_process cmd;
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.argv = argv;
-	cmd.no_stdin = opt & RUN_COMMAND_NO_STDIN ? 1 : 0;
-	cmd.git_cmd = opt & RUN_GIT_CMD ? 1 : 0;
-	cmd.stdout_to_stderr = opt & RUN_COMMAND_STDOUT_TO_STDERR ? 1 : 0;
+	prepare_run_command_v_opt(&cmd, argv, opt);
 	return run_command(&cmd);
 }
+
+int run_command_v_opt_cd(const char **argv, int opt, const char *dir)
+{
+	struct child_process cmd;
+	prepare_run_command_v_opt(&cmd, argv, opt);
+	cmd.dir = dir;
+	return run_command(&cmd);
+}
+
+int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const char *const *env)
+{
+	struct child_process cmd;
+	prepare_run_command_v_opt(&cmd, argv, opt);
+	cmd.dir = dir;
+	cmd.env = env;
+	return run_command(&cmd);
+}
+
diff --git a/run-command.h b/run-command.h
index 3680ef9..af1e0bf 100644
--- a/run-command.h
+++ b/run-command.h
@@ -16,6 +16,8 @@ struct child_process {
 	pid_t pid;
 	int in;
 	int out;
+	const char *dir;
+	const char *const *env;
 	unsigned close_in:1;
 	unsigned close_out:1;
 	unsigned no_stdin:1;
@@ -32,5 +34,7 @@ int run_command(struct child_process *);
 #define RUN_GIT_CMD	     2	/*If this is to be git sub-command */
 #define RUN_COMMAND_STDOUT_TO_STDERR 4
 int run_command_v_opt(const char **argv, int opt);
+int run_command_v_opt_cd(const char **argv, int opt, const char *dir);
+int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const char *const *env);
 
 #endif

[PATCH] Add ability to specify environment extension to run_command

[Alex Riesen]

There is no way to specify and override for the environment: there is
no visible user for it (yet, something in git-daemon could need it).

Signed-off-by: Alex Riesen <raa.lkml@gmail.com>
---

Sven Verdoolaege, Mon, May 21, 2007 11:03:39 +0200:
> Could you sign-off on this for me so I can use it my patch set?
> 

So here it is. On top of the previos patch regarding chdir before
exec. Junio, if needed, I can resend that first patch about chdir.

 run-command.c |   17 ++++++++++++++++-
 run-command.h |    2 ++
 2 files changed, 18 insertions(+), 1 deletions(-)

diff --git a/run-command.c b/run-command.c
index 043b570..605aa1e 100644
--- a/run-command.c
+++ b/run-command.c
@@ -76,6 +76,10 @@ int start_command(struct child_process *cmd)
 		if (cmd->dir && chdir(cmd->dir))
 			die("exec %s: cd to %s failed (%s)", cmd->argv[0],
 			    cmd->dir, strerror(errno));
+		if (cmd->env) {
+			for (; *cmd->env; cmd->env++)
+				putenv((char*)*cmd->env);
+		}
 		if (cmd->git_cmd) {
 			execv_git_cmd(cmd->argv);
 		} else {
@@ -137,7 +141,8 @@ int run_command(struct child_process *cmd)
 }
 
 static void prepare_run_command_v_opt(struct child_process *cmd,
-				      const char **argv, int opt)
+				      const char **argv,
+				      int opt)
 {
 	memset(cmd, 0, sizeof(*cmd));
 	cmd->argv = argv;
@@ -160,3 +165,13 @@ int run_command_v_opt_cd(const char **argv, int opt, const char *dir)
 	cmd.dir = dir;
 	return run_command(&cmd);
 }
+
+int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const char *const *env)
+{
+	struct child_process cmd;
+	prepare_run_command_v_opt(&cmd, argv, opt);
+	cmd.dir = dir;
+	cmd.env = env;
+	return run_command(&cmd);
+}
+
diff --git a/run-command.h b/run-command.h
index cbd7484..af1e0bf 100644
--- a/run-command.h
+++ b/run-command.h
@@ -17,6 +17,7 @@ struct child_process {
 	int in;
 	int out;
 	const char *dir;
+	const char *const *env;
 	unsigned close_in:1;
 	unsigned close_out:1;
 	unsigned no_stdin:1;
@@ -34,5 +35,6 @@ int run_command(struct child_process *);
 #define RUN_COMMAND_STDOUT_TO_STDERR 4
 int run_command_v_opt(const char **argv, int opt);
 int run_command_v_opt_cd(const char **argv, int opt, const char *dir);
+int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const char *const *env);
 
 #endif
-- 
1.5.2.rc3.112.gc1e43

Re: [PATCH] Add ability to specify environment extension to run_command

[Junio C Hamano]

Alex Riesen <raa.lkml@gmail.com> writes:

> There is no way to specify and override for the environment: there is
> no visible user for it (yet, something in git-daemon could need it).
>
> Signed-off-by: Alex Riesen <raa.lkml@gmail.com>
> ---
>
> Sven Verdoolaege, Mon, May 21, 2007 11:03:39 +0200:
>> Could you sign-off on this for me so I can use it my patch set?
>> 
>
> So here it is. On top of the previos patch regarding chdir before
> exec. Junio, if needed, I can resend that first patch about chdir.

Both of them in a row would be good, so yes, resend is
appreciated.

> @@ -76,6 +76,10 @@ int start_command(struct child_process *cmd)
>  		if (cmd->dir && chdir(cmd->dir))
>  			die("exec %s: cd to %s failed (%s)", cmd->argv[0],
>  			    cmd->dir, strerror(errno));
> +		if (cmd->env) {
> +			for (; *cmd->env; cmd->env++)
> +				putenv((char*)*cmd->env);
> +		}
>  		if (cmd->git_cmd) {
>  			execv_git_cmd(cmd->argv);
>  		} else {

I had a feeling that some callers needed to be able to unsetenv
some.  How would this patch help them, or are they outside of
the scope?

Re: [PATCH] Add ability to specify environment extension to run_command

[Alex Riesen]

Junio C Hamano, Tue, May 22, 2007 01:02:42 +0200:
> >
> > So here it is. On top of the previos patch regarding chdir before
> > exec. Junio, if needed, I can resend that first patch about chdir.
> 
> Both of them in a row would be good, so yes, resend is
> appreciated.

Will be resent.

> > @@ -76,6 +76,10 @@ int start_command(struct child_process *cmd)
> >  		if (cmd->dir && chdir(cmd->dir))
> >  			die("exec %s: cd to %s failed (%s)", cmd->argv[0],
> >  			    cmd->dir, strerror(errno));
> > +		if (cmd->env) {
> > +			for (; *cmd->env; cmd->env++)
> > +				putenv((char*)*cmd->env);
> > +		}
> >  		if (cmd->git_cmd) {
> >  			execv_git_cmd(cmd->argv);
> >  		} else {
> 
> I had a feeling that some callers needed to be able to unsetenv
> some.  How would this patch help them, or are they outside of
> the scope?
> 

Others already discussed the issue. Just to be sure, I reimplemented
that comfortable putenv with unsetenv: if an environment entry ends
with a "=" it will be unset.

[PATCH] Add run_command_v_opt_cd: chdir into a directory before exec

[Alex Riesen]

It can make code simplier (no need to preserve cwd) and safer
(no chance the cwd of the current process is accidentally forgotten).

Signed-off-by: Alex Riesen <raa.lkml@gmail.com>
---
 run-command.c |   27 ++++++++++++++++++++++-----
 run-command.h |    2 ++
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/run-command.c b/run-command.c
index eff523e..043b570 100644
--- a/run-command.c
+++ b/run-command.c
@@ -73,6 +73,9 @@ int start_command(struct child_process *cmd)
 			close(cmd->out);
 		}
 
+		if (cmd->dir && chdir(cmd->dir))
+			die("exec %s: cd to %s failed (%s)", cmd->argv[0],
+			    cmd->dir, strerror(errno));
 		if (cmd->git_cmd) {
 			execv_git_cmd(cmd->argv);
 		} else {
@@ -133,13 +136,27 @@ int run_command(struct child_process *cmd)
 	return finish_command(cmd);
 }
 
+static void prepare_run_command_v_opt(struct child_process *cmd,
+				      const char **argv, int opt)
+{
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->argv = argv;
+	cmd->no_stdin = opt & RUN_COMMAND_NO_STDIN ? 1 : 0;
+	cmd->git_cmd = opt & RUN_GIT_CMD ? 1 : 0;
+	cmd->stdout_to_stderr = opt & RUN_COMMAND_STDOUT_TO_STDERR ? 1 : 0;
+}
+
 int run_command_v_opt(const char **argv, int opt)
 {
 	struct child_process cmd;
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.argv = argv;
-	cmd.no_stdin = opt & RUN_COMMAND_NO_STDIN ? 1 : 0;
-	cmd.git_cmd = opt & RUN_GIT_CMD ? 1 : 0;
-	cmd.stdout_to_stderr = opt & RUN_COMMAND_STDOUT_TO_STDERR ? 1 : 0;
+	prepare_run_command_v_opt(&cmd, argv, opt);
+	return run_command(&cmd);
+}
+
+int run_command_v_opt_cd(const char **argv, int opt, const char *dir)
+{
+	struct child_process cmd;
+	prepare_run_command_v_opt(&cmd, argv, opt);
+	cmd.dir = dir;
 	return run_command(&cmd);
 }
diff --git a/run-command.h b/run-command.h
index 3680ef9..cbd7484 100644
--- a/run-command.h
+++ b/run-command.h
@@ -16,6 +16,7 @@ struct child_process {
 	pid_t pid;
 	int in;
 	int out;
+	const char *dir;
 	unsigned close_in:1;
 	unsigned close_out:1;
 	unsigned no_stdin:1;
@@ -32,5 +33,6 @@ int run_command(struct child_process *);
 #define RUN_GIT_CMD	     2	/*If this is to be git sub-command */
 #define RUN_COMMAND_STDOUT_TO_STDERR 4
 int run_command_v_opt(const char **argv, int opt);
+int run_command_v_opt_cd(const char **argv, int opt, const char *dir);
 
 #endif
-- 
1.5.2.51.g16099