* [PATCH v8 0/3] git-remote-fd & git-remote-ext
@ 2010-11-16 3:34 Ilari Liusvaara
2010-11-16 3:34 ` [PATCH v8 1/3] Add bidirectional_transfer_loop() Ilari Liusvaara
` (3 more replies)
0 siblings, 4 replies; 6+ messages in thread
From: Ilari Liusvaara @ 2010-11-16 3:34 UTC (permalink / raw)
To: git
This adds two new remote helpers.
* git-remote-fd, which connects to git service on given file descriptor(s),
useful for graphical user interfaces that want to use internal ssh client.
* git-remote-ext, which connect to git service using external program. Useful
for connecting using odd one-off ssh options, to services in abstract
namespace, using unix domain sockets, using TLS, etc...
Changes from last time:
* Some manpage formatting changes.
* Some manpage wording changes.
* Don't try to write outside buffers if all-whitespace command line is
received.
* Some code cleanups.
Ilari Liusvaara (3):
Add bidirectional_transfer_loop()
git-remote-fd
git-remote-ext
.gitignore | 2 +
Documentation/git-remote-ext.txt | 125 +++++++++++++++
Documentation/git-remote-fd.txt | 59 +++++++
Makefile | 2 +
builtin.h | 2 +
builtin/remote-ext.c | 246 +++++++++++++++++++++++++++++
builtin/remote-fd.c | 79 ++++++++++
compat/mingw.h | 5 +
git.c | 2 +
transport-helper.c | 316 ++++++++++++++++++++++++++++++++++++++
transport.h | 1 +
11 files changed, 839 insertions(+), 0 deletions(-)
create mode 100644 Documentation/git-remote-ext.txt
create mode 100644 Documentation/git-remote-fd.txt
create mode 100644 builtin/remote-ext.c
create mode 100644 builtin/remote-fd.c
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH v8 1/3] Add bidirectional_transfer_loop()
2010-11-16 3:34 [PATCH v8 0/3] git-remote-fd & git-remote-ext Ilari Liusvaara
@ 2010-11-16 3:34 ` Ilari Liusvaara
2010-11-16 3:34 ` [PATCH v8 2/3] git-remote-fd Ilari Liusvaara
` (2 subsequent siblings)
3 siblings, 0 replies; 6+ messages in thread
From: Ilari Liusvaara @ 2010-11-16 3:34 UTC (permalink / raw)
To: git
This helper function copies bidirectional stream of data between
stdin/stdout and specified file descriptors.
Signed-off-by: Ilari Liusvaara <ilari.liusvaara@elisanet.fi>
---
compat/mingw.h | 5 +
transport-helper.c | 316 ++++++++++++++++++++++++++++++++++++++++++++++++++++
transport.h | 1 +
3 files changed, 322 insertions(+), 0 deletions(-)
diff --git a/compat/mingw.h b/compat/mingw.h
index f465566..00ee685 100644
--- a/compat/mingw.h
+++ b/compat/mingw.h
@@ -23,6 +23,9 @@ typedef int pid_t;
#define WEXITSTATUS(x) ((x) & 0xff)
#define WTERMSIG(x) SIGTERM
+#define EWOULDBLOCK EAGAIN
+#define SHUT_WR SD_SEND
+
#define SIGHUP 1
#define SIGQUIT 3
#define SIGKILL 9
@@ -50,6 +53,8 @@ struct pollfd {
};
#define POLLIN 1
#define POLLHUP 2
+#define POLLOUT 4
+#define POLLNVAL 8
#endif
typedef void (__cdecl *sig_handler_t)(int);
diff --git a/transport-helper.c b/transport-helper.c
index 0381de5..b52c08e 100644
--- a/transport-helper.c
+++ b/transport-helper.c
@@ -9,6 +9,11 @@
#include "remote.h"
#include "string-list.h"
+#ifndef NO_PTHREADS
+#include <pthread.h>
+#include "thread-utils.h"
+#endif
+
static int debug;
struct helper_data
@@ -862,3 +867,314 @@ int transport_helper_init(struct transport *transport, const char *name)
transport->smart_options = &(data->transport_options);
return 0;
}
+
+/*
+ * Linux pipes can buffer 65536 bytes at once (and most platforms can
+ * buffer less), so attempt reads and writes with up to that size.
+ */
+#define BUFFERSIZE 65536
+/* This should be enough to hold debugging message. */
+#define PBUFFERSIZE 8192
+
+/* Print bidirectional transfer loop debug message. */
+static void transfer_debug(const char *fmt, ...)
+{
+ va_list args;
+ char msgbuf[PBUFFERSIZE];
+ static int debug_enabled = -1;
+
+ if (debug_enabled < 0)
+ debug_enabled = getenv("GIT_TRANSLOOP_DEBUG") ? 1 : 0;
+ if (!debug_enabled)
+ return;
+
+ va_start(args, fmt);
+ vsnprintf(msgbuf, PBUFFERSIZE, fmt, args);
+ va_end(args);
+ fprintf(stderr, "Transfer loop debugging: %s\n", msgbuf);
+}
+
+/* Stream state: More data may be coming in this direction. */
+#define SSTATE_TRANSFERING 0
+/*
+ * Stream state: No more data coming in this direction, flushing rest of
+ * data.
+ */
+#define SSTATE_FLUSHING 1
+/* Stream state: Transfer in this direction finished. */
+#define SSTATE_FINISHED 2
+
+#define STATE_NEEDS_READING(state) ((state) <= SSTATE_TRANSFERING)
+#define STATE_NEEDS_WRITING(state) ((state) <= SSTATE_FLUSHING)
+#define STATE_NEEDS_CLOSING(state) ((state) == SSTATE_FLUSHING)
+
+/* Unidirectional transfer. */
+struct unidirectional_transfer {
+ /* Source */
+ int src;
+ /* Destination */
+ int dest;
+ /* Is source socket? */
+ int src_is_sock;
+ /* Is destination socket? */
+ int dest_is_sock;
+ /* Transfer state (TRANSFERING/FLUSHING/FINISHED) */
+ int state;
+ /* Buffer. */
+ char buf[BUFFERSIZE];
+ /* Buffer used. */
+ size_t bufuse;
+ /* Name of source. */
+ const char *src_name;
+ /* Name of destination. */
+ const char *dest_name;
+};
+
+/* Closes the target (for writing) if transfer has finished. */
+static void udt_close_if_finished(struct unidirectional_transfer *t)
+{
+ if (STATE_NEEDS_CLOSING(t->state) && !t->bufuse) {
+ t->state = SSTATE_FINISHED;
+ if (t->dest_is_sock)
+ shutdown(t->dest, SHUT_WR);
+ else
+ close(t->dest);
+ transfer_debug("Closed %s.", t->dest_name);
+ }
+}
+
+/*
+ * Tries to read read data from source into buffer. If buffer is full,
+ * no data is read. Returns 0 on success, -1 on error.
+ */
+static int udt_do_read(struct unidirectional_transfer *t)
+{
+ ssize_t bytes;
+
+ if (t->bufuse == BUFFERSIZE)
+ return 0; /* No space for more. */
+
+ transfer_debug("%s is readable", t->src_name);
+ bytes = read(t->src, t->buf + t->bufuse, BUFFERSIZE - t->bufuse);
+ if (bytes < 0 && errno != EWOULDBLOCK && errno != EAGAIN &&
+ errno != EINTR) {
+ error("read(%s) failed: %s", t->src_name, strerror(errno));
+ return -1;
+ } else if (bytes == 0) {
+ transfer_debug("%s EOF (with %i bytes in buffer)",
+ t->src_name, t->bufuse);
+ t->state = SSTATE_FLUSHING;
+ } else if (bytes > 0) {
+ t->bufuse += bytes;
+ transfer_debug("Read %i bytes from %s (buffer now at %i)",
+ (int)bytes, t->src_name, (int)t->bufuse);
+ }
+ return 0;
+}
+
+/* Tries to write data from buffer into destination. If buffer is empty,
+ * no data is written. Returns 0 on success, -1 on error.
+ */
+static int udt_do_write(struct unidirectional_transfer *t)
+{
+ size_t bytes;
+
+ if (t->bufuse == 0)
+ return 0; /* Nothing to write. */
+
+ transfer_debug("%s is writable", t->dest_name);
+ bytes = write(t->dest, t->buf, t->bufuse);
+ if (bytes < 0 && errno != EWOULDBLOCK && errno != EAGAIN &&
+ errno != EINTR) {
+ error("write(%s) failed: %s", t->dest_name, strerror(errno));
+ return -1;
+ } else if (bytes > 0) {
+ t->bufuse -= bytes;
+ if (t->bufuse)
+ memmove(t->buf, t->buf + bytes, t->bufuse);
+ transfer_debug("Wrote %i bytes to %s (buffer now at %i)",
+ (int)bytes, t->dest_name, (int)t->bufuse);
+ }
+ return 0;
+}
+
+
+/* State of bidirectional transfer loop. */
+struct bidirectional_transfer_state {
+ /* Direction from program to git. */
+ struct unidirectional_transfer ptg;
+ /* Direction from git to program. */
+ struct unidirectional_transfer gtp;
+};
+
+static void *udt_copy_task_routine(void *udt)
+{
+ struct unidirectional_transfer *t = (struct unidirectional_transfer *)udt;
+ while (t->state != SSTATE_FINISHED) {
+ if (STATE_NEEDS_READING(t->state))
+ if (udt_do_read(t))
+ return NULL;
+ if (STATE_NEEDS_WRITING(t->state))
+ if (udt_do_write(t))
+ return NULL;
+ if (STATE_NEEDS_CLOSING(t->state))
+ udt_close_if_finished(t);
+ }
+ return udt; /* Just some non-NULL value. */
+}
+
+#ifndef NO_PTHREADS
+
+/*
+ * Join thread, with apporiate errors on failure. Name is name for the
+ * thread (for error messages). Returns 0 on success, 1 on failure.
+ */
+static int tloop_join(pthread_t thread, const char *name)
+{
+ int err;
+ void *tret;
+ err = pthread_join(thread, &tret);
+ if (!tret) {
+ error("%s thread failed", name);
+ return 1;
+ }
+ if (err) {
+ error("%s thread failed to join: %s", name, strerror(err));
+ return 1;
+ }
+ return 0;
+}
+
+/*
+ * Spawn the transfer tasks and then wait for them. Returns 0 on success,
+ * -1 on failure.
+ */
+static int tloop_spawnwait_tasks(struct bidirectional_transfer_state *s)
+{
+ pthread_t gtp_thread;
+ pthread_t ptg_thread;
+ int err;
+ int ret = 0;
+ err = pthread_create(>p_thread, NULL, udt_copy_task_routine,
+ &s->gtp);
+ if (err)
+ die("Can't start thread for copying data: %s", strerror(err));
+ err = pthread_create(&ptg_thread, NULL, udt_copy_task_routine,
+ &s->ptg);
+ if (err)
+ die("Can't start thread for copying data: %s", strerror(err));
+
+ ret |= tloop_join(gtp_thread, "Git to program copy");
+ ret |= tloop_join(ptg_thread, "Program to git copy");
+ return ret;
+}
+#else
+
+/* Close the source and target (for writing) for transfer. */
+static void udt_kill_transfer(struct unidirectional_transfer *t)
+{
+ t->state = SSTATE_FINISHED;
+ /*
+ * Socket read end left open isn't a disaster if nobody
+ * attempts to read from it (mingw compat headers do not
+ * have SHUT_RD)...
+ *
+ * We can't fully close the socket since otherwise gtp
+ * task would first close the socket it sends data to
+ * while closing the ptg file descriptors.
+ */
+ if (!t->src_is_sock)
+ close(t->src);
+ if (t->dest_is_sock)
+ shutdown(t->dest, SHUT_WR);
+ else
+ close(t->dest);
+}
+
+/*
+ * Join process, with apporiate errors on failure. Name is name for the
+ * process (for error messages). Returns 0 on success, 1 on failure.
+ */
+static int tloop_join(pid_t pid, const char *name)
+{
+ int tret;
+ if (waitpid(pid, &tret, 0) < 0) {
+ error("%s process failed to wait: %s", name, strerror(errno));
+ return 1;
+ }
+ if (!WIFEXITED(tret) || WEXITSTATUS(tret)) {
+ error("%s process failed", name);
+ return 1;
+ }
+ return 0;
+}
+
+/*
+ * Spawn the transfer tasks and then wait for them. Returns 0 on success,
+ * -1 on failure.
+ */
+static int tloop_spawnwait_tasks(struct bidirectional_transfer_state *s)
+{
+ pid_t pid1, pid2;
+ int ret = 0;
+
+ /* Fork thread #1: git to program. */
+ pid1 = fork();
+ if (pid1 < 0)
+ die_errno("Can't start thread for copying data");
+ else if (pid1 == 0) {
+ udt_kill_transfer(&s->ptg);
+ exit(udt_copy_task_routine(&s->gtp) ? 0 : 1);
+ }
+
+ /* Fork thread #2: program to git. */
+ pid2 = fork();
+ if (pid2 < 0)
+ die_errno("Can't start thread for copying data");
+ else if (pid2 == 0) {
+ udt_kill_transfer(&s->gtp);
+ exit(udt_copy_task_routine(&s->ptg) ? 0 : 1);
+ }
+
+ /*
+ * Close both streams in parent as to not interfere with
+ * end of file detection and wait for both tasks to finish.
+ */
+ udt_kill_transfer(&s->gtp);
+ udt_kill_transfer(&s->ptg);
+ ret |= tloop_join(pid1, "Git to program copy");
+ ret |= tloop_join(pid2, "Program to git copy");
+ return ret;
+}
+#endif
+
+/*
+ * Copies data from stdin to output and from input to stdout simultaneously.
+ * Additionally filtering through given filter. If filter is NULL, uses
+ * identity filter.
+ */
+int bidirectional_transfer_loop(int input, int output)
+{
+ struct bidirectional_transfer_state state;
+
+ /* Fill the state fields. */
+ state.ptg.src = input;
+ state.ptg.dest = 1;
+ state.ptg.src_is_sock = (input == output);
+ state.ptg.dest_is_sock = 0;
+ state.ptg.state = SSTATE_TRANSFERING;
+ state.ptg.bufuse = 0;
+ state.ptg.src_name = "remote input";
+ state.ptg.dest_name = "stdout";
+
+ state.gtp.src = 0;
+ state.gtp.dest = output;
+ state.gtp.src_is_sock = 0;
+ state.gtp.dest_is_sock = (input == output);
+ state.gtp.state = SSTATE_TRANSFERING;
+ state.gtp.bufuse = 0;
+ state.gtp.src_name = "stdin";
+ state.gtp.dest_name = "remote output";
+
+ return tloop_spawnwait_tasks(&state);
+}
diff --git a/transport.h b/transport.h
index c59d973..e803c0e 100644
--- a/transport.h
+++ b/transport.h
@@ -154,6 +154,7 @@ int transport_connect(struct transport *transport, const char *name,
/* Transport methods defined outside transport.c */
int transport_helper_init(struct transport *transport, const char *name);
+int bidirectional_transfer_loop(int input, int output);
/* common methods used by transport.c and builtin-send-pack.c */
void transport_verify_remote_names(int nr_heads, const char **heads);
--
1.7.1.336.ge601a.dirty
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PATCH v8 2/3] git-remote-fd
2010-11-16 3:34 [PATCH v8 0/3] git-remote-fd & git-remote-ext Ilari Liusvaara
2010-11-16 3:34 ` [PATCH v8 1/3] Add bidirectional_transfer_loop() Ilari Liusvaara
@ 2010-11-16 3:34 ` Ilari Liusvaara
2010-11-16 3:34 ` [PATCH v8 3/3] git-remote-ext Ilari Liusvaara
2010-11-17 17:15 ` [PATCH v8 0/3] git-remote-fd & git-remote-ext Junio C Hamano
3 siblings, 0 replies; 6+ messages in thread
From: Ilari Liusvaara @ 2010-11-16 3:34 UTC (permalink / raw)
To: git
This remote helper reflects raw smart remote transport stream back to the
calling program. This is useful for example if some UI wants to handle
ssh itself and not use hacks via GIT_SSH.
Signed-off-by: Ilari Liusvaara <ilari.liusvaara@elisanet.fi>
---
.gitignore | 1 +
Documentation/git-remote-fd.txt | 59 +++++++++++++++++++++++++++++
Makefile | 1 +
builtin.h | 1 +
builtin/remote-fd.c | 79 +++++++++++++++++++++++++++++++++++++++
git.c | 1 +
6 files changed, 142 insertions(+), 0 deletions(-)
create mode 100644 Documentation/git-remote-fd.txt
create mode 100644 builtin/remote-fd.c
diff --git a/.gitignore b/.gitignore
index 14e2b6b..7509b01 100644
--- a/.gitignore
+++ b/.gitignore
@@ -112,6 +112,7 @@
/git-remote-https
/git-remote-ftp
/git-remote-ftps
+/git-remote-fd
/git-remote-testgit
/git-repack
/git-replace
diff --git a/Documentation/git-remote-fd.txt b/Documentation/git-remote-fd.txt
new file mode 100644
index 0000000..abc4944
--- /dev/null
+++ b/Documentation/git-remote-fd.txt
@@ -0,0 +1,59 @@
+git-remote-fd(1)
+================
+
+NAME
+----
+git-remote-fd - Reflect smart transport stream back to caller
+
+SYNOPSIS
+--------
+"fd::<infd>[,<outfd>][/<anything>]" (as URL)
+
+DESCRIPTION
+-----------
+This helper uses specified file descriptors to connect to remote git server.
+This is not meant for end users but for programs and scripts calling git
+fetch, push or archive.
+
+If only <infd> is given, it is assumed to be bidirectional socket connected
+to remote git server (git-upload-pack, git-receive-pack or
+git-upload-achive). If both <infd> and <outfd> are given, they are assumed
+to be pipes connected to remote git server (<infd> being the inbound pipe
+and <outfd> being the outbound pipe.
+
+It is assumed that any handshaking procedures have already been completed
+(such as sending service request for git://) before this helper is started.
+
+<anything> can be any string. It is ignored. It is meant for provoding
+information to user in the URL in case that URL is displayed in some
+context.
+
+ENVIRONMENT VARIABLES
+---------------------
+GIT_TRANSLOOP_DEBUG::
+ If set, prints debugging information about various reads/writes.
+
+EXAMPLES
+--------
+git fetch fd::17 master::
+ Fetch master, using file descriptor #17 to communicate with
+ git-upload-pack.
+
+git fetch fd::17/foo master::
+ Same as above.
+
+git push fd::7,8 master (as URL)::
+ Push master, using file descriptor #7 to read data from
+ git-receive-pack and file descriptor #8 to write data to
+ same service.
+
+git push fd::7,8/bar master::
+ Same as above.
+
+Documentation
+--------------
+Documentation by Ilari Liusvaara and the git list <git@vger.kernel.org>
+
+GIT
+---
+Part of the linkgit:git[1] suite
diff --git a/Makefile b/Makefile
index 5fa893c..ad53b52 100644
--- a/Makefile
+++ b/Makefile
@@ -702,6 +702,7 @@ BUILTIN_OBJS += builtin/read-tree.o
BUILTIN_OBJS += builtin/receive-pack.o
BUILTIN_OBJS += builtin/reflog.o
BUILTIN_OBJS += builtin/remote.o
+BUILTIN_OBJS += builtin/remote-fd.o
BUILTIN_OBJS += builtin/replace.o
BUILTIN_OBJS += builtin/rerere.o
BUILTIN_OBJS += builtin/reset.o
diff --git a/builtin.h b/builtin.h
index 5c887ef..d870695 100644
--- a/builtin.h
+++ b/builtin.h
@@ -106,6 +106,7 @@ extern int cmd_read_tree(int argc, const char **argv, const char *prefix);
extern int cmd_receive_pack(int argc, const char **argv, const char *prefix);
extern int cmd_reflog(int argc, const char **argv, const char *prefix);
extern int cmd_remote(int argc, const char **argv, const char *prefix);
+extern int cmd_remote_fd(int argc, const char **argv, const char *prefix);
extern int cmd_config(int argc, const char **argv, const char *prefix);
extern int cmd_rerere(int argc, const char **argv, const char *prefix);
extern int cmd_reset(int argc, const char **argv, const char *prefix);
diff --git a/builtin/remote-fd.c b/builtin/remote-fd.c
new file mode 100644
index 0000000..1f2467b
--- /dev/null
+++ b/builtin/remote-fd.c
@@ -0,0 +1,79 @@
+#include "git-compat-util.h"
+#include "transport.h"
+
+/*
+ * URL syntax:
+ * 'fd::<inoutfd>[/<anything>]' Read/write socket pair
+ * <inoutfd>.
+ * 'fd::<infd>,<outfd>[/<anything>]' Read pipe <infd> and write
+ * pipe <outfd>.
+ * [foo] indicates 'foo' is optional. <anything> is any string.
+ *
+ * The data output to <outfd>/<inoutfd> should be passed unmolested to
+ * git-receive-pack/git-upload-pack/git-upload-archive and output of
+ * git-receive-pack/git-upload-pack/git-upload-archive should be passed
+ * unmolested to <infd>/<inoutfd>.
+ *
+ */
+
+#define MAXCOMMAND 4096
+
+static void command_loop(int input_fd, int output_fd)
+{
+ char buffer[MAXCOMMAND];
+
+ while (1) {
+ size_t i;
+ if (!fgets(buffer, MAXCOMMAND - 1, stdin)) {
+ if (ferror(stdin))
+ die("Input error");
+ return;
+ }
+ /* Strip end of line characters. */
+ i = strlen(buffer);
+ while (i > 0 && isspace(buffer[i - 1]))
+ buffer[--i] = 0;
+
+ if (!strcmp(buffer, "capabilities")) {
+ printf("*connect\n\n");
+ fflush(stdout);
+ } else if (!strncmp(buffer, "connect ", 8)) {
+ printf("\n");
+ fflush(stdout);
+ if (bidirectional_transfer_loop(input_fd,
+ output_fd))
+ die("Copying data between file descriptors failed");
+ return;
+ } else {
+ die("Bad command: %s", buffer);
+ }
+ }
+}
+
+int cmd_remote_fd(int argc, const char **argv, const char *prefix)
+{
+ int input_fd = -1;
+ int output_fd = -1;
+ char *end;
+
+ if (argc != 3)
+ die("Expected two arguments");
+
+ input_fd = (int)strtoul(argv[2], &end, 10);
+
+ if ((end == argv[2]) || (*end != ',' && *end != '/' && *end))
+ die("Bad URL syntax");
+
+ if (*end == '/' || !*end) {
+ output_fd = input_fd;
+ } else {
+ char *end2;
+ output_fd = (int)strtoul(end + 1, &end2, 10);
+
+ if ((end2 == end + 1) || (*end2 != '/' && *end2))
+ die("Bad URL syntax");
+ }
+
+ command_loop(input_fd, output_fd);
+ return 0;
+}
diff --git a/git.c b/git.c
index 99f0363..f5fc07b 100644
--- a/git.c
+++ b/git.c
@@ -368,6 +368,7 @@ static void handle_internal_command(int argc, const char **argv)
{ "receive-pack", cmd_receive_pack },
{ "reflog", cmd_reflog, RUN_SETUP },
{ "remote", cmd_remote, RUN_SETUP },
+ { "remote-fd", cmd_remote_fd },
{ "replace", cmd_replace, RUN_SETUP },
{ "repo-config", cmd_config },
{ "rerere", cmd_rerere, RUN_SETUP },
--
1.7.1.336.ge601a.dirty
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PATCH v8 3/3] git-remote-ext
2010-11-16 3:34 [PATCH v8 0/3] git-remote-fd & git-remote-ext Ilari Liusvaara
2010-11-16 3:34 ` [PATCH v8 1/3] Add bidirectional_transfer_loop() Ilari Liusvaara
2010-11-16 3:34 ` [PATCH v8 2/3] git-remote-fd Ilari Liusvaara
@ 2010-11-16 3:34 ` Ilari Liusvaara
2010-11-17 17:15 ` [PATCH v8 0/3] git-remote-fd & git-remote-ext Junio C Hamano
3 siblings, 0 replies; 6+ messages in thread
From: Ilari Liusvaara @ 2010-11-16 3:34 UTC (permalink / raw)
To: git
This remote helper invokes external command and passes raw smart transport
stream through it. This is useful for instance for invoking ssh with
one-off odd options, connecting to git services in unix domain
sockets, in abstract namespace, using TLS or other secure protocols,
etc...
Signed-off-by: Ilari Liusvaara <ilari.liusvaara@elisanet.fi>
---
.gitignore | 1 +
Documentation/git-remote-ext.txt | 125 +++++++++++++++++++
Makefile | 1 +
builtin.h | 1 +
builtin/remote-ext.c | 246 ++++++++++++++++++++++++++++++++++++++
git.c | 1 +
6 files changed, 375 insertions(+), 0 deletions(-)
create mode 100644 Documentation/git-remote-ext.txt
create mode 100644 builtin/remote-ext.c
diff --git a/.gitignore b/.gitignore
index 7509b01..2b3111f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -113,6 +113,7 @@
/git-remote-ftp
/git-remote-ftps
/git-remote-fd
+/git-remote-ext
/git-remote-testgit
/git-repack
/git-replace
diff --git a/Documentation/git-remote-ext.txt b/Documentation/git-remote-ext.txt
new file mode 100644
index 0000000..f4fbf67
--- /dev/null
+++ b/Documentation/git-remote-ext.txt
@@ -0,0 +1,125 @@
+git-remote-ext(1)
+=================
+
+NAME
+----
+git-remote-ext - Bridge smart transport to external command.
+
+SYNOPSIS
+--------
+git remote add nick "ext::<command>[ <arguments>...]"
+
+DESCRIPTION
+-----------
+This remote helper uses the specified 'program' to connect
+to a remote git server.
+
+Data written to stdin of this specified 'program' is assumed
+to be sent to git:// server, git-upload-pack, git-receive-pack
+or git-upload-archive (depending on situation), and data read
+from stdout of this program is assumed to be received from
+the same service.
+
+Command and arguments are separated by unescaped space.
+
+The following sequences have a special meaning:
+
+'% '::
+ Literal space in command or argument.
+
+'%%'::
+ Literal percent sign.
+
+'%s'::
+ Replaced with name (receive-pack, upload-pack, or
+ upload-archive) of the service git wants to invoke.
+
+'%S'::
+ Replaced with long name (git-receive-pack,
+ git-upload-pack, or git-upload-archive) of the service
+ git wants to invoke.
+
+'%G' (must be first characters in argument)::
+ This argument will not be passed to 'program'. Instead, it
+ will cause helper to start by sending git:// service request to
+ remote side with service field set to approiate value and
+ repository field set to rest of the argument. Default is not to send
+ such request.
++
+This is useful if remote side is git:// server accessed over
+some tunnel.
+
+'%V' (must be first characters in argument)::
+ This argument will not be passed to 'program'. Instead it sets
+ the vhost field in git:// service request (to rest of the argument).
+ Default is not to send vhost in such request (if sent).
+
+ENVIRONMENT VARIABLES:
+----------------------
+
+GIT_TRANSLOOP_DEBUG::
+ If set, prints debugging information about various reads/writes.
+
+ENVIRONMENT VARIABLES PASSED TO COMMAND:
+----------------------------------------
+
+GIT_EXT_SERVICE::
+ Set to long name (git-upload-pack, etc...) of service helper needs
+ to invoke.
+
+GIT_EXT_SERVICE_NOPREFIX::
+ Set to long name (upload-pack, etc...) of service helper needs
+ to invoke.
+
+
+EXAMPLES:
+---------
+This remote helper is transparently used by git when
+you use commands such as "git fetch <URL>", "git clone <URL>",
+, "git push <URL>" or "git remote add nick <URL>", where <URL>
+begins with `ext::`. Examples:
+
+"ext::ssh -i /home/foo/.ssh/somekey user@host.example %S 'foo/repo'"::
+ Like host.example:foo/repo, but use /home/foo/.ssh/somekey as
+ keypair and user as user on remote side. This avoids needing to
+ edit .ssh/config.
+
+"ext::socat -t3600 - ABSTRACT-CONNECT:/git-server %G/somerepo"::
+ Represents repository with path /somerepo accessable over
+ git protocol at abstract namespace address /git-server.
+
+"ext::git-server-alias foo %G/repo"::
+ Represents a repository with path /repo accessed using the
+ helper program "git-server-alias foo". The path to the
+ repository and type of request are not passed on the command
+ line but as part of the protocol stream, as usual with git://
+ protocol.
+
+"ext::git-server-alias foo %G/repo %Vfoo"::
+ Represents a repository with path /repo accessed using the
+ helper program "git-server-alias foo". The hostname for the
+ remote server passed in the protocol stream will be "foo"
+ (this allows multiple virtual git servers to share a
+ link-level address).
+
+"ext::git-server-alias foo %G/repo% with% spaces %Vfoo"::
+ Represents a repository with path '/repo with spaces' accessed
+ using the helper program "git-server-alias foo". The hostname for
+ the remote server passed in the protocol stream will be "foo"
+ (this allows multiple virtual git servers to share a
+ link-level address).
+
+"ext::git-ssl foo.example /bar"::
+ Represents a repository accessed using the helper program
+ "git-ssl foo.example /bar". The type of request can be
+ determined by the helper using environment variables (see
+ above).
+
+Documentation
+--------------
+Documentation by Ilari Liusvaara, Jonathan Nieder and the git list
+<git@vger.kernel.org>
+
+GIT
+---
+Part of the linkgit:git[1] suite
diff --git a/Makefile b/Makefile
index ad53b52..88e752f 100644
--- a/Makefile
+++ b/Makefile
@@ -702,6 +702,7 @@ BUILTIN_OBJS += builtin/read-tree.o
BUILTIN_OBJS += builtin/receive-pack.o
BUILTIN_OBJS += builtin/reflog.o
BUILTIN_OBJS += builtin/remote.o
+BUILTIN_OBJS += builtin/remote-ext.o
BUILTIN_OBJS += builtin/remote-fd.o
BUILTIN_OBJS += builtin/replace.o
BUILTIN_OBJS += builtin/rerere.o
diff --git a/builtin.h b/builtin.h
index d870695..2fb609f 100644
--- a/builtin.h
+++ b/builtin.h
@@ -106,6 +106,7 @@ extern int cmd_read_tree(int argc, const char **argv, const char *prefix);
extern int cmd_receive_pack(int argc, const char **argv, const char *prefix);
extern int cmd_reflog(int argc, const char **argv, const char *prefix);
extern int cmd_remote(int argc, const char **argv, const char *prefix);
+extern int cmd_remote_ext(int argc, const char **argv, const char *prefix);
extern int cmd_remote_fd(int argc, const char **argv, const char *prefix);
extern int cmd_config(int argc, const char **argv, const char *prefix);
extern int cmd_rerere(int argc, const char **argv, const char *prefix);
diff --git a/builtin/remote-ext.c b/builtin/remote-ext.c
new file mode 100644
index 0000000..1f77317
--- /dev/null
+++ b/builtin/remote-ext.c
@@ -0,0 +1,246 @@
+#include "git-compat-util.h"
+#include "transport.h"
+#include "run-command.h"
+
+/*
+ * URL syntax:
+ * 'command [arg1 [arg2 [...]]]' Invoke command with given arguments.
+ * Special characters:
+ * '% ': Literal space in argument.
+ * '%%': Literal percent sign.
+ * '%S': Name of service (git-upload-pack/git-upload-archive/
+ * git-receive-pack.
+ * '%s': Same as \s, but with possible git- prefix stripped.
+ * '%G': Only allowed as first 'character' of argument. Do not pass this
+ * Argument to command, instead send this as name of repository
+ * in in-line git://-style request (also activates sending this
+ * style of request).
+ * '%V': Only allowed as first 'character' of argument. Used in
+ * conjunction with '%G': Do not pass this argument to command,
+ * instead send this as vhost in git://-style request (note: does
+ * not activate sending git:// style request).
+ */
+
+static char *git_req;
+static char *git_req_vhost;
+
+static char *strip_escapes(const char *str, const char *service,
+ const char **next)
+{
+ size_t rpos = 0;
+ int escape = 0;
+ char special = 0;
+ size_t pslen = 0;
+ size_t pSlen = 0;
+ size_t psoff = 0;
+ struct strbuf ret = STRBUF_INIT;
+
+ /* Calculate prefix length for \s and lengths for \s and \S */
+ if (!strncmp(service, "git-", 4))
+ psoff = 4;
+ pSlen = strlen(service);
+ pslen = pSlen - psoff;
+
+ /* Pass the service to command. */
+ setenv("GIT_EXT_SERVICE", service, 1);
+ setenv("GIT_EXT_SERVICE_NOPREFIX", service + psoff, 1);
+
+ /* Scan the length of argument. */
+ while (str[rpos] && (escape || str[rpos] != ' ')) {
+ if (escape) {
+ switch (str[rpos]) {
+ case ' ':
+ case '%':
+ case 's':
+ case 'S':
+ break;
+ case 'G':
+ case 'V':
+ special = str[rpos];
+ if (rpos == 1)
+ break;
+ /* Fall-through to error. */
+ default:
+ die("Bad remote-ext placeholder '%%%c'.",
+ str[rpos]);
+ }
+ escape = 0;
+ } else
+ escape = (str[rpos] == '%');
+ rpos++;
+ }
+ if (escape && !str[rpos])
+ die("remote-ext command has incomplete placeholder");
+ *next = str + rpos;
+ if (**next == ' ')
+ ++*next; /* Skip over space */
+
+ /*
+ * Do the actual placeholder substitution. The string will be short
+ * enough not to overflow integers.
+ */
+ rpos = special ? 2 : 0; /* Skip first 2 bytes in specials. */
+ escape = 0;
+ while (str[rpos] && (escape || str[rpos] != ' ')) {
+ if (escape) {
+ switch (str[rpos]) {
+ case ' ':
+ case '%':
+ strbuf_addch(&ret, str[rpos]);
+ break;
+ case 's':
+ strbuf_addstr(&ret, service + psoff);
+ break;
+ case 'S':
+ strbuf_addstr(&ret, service);
+ break;
+ }
+ escape = 0;
+ } else
+ switch (str[rpos]) {
+ case '%':
+ escape = 1;
+ break;
+ default:
+ strbuf_addch(&ret, str[rpos]);
+ break;
+ }
+ rpos++;
+ }
+ switch (special) {
+ case 'G':
+ git_req = strbuf_detach(&ret, NULL);
+ return NULL;
+ case 'V':
+ git_req_vhost = strbuf_detach(&ret, NULL);
+ return NULL;
+ default:
+ return strbuf_detach(&ret, NULL);
+ }
+}
+
+/* Should be enough... */
+#define MAXARGUMENTS 256
+
+static const char **parse_argv(const char *arg, const char *service)
+{
+ int arguments = 0;
+ int i;
+ const char **ret;
+ char *temparray[MAXARGUMENTS + 1];
+
+ while (*arg) {
+ char *expanded;
+ if (arguments == MAXARGUMENTS)
+ die("remote-ext command has too many arguments");
+ expanded = strip_escapes(arg, service, &arg);
+ if (expanded)
+ temparray[arguments++] = expanded;
+ }
+
+ ret = xmalloc((arguments + 1) * sizeof(char *));
+ for (i = 0; i < arguments; i++)
+ ret[i] = temparray[i];
+ ret[arguments] = NULL;
+ return ret;
+}
+
+static void send_git_request(int stdin_fd, const char *serv, const char *repo,
+ const char *vhost)
+{
+ size_t bufferspace;
+ size_t wpos = 0;
+ char *buffer;
+
+ /*
+ * Request needs 12 bytes extra if there is vhost (xxxx \0host=\0) and
+ * 6 bytes extra (xxxx \0) if there is no vhost.
+ */
+ if (vhost)
+ bufferspace = strlen(serv) + strlen(repo) + strlen(vhost) + 12;
+ else
+ bufferspace = strlen(serv) + strlen(repo) + 6;
+
+ if (bufferspace > 0xFFFF)
+ die("Request too large to send");
+ buffer = xmalloc(bufferspace);
+
+ /* Make the packet. */
+ wpos = sprintf(buffer, "%04x%s %s%c", (unsigned)bufferspace,
+ serv, repo, 0);
+
+ /* Add vhost if any. */
+ if (vhost)
+ sprintf(buffer + wpos, "host=%s%c", vhost, 0);
+
+ /* Send the request */
+ if (write_in_full(stdin_fd, buffer, bufferspace) < 0)
+ die_errno("Failed to send request");
+
+ free(buffer);
+}
+
+static int run_child(const char *arg, const char *service)
+{
+ int r;
+ struct child_process child;
+
+ memset(&child, 0, sizeof(child));
+ child.in = -1;
+ child.out = -1;
+ child.err = 0;
+ child.argv = parse_argv(arg, service);
+
+ if (start_command(&child) < 0)
+ die("Can't run specified command");
+
+ if (git_req)
+ send_git_request(child.in, service, git_req, git_req_vhost);
+
+ r = bidirectional_transfer_loop(child.out, child.in);
+ if (!r)
+ r = finish_command(&child);
+ else
+ finish_command(&child);
+ return r;
+}
+
+#define MAXCOMMAND 4096
+
+static int command_loop(const char *child)
+{
+ char buffer[MAXCOMMAND];
+
+ while (1) {
+ size_t length;
+ if (!fgets(buffer, MAXCOMMAND - 1, stdin)) {
+ if (ferror(stdin))
+ die("Comammand input error");
+ exit(0);
+ }
+ /* Strip end of line characters. */
+ length = strlen(buffer);
+ while (isspace((unsigned char)buffer[length - 1]))
+ buffer[--length] = 0;
+
+ if (!strcmp(buffer, "capabilities")) {
+ printf("*connect\n\n");
+ fflush(stdout);
+ } else if (!strncmp(buffer, "connect ", 8)) {
+ printf("\n");
+ fflush(stdout);
+ return run_child(child, buffer + 8);
+ } else {
+ fprintf(stderr, "Bad command");
+ return 1;
+ }
+ }
+}
+
+int cmd_remote_ext(int argc, const char **argv, const char *prefix)
+{
+ if (argc != 3)
+ die("Expected two arguments");
+
+ return command_loop(argv[2]);
+}
diff --git a/git.c b/git.c
index f5fc07b..ab67f9f 100644
--- a/git.c
+++ b/git.c
@@ -368,6 +368,7 @@ static void handle_internal_command(int argc, const char **argv)
{ "receive-pack", cmd_receive_pack },
{ "reflog", cmd_reflog, RUN_SETUP },
{ "remote", cmd_remote, RUN_SETUP },
+ { "remote-ext", cmd_remote_ext },
{ "remote-fd", cmd_remote_fd },
{ "replace", cmd_replace, RUN_SETUP },
{ "repo-config", cmd_config },
--
1.7.1.336.ge601a.dirty
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [PATCH v8 0/3] git-remote-fd & git-remote-ext
2010-11-16 3:34 [PATCH v8 0/3] git-remote-fd & git-remote-ext Ilari Liusvaara
` (2 preceding siblings ...)
2010-11-16 3:34 ` [PATCH v8 3/3] git-remote-ext Ilari Liusvaara
@ 2010-11-17 17:15 ` Junio C Hamano
2010-11-18 2:31 ` Ilari Liusvaara
3 siblings, 1 reply; 6+ messages in thread
From: Junio C Hamano @ 2010-11-17 17:15 UTC (permalink / raw)
To: Ilari Liusvaara; +Cc: git
Thanks; the changes from the last round all look sensible to me, but did
you know that the last round has already been in 'next' for a few weeks by
now already?
If you don't mind I'll turn these three into the following and queue as
4/3 on top of what is already merged, as reverting the merge with the old
series and merging the new round to 'next' makes it messier than it
already is.
-- >8 --
From: Ilari Liusvaara <ilari.liusvaara@elisanet.fi>
Subject: remote-fd/ext: finishing touches after code review
When compiling with pthread support, transport-helper.c needs to include
necessary header files. Also fix a few error messages in remote-ext and
remote-fd programs, and a potential buffer underrun in remote-fd.
In the documentation, clarify how %G and %V are used; the old description
looked as if they take repository/vhost parameters, which was wrong.
Also fix AsciiDoc markup for the page title of remote-fd/remote-ext manpages,
and tweak the way how section headers are shown.
Signed-off-by: Ilari Liusvaara <ilari.liusvaara@elisanet.fi>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
Documentation/git-remote-ext.txt | 10 +++++-----
Documentation/git-remote-fd.txt | 10 +++++-----
builtin/remote-ext.c | 8 +++-----
builtin/remote-fd.c | 6 +++---
transport-helper.c | 5 +++++
5 files changed, 21 insertions(+), 18 deletions(-)
diff --git a/Documentation/git-remote-ext.txt b/Documentation/git-remote-ext.txt
index 8fb260b..f4fbf67 100644
--- a/Documentation/git-remote-ext.txt
+++ b/Documentation/git-remote-ext.txt
@@ -39,20 +39,20 @@ The following sequences have a special meaning:
git-upload-pack, or git-upload-archive) of the service
git wants to invoke.
-'%G<repository>' (as argument)::
+'%G' (must be first characters in argument)::
This argument will not be passed to 'program'. Instead, it
will cause helper to start by sending git:// service request to
remote side with service field set to approiate value and
- repository field set to <repository>. Default is not to send
+ repository field set to rest of the argument. Default is not to send
such request.
+
This is useful if remote side is git:// server accessed over
some tunnel.
-'%V<host>' (as argument)::
+'%V' (must be first characters in argument)::
This argument will not be passed to 'program'. Instead it sets
- the vhost field in git:// service request. Default is not to
- send vhost in such request (if sent).
+ the vhost field in git:// service request (to rest of the argument).
+ Default is not to send vhost in such request (if sent).
ENVIRONMENT VARIABLES:
----------------------
diff --git a/Documentation/git-remote-fd.txt b/Documentation/git-remote-fd.txt
index 1c1a179..abc4944 100644
--- a/Documentation/git-remote-fd.txt
+++ b/Documentation/git-remote-fd.txt
@@ -1,5 +1,5 @@
git-remote-fd(1)
-=================
+================
NAME
----
@@ -28,13 +28,13 @@ It is assumed that any handshaking procedures have already been completed
information to user in the URL in case that URL is displayed in some
context.
-ENVIRONMENT VARIABLES:
-----------------------
+ENVIRONMENT VARIABLES
+---------------------
GIT_TRANSLOOP_DEBUG::
If set, prints debugging information about various reads/writes.
-EXAMPLES:
----------
+EXAMPLES
+--------
git fetch fd::17 master::
Fetch master, using file descriptor #17 to communicate with
git-upload-pack.
diff --git a/builtin/remote-ext.c b/builtin/remote-ext.c
index 1d3f1cf..1f77317 100644
--- a/builtin/remote-ext.c
+++ b/builtin/remote-ext.c
@@ -142,7 +142,7 @@ static const char **parse_argv(const char *arg, const char *service)
for (i = 0; i < arguments; i++)
ret[i] = temparray[i];
ret[arguments] = NULL;
- return (const char **)ret;
+ return ret;
}
static void send_git_request(int stdin_fd, const char *serv, const char *repo,
@@ -239,10 +239,8 @@ static int command_loop(const char *child)
int cmd_remote_ext(int argc, const char **argv, const char *prefix)
{
- if (argc < 3) {
- fprintf(stderr, "Error: URL missing");
- exit(1);
- }
+ if (argc != 3)
+ die("Expected two arguments");
return command_loop(argv[2]);
}
diff --git a/builtin/remote-fd.c b/builtin/remote-fd.c
index 7517f24..1f2467b 100644
--- a/builtin/remote-fd.c
+++ b/builtin/remote-fd.c
@@ -31,7 +31,7 @@ static void command_loop(int input_fd, int output_fd)
}
/* Strip end of line characters. */
i = strlen(buffer);
- while (isspace(buffer[i - 1]))
+ while (i > 0 && isspace(buffer[i - 1]))
buffer[--i] = 0;
if (!strcmp(buffer, "capabilities")) {
@@ -56,8 +56,8 @@ int cmd_remote_fd(int argc, const char **argv, const char *prefix)
int output_fd = -1;
char *end;
- if (argc < 3)
- die("URL missing");
+ if (argc != 3)
+ die("Expected two arguments");
input_fd = (int)strtoul(argv[2], &end, 10);
diff --git a/transport-helper.c b/transport-helper.c
index 0c094d6..3a50856 100644
--- a/transport-helper.c
+++ b/transport-helper.c
@@ -9,6 +9,11 @@
#include "remote.h"
#include "string-list.h"
+#ifndef NO_PTHREADS
+#include <pthread.h>
+#include "thread-utils.h"
+#endif
+
static int debug;
struct helper_data
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [PATCH v8 0/3] git-remote-fd & git-remote-ext
2010-11-17 17:15 ` [PATCH v8 0/3] git-remote-fd & git-remote-ext Junio C Hamano
@ 2010-11-18 2:31 ` Ilari Liusvaara
0 siblings, 0 replies; 6+ messages in thread
From: Ilari Liusvaara @ 2010-11-18 2:31 UTC (permalink / raw)
To: Junio C Hamano; +Cc: git
On Wed, Nov 17, 2010 at 09:15:34AM -0800, Junio C Hamano wrote:
> Thanks; the changes from the last round all look sensible to me, but did
> you know that the last round has already been in 'next' for a few weeks by
> now already?
Oops... Sorry. :-/
> If you don't mind I'll turn these three into the following and queue as
> 4/3 on top of what is already merged, as reverting the merge with the old
> series and merging the new round to 'next' makes it messier than it
> already is.
I tested applying that patch. Doesn't seem to break things at least code-
wise (no surprise, it only adds headers and adds one conditional that only
evaluates to false on invalid input).
So I don't mind applying that patch. Acking the patch.
-Ilari
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2010-11-18 2:31 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2010-11-16 3:34 [PATCH v8 0/3] git-remote-fd & git-remote-ext Ilari Liusvaara
2010-11-16 3:34 ` [PATCH v8 1/3] Add bidirectional_transfer_loop() Ilari Liusvaara
2010-11-16 3:34 ` [PATCH v8 2/3] git-remote-fd Ilari Liusvaara
2010-11-16 3:34 ` [PATCH v8 3/3] git-remote-ext Ilari Liusvaara
2010-11-17 17:15 ` [PATCH v8 0/3] git-remote-fd & git-remote-ext Junio C Hamano
2010-11-18 2:31 ` Ilari Liusvaara
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).