extended hook api and tweak-fetch hook

extended hook api and tweak-fetch hook

[Joey Hess]

This patch series adds an extended hook API, and uses it to implement
the new tweak-fetch hook.

The remaining hooks (that do not already use run_hook()) could be
refactored later to use this new API.

Also, the API has been designed to allow several programs to be run
for a single hook, when someone wants to add that into git.

[PATCH 1/3] expanded hook api with stdio support

[Joey Hess]

Adds run_hook_complex() and the struct hook, and implements
run_hook() using it.

This new API for hooks will allow for a later refactoring of the existing
ad-hoc hook code for hooks that are fed input on stdin, such as pre-receive,
post-receive, and post-rewrite.

It also adds support for a new class of hooks whose stdout is processed by
git, as well as hooks that both receive stdin and have their stdout
processed. This is controlled by setting "generator" and "reader" members
of the struct hook.

The API provides control over whether a hook must consume all its stdin,
or is free to ignore some of it; this can be specified by using either
feed_hook_in_full() or feed_hook_incomplete() as the "feeder" member of
the struct hook. The stdin feeder runs asynchronously, to avoid blocking
when the hook's stdin is also being read. So the API design limits the
code that needs to run asynchronously, to make it easy to implement
thread-safe feeders.

Finally, the API is designed to be extended in the future, to support
running multiple programs for a single hook action (such as the contents
of a .git/hooks/hook.d/ , or a system-wide hook). This design goal led
to the "generator" and "reader" members of the struct hook, which are
specified such that they can be called repeatedly, with data flowing
between them (via the "data" member), like this:
    generator | hook_prog_1 | reader | generator | hook_prog_2 | reader

Signed-off-by: Joey Hess <joey@kitenet.net>
---
 Documentation/technical/api-run-command.txt |   53 +++++++++++
 run-command.c                               |  132 ++++++++++++++++++++++++---
 run-command.h                               |   58 +++++++++++-
 3 files changed, 226 insertions(+), 17 deletions(-)

diff --git a/Documentation/technical/api-run-command.txt b/Documentation/technical/api-run-command.txt
index f18b4f4..ff50d2f 100644
--- a/Documentation/technical/api-run-command.txt
+++ b/Documentation/technical/api-run-command.txt
@@ -87,6 +87,17 @@ The functions above do the following:
 	On execution, .stdout_to_stderr and .no_stdin will be set.
 	(See below.)
 
+`run_hook_complex`::
+
+	Run a hook, with the caller providing its stdin and/or parsing its
+	stdout.
+	Takes a pointer to a `struct hook` that specifies the details,
+	including the name of the hook, any parameters to pass to it,
+	and how to handle the stdin and stdout. (See below.)
+	If the hook does not exist or is not executable, the return value
+	will be zero.
+	If it is executable, the hook will be executed and the exit
+	status of the hook is returned.
 
 Data structures
 ---------------
@@ -241,3 +252,45 @@ a forked process otherwise:
 
 . It must not change the program's state that the caller of the
   facility also uses.
+
+* `struct hook`
+
+This describes a hook to run.
+
+The caller:
+
+1. allocates and clears (memset(&hook, 0, sizeof(hook));) a
+   struct hook variable;
+2. initializes the members;
+3. calls hook();
+4. if necessary, accesses data read from the hook in .data
+5. frees the struct hook.
+
+The `struct hook` has three function pointers that may be set to
+control the stdin that is sent to the hook, and to collect its stdout.
+
+The `generator` generates the stdin for the hook, returning it in a strbuf.
+It is passed a pointer to the `struct hook`.
+
+The `feeder` is run asynchronously to feed the generated stdin into the hook.
+It is passed the handle to write to, the strbuf containing the stdin, and 
+a pointer to the `struct hook`, and should return non-zero on failure.
+Typically it is set to either `feed_hook_in_full`, or `feed_hook_incomplete`.
+
+The `reader` reads and processes the hook's stdout. It is passed 
+a handle to read from and a pointer to the `struct hook`, and should return
+non-zero on failure.
+
+If the generator or feeder are NULL, the hook is not fed anything
+on stdin; if the `reader` is NULL, the hook's stdout is sent to
+stderr instead.
+
+Note that in the future, the generator, feeder, and reader might be run
+more than once, if multiple programs are run as part of a single hook.
+Therefore, all three should avoid taking any actions except for generating
+the stdin, writing it to the hook, reading/parsing the hook's stdout,
+and printing any necessary warning messages.
+
+The `struct hook` also contains a `data` pointer, which can be used
+to communicate between the generator, feeder, reader, and the
+caller of the hook.
diff --git a/run-command.c b/run-command.c
index 1c51043..42a9b06 100644
--- a/run-command.c
+++ b/run-command.c
@@ -605,34 +605,136 @@ int finish_async(struct async *async)
 
 int run_hook(const char *index_file, const char *name, ...)
 {
-	struct child_process hook;
+	struct hook hook;
 	struct argv_array argv = ARGV_ARRAY_INIT;
-	const char *p, *env[2];
-	char index[PATH_MAX];
+	const char *p;
 	va_list args;
 	int ret;
 
-	if (access(git_path("hooks/%s", name), X_OK) < 0)
-		return 0;
-
 	va_start(args, name);
-	argv_array_push(&argv, git_path("hooks/%s", name));
 	while ((p = va_arg(args, const char *)))
 		argv_array_push(&argv, p);
 	va_end(args);
 
 	memset(&hook, 0, sizeof(hook));
-	hook.argv = argv.argv;
-	hook.no_stdin = 1;
-	hook.stdout_to_stderr = 1;
-	if (index_file) {
-		snprintf(index, sizeof(index), "GIT_INDEX_FILE=%s", index_file);
+	hook.name = name;
+	hook.index_file = index_file;
+	hook.argv_array = &argv;
+	ret = run_hook_complex(&hook);
+
+	argv_array_clear(&argv);
+	return ret;
+}
+
+struct feed_hook_with {
+	struct strbuf *buf;
+	struct hook *hook;
+};
+
+/*
+ * An async process is used to feed the hook its stdin.
+ * This allows the hook to read and write at its own pace without blocking.
+ */
+static int feed_hook_async(int in, int out, void *data)
+{
+	struct feed_hook_with *feedwith = data;
+	int ret = feedwith->hook->feeder(out, feedwith->buf, feedwith->hook);
+	close(out);
+	return ret;
+}
+
+/*
+ * Runs a hook, if it exists. Returns non-zero if the hook fails to run
+ * correctly.
+ */
+int run_hook_complex(struct hook *hook)
+{
+	char *command;
+	struct child_process child;
+	struct async async;
+	struct feed_hook_with feedwith = { 0, hook };
+	struct argv_array argv = ARGV_ARRAY_INIT;
+	const char *env[2];
+	char index[PATH_MAX];
+	int res = 0;
+	int i;
+
+	command = git_path("hooks/%s", hook->name);
+	if (access(command, X_OK) < 0)
+		return 0;
+
+	memset(&child, 0, sizeof(child));
+	argv_array_push(&argv, command);
+	if (hook->argv_array)
+		for (i = 0; i < hook->argv_array->argc; i++)
+			argv_array_push(&argv, hook->argv_array->argv[i]);
+	child.argv = argv.argv;
+	if (hook->generator && hook->feeder)
+		child.in = -1;
+	else
+		child.no_stdin = 1;
+	if (hook->reader)
+		child.out = -1;
+	else
+		child.stdout_to_stderr = 1;
+	if (hook->index_file) {
+		snprintf(index, sizeof(index), "GIT_INDEX_FILE=%s",
+				hook->index_file);
 		env[0] = index;
 		env[1] = NULL;
-		hook.env = env;
+		child.env = env;
+	}
+	res |= start_command(&child);
+	if (res) goto hook_out;
+
+	if (hook->generator && hook->feeder) {
+		feedwith.buf = hook->generator(hook);
+		if (! feedwith.buf) {
+			res = 1;
+			goto hook_out;
+		}
+
+		memset(&async, 0, sizeof(async));
+		async.proc = feed_hook_async;
+		async.data = &feedwith;
+		async.out = child.in;
+		res |= start_async(&async);
+		if (res) {
+			close(child.in);
+			close(child.out);
+			finish_command(&child);
+			goto hook_out;
+		}
 	}
 
-	ret = run_command(&hook);
+	if (hook->reader)
+		res |= hook->reader(child.out, hook);
+	if (hook->generator && hook->feeder)
+	       res |= finish_async(&async);
+	res |= finish_command(&child);
+
+ hook_out:
+	if (feedwith.buf)
+		strbuf_release(feedwith.buf);
 	argv_array_clear(&argv);
-	return ret;
+
+	return res;
+}
+
+/* A feeder that ensures the hook consumes all its stdin. */
+int feed_hook_in_full(int handle, struct strbuf *buf, struct hook *hook)
+{
+	if (write_in_full(handle, buf->buf, buf->len) != buf->len) {
+		warning("%s hook failed to consume all its input", hook->name);
+		return 1;
+	}
+	else
+		return 0;
+}
+
+/* A feeder that does not require the hook consume all its stdin. */
+int feed_hook_incomplete(int handle, struct strbuf *buf, struct hook *hook)
+{
+	write_in_full(handle, buf->buf, buf->len);
+	return 0;
 }
diff --git a/run-command.h b/run-command.h
index 56491b9..54c9b83 100644
--- a/run-command.h
+++ b/run-command.h
@@ -45,8 +45,6 @@ int start_command(struct child_process *);
 int finish_command(struct child_process *);
 int run_command(struct child_process *);
 
-extern int run_hook(const char *index_file, const char *name, ...);
-
 #define RUN_COMMAND_NO_STDIN 1
 #define RUN_GIT_CMD	     2	/*If this is to be git sub-command */
 #define RUN_COMMAND_STDOUT_TO_STDERR 4
@@ -90,4 +88,60 @@ struct async {
 int start_async(struct async *async);
 int finish_async(struct async *async);
 
+/*
+ * This data structure controls how a hook is run.
+ */
+struct hook {
+	/* The name of the hook being run. */
+	const char *name;
+	/*
+	 * Parameters to pass to the hook program, not including the name
+	 * of the hook. May be NULL.
+	 */
+	struct argv_array *argv_array;
+	/*
+	 * Pathname to an index file to use, or NULL if the hook
+	 * uses the default index file or no index is needed.
+	 */
+	const char *index_file;
+	/*
+	 * An arbitrary data structure, can be used to communicate between
+	 * the generator, feeder, reader, and the caller of the hook.
+	 */
+	void *data;
+	/*
+	 * Populates a strbuf with the content to send to the
+	 * hook on its standard input.
+	 *
+	 * May be NULL, if the hook does not consume standard input.
+	 */
+	struct strbuf *(*generator)(struct hook *hook);
+	/*
+	 * Feeds generated content to the hook on its standard input
+	 * via the handle. Returns non-zero on failure.
+	 *
+	 * May be NULL, if the hook does not consume standard input.
+	 *
+	 * Note that the feeder is run as an async process, and so should
+	 * avoid modifying any global state, and only use thread-safe
+	 * operations. It may be run more than once.
+	 */
+	int (*feeder)(int handle, struct strbuf *data, struct hook *hook);
+	/*
+	 * Processes the hook's standard output from the handle,
+	 * returning non-zero on failure.
+	 *
+	 * May be NULL, if the hook's stdin is not processed. (It will
+	 * instead be redirected to stderr.)
+	 */
+	int (*reader)(int handle, struct hook *hook);
+};
+
+extern int run_hook(const char *index_file, const char *name, ...);
+
+extern int run_hook_complex(struct hook *hook);
+
+extern int feed_hook_in_full(int handle, struct strbuf *buf, struct hook *hook);
+extern int feed_hook_incomplete(int handle, struct strbuf *buf, struct hook *hook);
+
 #endif
-- 
1.7.7.3

Re: [PATCH 1/3] expanded hook api with stdio support

[Johannes Sixt]

Am 30.12.2011 02:07, schrieb Joey Hess:
> Finally, the API is designed to be extended in the future, to support
> running multiple programs for a single hook action (such as the contents
> of a .git/hooks/hook.d/ , or a system-wide hook). This design goal led
> to the "generator" and "reader" members of the struct hook, which are
> specified such that they can be called repeatedly, with data flowing
> between them (via the "data" member), like this:
>     generator | hook_prog_1 | reader | generator | hook_prog_2 | reader

IMHO, this is overengineered. I don't think that we need something like
this in the foreseeable future, particularly because such a pipeline or
multi-hook infrastructure can easily be constructed by the (single) hook
script itself.

IOW, none of the three function pointers should be needed (not even the
feeder, see below), at least not in a first step.

IMO, as the first step, the user of this infrastructure should only be
required to construct the hook input as a strbuf, and receive the hook
output, if needed, also as a strbuf.

> +`run_hook_complex`::
> +
> +	Run a hook, with the caller providing its stdin and/or parsing its
> +	stdout.
> +	Takes a pointer to a `struct hook` that specifies the details,
> +	including the name of the hook, any parameters to pass to it,
> +	and how to handle the stdin and stdout. (See below.)
> +	If the hook does not exist or is not executable, the return value
> +	will be zero.
> +	If it is executable, the hook will be executed and the exit
> +	status of the hook is returned.

What is the rationale for these error modes? It is as if a non-existent
or non-executable hook counts as 'success'. (I'm not saying that this
would be wrong, I'm just asking.)

>  
>  Data structures
>  ---------------
> @@ -241,3 +252,45 @@ a forked process otherwise:
>  
>  . It must not change the program's state that the caller of the
>    facility also uses.
> +
> +* `struct hook`
> +
> +This describes a hook to run.
> +
> +The caller:
> +
> +1. allocates and clears (memset(&hook, 0, sizeof(hook));) a
> +   struct hook variable;
> +2. initializes the members;
> +3. calls hook();

run_hook_complex()?

> +4. if necessary, accesses data read from the hook in .data
> +5. frees the struct hook.

> +The `feeder` is run asynchronously to feed the generated stdin into the hook.
> +It is passed the handle to write to, the strbuf containing the stdin, and 
> +a pointer to the `struct hook`, and should return non-zero on failure.
> +Typically it is set to either `feed_hook_in_full`, or `feed_hook_incomplete`.

IMO, this is overengineered. See below.

> +	res |= start_command(&child);
> +	if (res) goto hook_out;

Please write this conditional in two lines.

> +/* A feeder that ensures the hook consumes all its stdin. */
> +int feed_hook_in_full(int handle, struct strbuf *buf, struct hook *hook)
> +{
> +	if (write_in_full(handle, buf->buf, buf->len) != buf->len) {
> +		warning("%s hook failed to consume all its input", hook->name);

Really? Could there not be any other error condition? The warning would
be correct only if errno == EPIPE, and this error will be returned only
if SIGPIPE is ignored. Does this happen anywhere?

Futhermore, if all data was written to the pipe successfully, there is
no way to know whether the reader consumed everything.

IOW, I don't it is worth to make the distinction. However, I do think
that the parent process must be protected against SIGPIPE.

> +		return 1;
> +	}
> +	else
> +		return 0;
> +}

-- Hannes

Re: [PATCH 1/3] expanded hook api with stdio support

[Joey Hess]

[-- Attachment #1: Type: text/plain, Size: 3162 bytes --]

Johannes Sixt wrote:
> IMHO, this is overengineered. I don't think that we need something like
> this in the foreseeable future, particularly because such a pipeline or
> multi-hook infrastructure can easily be constructed by the (single) hook
> script itself.

Junio seemed to think this was a good direction to move in and gave some
examples in <7vlipz930t.fsf@alter.siamese.dyndns.org>

Anyway, the minimum cases for run_hook_complex() to support are:

* no stdin, no stdout
* only stdin
* stdin and stdout (needed for tweak-fetch)
* only stdout (perhaps)

The generator and reader members of struct hook allow the caller to
easily specify which of these cases applies to a hook, and also provides
a natural separation of the caller's stdin generation and stdout parsing
code.

That leaves the feeder and data members of struct hook as possible
overengineering. See below regarding the feeder. The data member could
be eliminated and global variables used by callers that need that,
but I prefer designs that don't require global variables.

> > +`run_hook_complex`::
> > +
> > +	Run a hook, with the caller providing its stdin and/or parsing its
> > +	stdout.
> > +	Takes a pointer to a `struct hook` that specifies the details,
> > +	including the name of the hook, any parameters to pass to it,
> > +	and how to handle the stdin and stdout. (See below.)
> > +	If the hook does not exist or is not executable, the return value
> > +	will be zero.
> > +	If it is executable, the hook will be executed and the exit
> > +	status of the hook is returned.
> 
> What is the rationale for these error modes? It is as if a non-existent
> or non-executable hook counts as 'success'. (I'm not saying that this
> would be wrong, I'm just asking.)

They are identical to how run_hook already works.
A non-existant/non-executable hook *is* a valid configuration,
indeed it's the most likely configuration.

> > +/* A feeder that ensures the hook consumes all its stdin. */
> > +int feed_hook_in_full(int handle, struct strbuf *buf, struct hook *hook)
> > +{
> > +	if (write_in_full(handle, buf->buf, buf->len) != buf->len) {
> > +		warning("%s hook failed to consume all its input", hook->name);
> 
> Really? Could there not be any other error condition? The warning would
> be correct only if errno == EPIPE, and this error will be returned only
> if SIGPIPE is ignored. Does this happen anywhere?
> 
> Futhermore, if all data was written to the pipe successfully, there is
> no way to know whether the reader consumed everything.
> 
> IOW, I don't it is worth to make the distinction. However, I do think
> that the parent process must be protected against SIGPIPE.

Yes, this was not thought through, I missed that a write to a pipe can
succeed (due to buffering) despite not being fully consumed.

Dealing with the hook SIGPIPE issue is complicated as Jeff explains in
<20111205214351.GA15029@sigill.intra.peff.net>, and I don't feel I'm the
one to do it. I'm feeling like ripping the "feeder" stuff out of my
patch, and not having my patch change the status quo on this issue.

-- 
see shy jo

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 828 bytes --]

Re: [PATCH 1/3] expanded hook api with stdio support

[Johannes Sixt]

Am 30.12.2011 18:13, schrieb Joey Hess:
> Johannes Sixt wrote:
>> IMHO, this is overengineered. I don't think that we need something like
>> this in the foreseeable future, particularly because such a pipeline or
>> multi-hook infrastructure can easily be constructed by the (single) hook
>> script itself.
> 
> Junio seemed to think this was a good direction to move in and gave some
> examples in <7vlipz930t.fsf@alter.siamese.dyndns.org>
> 
> Anyway, the minimum cases for run_hook_complex() to support are:
> 
> * no stdin, no stdout
> * only stdin
> * stdin and stdout (needed for tweak-fetch)
> * only stdout (perhaps)
> 
> The generator and reader members of struct hook allow the caller to
> easily specify which of these cases applies to a hook, and also provides
> a natural separation of the caller's stdin generation and stdout parsing
> code.

But as long as the generator only needs to generate a strbuf *and* only
one hook is run, there is no value to have it as a callback; the caller
can just specify the strbuf itself, run_hook_* does not need to care how
it was generated.

I can see some value in a reader callback to avoid allocating yet
another strbuf.

> ... The data member could
> be eliminated and global variables used by callers that need that,
> but I prefer designs that don't require global variables.

Absolutely.

>>> +	If the hook does not exist or is not executable, the return value
>>> +	will be zero.
>>> +	If it is executable, the hook will be executed and the exit
>>> +	status of the hook is returned.
>>
>> What is the rationale for these error modes? It is as if a non-existent
>> or non-executable hook counts as 'success'. (I'm not saying that this
>> would be wrong, I'm just asking.)
> 
> They are identical to how run_hook already works.
> A non-existant/non-executable hook *is* a valid configuration,
> indeed it's the most likely configuration.

So, it is so that the caller does not itself have to check whether a
hook exists. That may be worth a word in the API documentation.

-- Hannes

Re: [PATCH 1/3] expanded hook api with stdio support

[Junio C Hamano]

Johannes Sixt <j6t@kdbg.org> writes:

> IMO, as the first step, the user of this infrastructure should only be
> required to construct the hook input as a strbuf, and receive the hook
> output, if needed, also as a strbuf.

Now you brought it up, I think I would agree. The only reason I suggested
a callback feeder approach was because I somehow was hoping that it may be
possible to share more code with the codepath for textconv that may not
want to hold too much buffer in core when we know the data is only used
sequencially and I wanted to see more things to go through streaming API
in the future.

>> +`run_hook_complex`::

Also, I think the updated interface should become the "run_hook" function;
nothing "complex" about it. The name "run_hook()" was a perfectly fine
abstraction for what it did when it used to be a static helper function
within builtin-commit.c, but its special-casing of GIT_INDEX_FILE
environment is _not_ general enough to deserve it to be called the
"run_hook" in the global scope.

IOW, I am saying that we screwed up at ae98a00 (Move run_hook() from
builtin-commit.c into run-command.c (libgit), 2009-01-16.

The environment tweaking should not take a "index_file" field in the
structure, but an array "environ" that is used to tweak the environment
variables for the hook process.

Re: [PATCH 1/3] expanded hook api with stdio support

[Jeff King]

On Tue, Jan 03, 2012 at 11:53:22AM -0800, Junio C Hamano wrote:

> Johannes Sixt <j6t@kdbg.org> writes:
> 
> > IMO, as the first step, the user of this infrastructure should only be
> > required to construct the hook input as a strbuf, and receive the hook
> > output, if needed, also as a strbuf.
> 
> Now you brought it up, I think I would agree. The only reason I suggested
> a callback feeder approach was because I somehow was hoping that it may be
> possible to share more code with the codepath for textconv that may not
> want to hold too much buffer in core when we know the data is only used
> sequencially and I wanted to see more things to go through streaming API
> in the future.

Even if we don't make the input streaming, it would be nice to factor
the concept of "feed input to program and read its output without
deadlocking" into something independent of hooks.

The credential helper code could potentially have the same deadlock.
Possibly also clean/smudge filters.

Maybe it could even be part of the run-command interface?

-Peff

Re: [PATCH 1/3] expanded hook api with stdio support

[Junio C Hamano]

Jeff King <peff@peff.net> writes:

> The credential helper code could potentially have the same deadlock.
> Possibly also clean/smudge filters.
>
> Maybe it could even be part of the run-command interface?

Hmm, that smells like the right thing to do.

[PATCH 2/3] preparations for tweak-fetch hook

[Joey Hess]

No behavior changes yet, only some groundwork for the next
change.

The refs_result structure combines a status code with a ref map,
which can be NULL even on success. This will be needed when
there's a tweak-fetch hook, because it can filter out all refs,
while still succeeding.

fetch_refs returns a refs_result, so that it can modify the ref_map.

Signed-off-by: Joey Hess <joey@kitenet.net>
---
 builtin/fetch.c |   55 ++++++++++++++++++++++++++++++++++++-------------------
 1 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index 33ad3aa..a48358a 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -29,6 +29,11 @@ enum {
 	TAGS_SET = 2
 };
 
+struct refs_result {
+	struct ref *new_refs;
+	int status;
+};
+
 static int all, append, dry_run, force, keep, multiple, prune, update_head_ok, verbosity;
 static int progress, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
 static int tags = TAGS_DEFAULT;
@@ -89,6 +94,15 @@ static struct option builtin_fetch_options[] = {
 	OPT_END()
 };
 
+static int add_existing(const char *refname, const unsigned char *sha1,
+			int flag, void *cbdata)
+{
+	struct string_list *list = (struct string_list *)cbdata;
+	struct string_list_item *item = string_list_insert(list, refname);
+	item->util = (void *)sha1;
+	return 0;
+}
+
 static void unlock_pack(void)
 {
 	if (transport)
@@ -507,17 +521,25 @@ static int quickfetch(struct ref *ref_map)
 	return check_everything_connected(iterate_ref_map, 1, &rm);
 }
 
-static int fetch_refs(struct transport *transport, struct ref *ref_map)
+static struct refs_result fetch_refs(struct transport *transport,
+		struct ref *ref_map)
 {
-	int ret = quickfetch(ref_map);
-	if (ret)
-		ret = transport_fetch_refs(transport, ref_map);
-	if (!ret)
-		ret |= store_updated_refs(transport->url,
+	struct refs_result res;
+	res.status = quickfetch(ref_map);
+	if (res.status)
+		res.status = transport_fetch_refs(transport, ref_map);
+	if (!res.status) {
+		res.new_refs = ref_map;
+
+		res.status |= store_updated_refs(transport->url,
 				transport->remote->name,
-				ref_map);
+				res.new_refs);
+	}
+	else {
+		res.new_refs = ref_map;
+	}
 	transport_unlock_pack(transport);
-	return ret;
+	return res;
 }
 
 static int prune_refs(struct refspec *refs, int ref_count, struct ref *ref_map)
@@ -542,15 +564,6 @@ static int prune_refs(struct refspec *refs, int ref_count, struct ref *ref_map)
 	return result;
 }
 
-static int add_existing(const char *refname, const unsigned char *sha1,
-			int flag, void *cbdata)
-{
-	struct string_list *list = (struct string_list *)cbdata;
-	struct string_list_item *item = string_list_insert(list, refname);
-	item->util = (void *)sha1;
-	return 0;
-}
-
 static int will_fetch(struct ref **head, const unsigned char *sha1)
 {
 	struct ref *rm = *head;
@@ -673,6 +686,7 @@ static int do_fetch(struct transport *transport,
 	struct string_list_item *peer_item = NULL;
 	struct ref *ref_map;
 	struct ref *rm;
+	struct refs_result res;
 	int autotags = (transport->remote->fetch_tags == 1);
 
 	for_each_ref(add_existing, &existing_refs);
@@ -710,7 +724,9 @@ static int do_fetch(struct transport *transport,
 
 	if (tags == TAGS_DEFAULT && autotags)
 		transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, "1");
-	if (fetch_refs(transport, ref_map)) {
+	res = fetch_refs(transport, ref_map);
+	ref_map = res.new_refs;
+	if (res.status) {
 		free_refs(ref_map);
 		return 1;
 	}
@@ -750,7 +766,8 @@ static int do_fetch(struct transport *transport,
 		if (ref_map) {
 			transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, NULL);
 			transport_set_option(transport, TRANS_OPT_DEPTH, "0");
-			fetch_refs(transport, ref_map);
+			res = fetch_refs(transport, ref_map);
+			ref_map = res.new_refs;
 		}
 		free_refs(ref_map);
 	}
-- 
1.7.7.3

[PATCH 3/3] add tweak-fetch hook

[Joey Hess]

The tweak-fetch hook is fed lines on stdin for all refs that were fetched,
and outputs on stdout possibly modified lines. Its output is parsed and
used when git fetch updates the remote tracking refs, records the entries
in FETCH_HEAD, and produces its report.

This is implemented using the new run_hook_complex API.

Signed-off-by: Joey Hess <joey@kitenet.net>
---
 Documentation/githooks.txt |   29 +++++++++
 builtin/fetch.c            |  144 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 172 insertions(+), 1 deletions(-)

diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt
index 28edefa..bea450a 100644
--- a/Documentation/githooks.txt
+++ b/Documentation/githooks.txt
@@ -162,6 +162,35 @@ This hook can be used to perform repository validity checks, auto-display
 differences from the previous HEAD if different, or set working dir metadata
 properties.
 
+tweak-fetch
+~~~~~~~~~~~
+
+This hook is invoked by 'git fetch' (commonly called by 'git pull'), after
+refs have been fetched from the remote repository. It is not executed, if
+nothing was fetched.
+
+The output of the hook is used to update the remote-tracking branches, and
+`.git/FETCH_HEAD`, in preparation for for a later merge operation done by
+'git merge'.
+
+It takes no arguments, but is fed a line of the following format on
+its standard input for each ref that was fetched.
+
+  <sha1> SP not-for-merge|merge SP <remote-refname> SP <local-refname> LF
+
+Where the "not-for-merge" flag indicates the ref is not to be merged into the
+current branch, and the "merge" flag indicates that 'git merge' should
+later merge it. The `<remote-refname>` is the remote's name for the ref
+that was fetched, and `<local-refname>` is a name of a remote-tracking branch,
+like "refs/remotes/origin/master", or can be empty if the fetched ref is not
+being stored in a local refname.
+
+The hook must consume all of its standard input, and output back lines
+of the same format. It can modify its input as desired, including
+adding or removing lines, updating the sha1 (i.e. re-point the
+remote-tracking branch), changing the merge flag, and changing the
+`<local-refname>` (i.e. use different remote-tracking branch).
+
 post-merge
 ~~~~~~~~~~
 
diff --git a/builtin/fetch.c b/builtin/fetch.c
index a48358a..80178d0 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -103,6 +103,148 @@ static int add_existing(const char *refname, const unsigned char *sha1,
 	return 0;
 }
 
+struct strbuf *tweak_fetch_hook_generator(struct hook *hook)
+{
+	struct ref *ref;
+	struct strbuf *buf = xmalloc(sizeof(struct strbuf));
+
+	strbuf_init(buf, 128);
+	for (ref = hook->data; ref; ref = ref->next)
+		strbuf_addf(buf, "%s %s %s %s\n",
+			sha1_to_hex(ref->old_sha1),
+			ref->merge ? "merge" : "not-for-merge",
+			ref->name ? ref->name : "",
+			ref->peer_ref && ref->peer_ref->name ?
+				ref->peer_ref->name : "");
+	return buf;
+}
+
+static struct ref *parse_tweak_fetch_hook_line(char *l, 
+		struct string_list *existing_refs, struct hook *hook)
+{
+	struct ref *ref = NULL, *peer_ref = NULL;
+	struct string_list_item *peer_item = NULL;
+	char *p, *words[4];
+	char *problem;
+	int i;
+
+	for (i = 0 ; i <= ARRAY_SIZE(words) - 1; i++) {
+		p = strchr(l, ' ');
+		words[i] = l;
+		if (!p)
+			break;
+		p[0] = '\0';
+		l = p+1;
+	}
+	if (i != ARRAY_SIZE(words) - 1) {
+		problem = "wrong number of words";
+		goto unparsable;
+	}
+
+	ref = alloc_ref(words[2]);
+	peer_ref = ref->peer_ref = alloc_ref(words[3]);
+	ref->peer_ref->force = 1;
+
+	if (get_sha1_hex(words[0], ref->old_sha1)) {
+		problem = "bad sha1";
+		goto unparsable;
+	}
+
+	if (!strcmp(words[1], "merge")) {
+		ref->merge = 1;
+	}
+	else if (strcmp(words[1], "not-for-merge")) {
+		problem = "bad merge flag";
+		goto unparsable;
+	}
+
+	peer_item = string_list_lookup(existing_refs, peer_ref->name);
+	if (peer_item)
+		hashcpy(peer_ref->old_sha1, peer_item->util);
+
+	return ref;
+
+ unparsable:
+	warning("%s hook output a wrongly formed line: %s",
+			hook->name, problem);
+	free(ref);
+	free(peer_ref);
+	return NULL;
+}
+
+int tweak_fetch_hook_reader(int handle, struct hook *hook)
+{
+	FILE *f;
+	struct strbuf buf;
+	struct string_list existing_refs = STRING_LIST_INIT_NODUP;
+	struct ref *old_refs, *ref, *prevref = NULL;
+	int res = 0;
+
+	f = fdopen(handle, "r");
+	if (f == NULL)
+		return 1;
+
+	old_refs = hook->data;
+	hook->data = NULL;
+
+	strbuf_init(&buf, 128);
+	for_each_ref(add_existing, &existing_refs);
+
+	while (strbuf_getline(&buf, f, '\n') != EOF) {
+		char *l = strbuf_detach(&buf, NULL);
+		ref = parse_tweak_fetch_hook_line(l, &existing_refs, hook);
+		if (ref) {
+			if (prevref) {
+				prevref->next = ref;
+				prevref = ref;
+			} else {
+				hook->data = prevref = ref;
+			}
+		} else {
+			res = 1;
+		}
+		free(l);
+	}
+
+	string_list_clear(&existing_refs, 0);
+	strbuf_release(&buf);
+	fclose(f);
+
+	if (res)
+		hook->data = old_refs;
+	else
+		free_refs(old_refs);
+
+	return res;
+}
+
+/*
+ * The tweak-fetch hook is fed lines of the form:
+ * <sha1> SP <not-for-merge|merge> SP <remote-refname> SP <local-refname> LF
+ * And should output rewritten lines of the same form, which are used
+ * to generate a tweaked set of fetched_refs.
+ */
+static struct ref *run_tweak_fetch_hook(struct ref *fetched_refs)
+{
+	struct hook hook;
+
+	if (! fetched_refs)
+		return fetched_refs;
+
+	memset(&hook, 0, sizeof(hook));
+	hook.name = "tweak-fetch";
+	hook.generator = tweak_fetch_hook_generator;
+	hook.feeder = feed_hook_in_full;
+	hook.reader = tweak_fetch_hook_reader;
+	hook.data = fetched_refs;
+
+	if (run_hook_complex(&hook)) {
+		warning("%s hook failed, ignoring its output", hook.name);
+	}
+
+	return hook.data;
+}
+
 static void unlock_pack(void)
 {
 	if (transport)
@@ -529,7 +671,7 @@ static struct refs_result fetch_refs(struct transport *transport,
 	if (res.status)
 		res.status = transport_fetch_refs(transport, ref_map);
 	if (!res.status) {
-		res.new_refs = ref_map;
+		res.new_refs = run_tweak_fetch_hook(ref_map);
 
 		res.status |= store_updated_refs(transport->url,
 				transport->remote->name,
-- 
1.7.7.3