threads / patch / 30243

patchgit-daemon wrapper to wait until daemon is ready

Subject: [PATCH] git-daemon wrapper to wait until daemon is ready

## tl;dr

18 messages between Apr 14, 2012 and Apr 19, 2012. Diffs are folded; open one to read it.

replies: 17people: 5as markdown or json

Clemens Buchacher· Apr 14, 2012, 18:29 UTC · lore

The shell script which is currently used to parse git daemon output does not seem to work unreliably. In order to work around such issues, re-implement the same procedure in C and write the daemon pid to a file.

This means that we can no longer wait on the daemon process, since it is no longer a direct child of the shell process.

Signed-off-by: Clemens Buchacher <drizzd@aon.at>
---
Does this patch improve your situation?

Note that t5570 fails on current pu, because of the push.default warnings. I am sending an independent patch for that.

Clemens
 .gitignore          |    1 +
 Makefile            |    1 +
 cache.h             |    1 +
 t/lib-git-daemon.sh |   30 +++-------------------
 test-git-daemon.c   |   71 +++++++++++++++++++++++++++++++++++++++++++++++++++
 wrapper.c           |   21 +++++++++++++++
 6 files changed, 99 insertions(+), 26 deletions(-)
 create mode 100644 test-git-daemon.c
Show changes to 6 files +99 −26

.gitignore, Makefile, cache.h, t/lib-git-daemon.sh, test-git-daemon.c, wrapper.c

diff --git a/.gitignore b/.gitignore
index 87fcc5f..18a484c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -177,6 +177,7 @@
 /test-dump-cache-tree
 /test-scrap-cache-tree
 /test-genrandom
+/test-git-daemon
 /test-index-version
 /test-line-buffer
 /test-match-trees
diff --git a/Makefile b/Makefile
index be1957a..7317daa 100644
--- a/Makefile
+++ b/Makefile
@@ -477,6 +477,7 @@ TEST_PROGRAMS_NEED_X += test-delta
 TEST_PROGRAMS_NEED_X += test-dump-cache-tree
 TEST_PROGRAMS_NEED_X += test-scrap-cache-tree
 TEST_PROGRAMS_NEED_X += test-genrandom
+TEST_PROGRAMS_NEED_X += test-git-daemon
 TEST_PROGRAMS_NEED_X += test-index-version
 TEST_PROGRAMS_NEED_X += test-line-buffer
 TEST_PROGRAMS_NEED_X += test-match-trees
diff --git a/cache.h b/cache.h
index e5e1aa4..6351c15 100644
--- a/cache.h
+++ b/cache.h
@@ -1176,6 +1176,7 @@ extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg
 extern int write_or_whine_pipe(int fd, const void *buf, size_t count, const char *msg);
 extern void fsync_or_die(int fd, const char *);
 
+extern ssize_t read_line(int fd, void *buf, size_t count);
 extern ssize_t read_in_full(int fd, void *buf, size_t count);
 extern ssize_t write_in_full(int fd, const void *buf, size_t count);
 static inline ssize_t write_str_in_full(int fd, const char *str)
diff --git a/t/lib-git-daemon.sh b/t/lib-git-daemon.sh
index ef2d01f..9fefae1 100644
--- a/t/lib-git-daemon.sh
+++ b/t/lib-git-daemon.sh
@@ -23,27 +23,13 @@ start_git_daemon() {
 	trap 'code=$?; stop_git_daemon; (exit $code); die' EXIT
 
 	say >&3 "Starting git daemon ..."
-	mkfifo git_daemon_output
-	git daemon --listen=127.0.0.1 --port="$LIB_GIT_DAEMON_PORT" \
+	test-git-daemon --listen=127.0.0.1 --port="$LIB_GIT_DAEMON_PORT" \
 		--reuseaddr --verbose \
 		--base-path="$GIT_DAEMON_DOCUMENT_ROOT_PATH" \
 		"$@" "$GIT_DAEMON_DOCUMENT_ROOT_PATH" \
-		>&3 2>git_daemon_output &
-	GIT_DAEMON_PID=$!
-	{
-		read line
-		echo >&4 "$line"
-		cat >&4 &
-
-		# Check expected output
-		if test x"$(expr "$line" : "\[[0-9]*\] \(.*\)")" != x"Ready to rumble"
-		then
-			kill "$GIT_DAEMON_PID"
-			wait "$GIT_DAEMON_PID"
-			trap 'die' EXIT
-			error "git daemon failed to start"
-		fi
-	} <git_daemon_output
+		>&3 2>&4 ||
+		error "git daemon failed to start"
+	GIT_DAEMON_PID=$(cat git-daemon.pid)
 }
 
 stop_git_daemon() {
@@ -57,13 +43,5 @@ stop_git_daemon() {
 	# kill git-daemon child of git
 	say >&3 "Stopping git daemon ..."
 	kill "$GIT_DAEMON_PID"
-	wait "$GIT_DAEMON_PID" >&3 2>&4
-	ret=$?
-	# expect exit with status 143 = 128+15 for signal TERM=15
-	if test $ret -ne 143
-	then
-		error "git daemon exited with status: $ret"
-	fi
 	GIT_DAEMON_PID=
-	rm -f git_daemon_output
 }
diff --git a/test-git-daemon.c b/test-git-daemon.c
new file mode 100644
index 0000000..323bb65
--- /dev/null
+++ b/test-git-daemon.c
@@ -0,0 +1,71 @@
+#include "git-compat-util.h"
+#include "run-command.h"
+#include "exec_cmd.h"
+#include "strbuf.h"
+#include "cache.h"
+#include <string.h>
+#include <errno.h>
+
+static int parse_daemon_output(char *s)
+{
+	if (*s++ != '[')
+		return 1;
+	s = strchr(s, ']');
+	if (!s)
+		return 1;
+	if (strcmp(s, "] Ready to rumble\n"))
+		return 1;
+
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	FILE *fp;
+	struct child_process proc, cat;
+	char *cat_argv[] = { "cat", NULL };
+	int parse_error;
+	char buf[PATH_MAX];
+	int r;
+
+	setup_path();
+
+	memset(&proc, 0, sizeof(proc));
+	argv[0] = "git-daemon";
+	proc.argv = (const char **)argv;
+	proc.no_stdin = 1;
+	proc.err = -1;
+
+	if (start_command(&proc) < 0)
+		return 1;
+
+	r = read_line(proc.err, buf, sizeof(buf));
+	if (r < 0) {
+		finish_command(&proc);
+		return 1;
+	}
+	fprintf(stderr, "%s", buf);
+
+	parse_error = parse_daemon_output(buf);
+
+	memset(&cat, 0, sizeof(cat));
+	cat.argv = (const char **)cat_argv;
+	cat.in = proc.err;
+	cat.out = 2;
+
+	if (start_command(&cat) < 0)
+		return 1;
+
+	if (parse_error) {
+		kill(proc.pid, SIGTERM);
+		finish_command(&proc);
+		finish_command(&cat);
+		return 1;
+	}
+
+	fp = fopen("git-daemon.pid", "w");
+	fprintf(fp, "%"PRIuMAX"\n", (uintmax_t)proc.pid);
+	fclose(fp);
+
+	return 0;
+}
diff --git a/wrapper.c b/wrapper.c
index 85f09df..7bf6dda 100644
--- a/wrapper.c
+++ b/wrapper.c
@@ -141,6 +141,27 @@ ssize_t xwrite(int fd, const void *buf, size_t len)
 	}
 }
 
+ssize_t read_line(int fd, void *buf, size_t count)
+{
+	char *p = buf;
+	ssize_t total = 0;
+
+	while (count > 0) {
+		ssize_t loaded = xread(fd, p, 1);
+		if (loaded < 0)
+			return -1;
+		if (loaded == 0)
+			return total;
+		count -= loaded;
+		total += loaded;
+		if (*p == '\n')
+			break;
+		p += loaded;
+	}
+
+	return total;
+}
+
 ssize_t read_in_full(int fd, void *buf, size_t count)
 {
 	char *p = buf;
-- 
1.7.9.6
Clemens Buchacher· Apr 14, 2012, 18:32 UTC · re: Clemens Buchacher · lore

[PATCH] t5570: use explicit push refspec

The default mode for push without arguments will change. Some warnings are about to be enabled for such use, which causes some t5570 tests to fail because they do not expect this output. Fix this by passing an explicit refspec to git push.

Signed-off-by: Clemens Buchacher <drizzd@aon.at>
---
On Sat, Apr 14, 2012 at 08:29:07PM +0200, Clemens Buchacher wrote:
> 
> Note that t5570 fails on current pu, because of the push.default
> warnings. I am sending an independent patch for that.
Here we go.
 t/t5570-git-daemon.sh |   30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)
Show changes to t/t5570-git-daemon.sh +14 −16
diff --git a/t/t5570-git-daemon.sh b/t/t5570-git-daemon.sh
index 7cbc999..a3a4e47 100755
--- a/t/t5570-git-daemon.sh
+++ b/t/t5570-git-daemon.sh
@@ -103,14 +103,12 @@ test_remote_error()
 		esac
 	done
 
-	if test $# -ne 3
-	then
-		error "invalid number of arguments"
-	fi
-
+	msg=$1
+	shift
 	cmd=$1
-	repo=$2
-	msg=$3
+	shift
+	repo=$1
+	shift || error "invalid number of arguments"
 
 	if test -x "$GIT_DAEMON_DOCUMENT_ROOT_PATH/$repo"
 	then
@@ -122,7 +120,7 @@ test_remote_error()
 		fi
 	fi
 
-	test_must_fail git "$cmd" "$GIT_DAEMON_URL/$repo" 2>output &&
+	test_must_fail git "$cmd" "$GIT_DAEMON_URL/$repo" "$@" 2>output &&
 	echo "fatal: remote error: $msg: /$repo" >expect &&
 	test_cmp expect output
 	ret=$?
@@ -131,18 +129,18 @@ test_remote_error()
 }
 
 msg="access denied or repository not exported"
-test_expect_success 'clone non-existent' "test_remote_error    clone nowhere.git '$msg'"
-test_expect_success 'push disabled'      "test_remote_error    push  repo.git    '$msg'"
-test_expect_success 'read access denied' "test_remote_error -x fetch repo.git    '$msg'"
-test_expect_success 'not exported'       "test_remote_error -n fetch repo.git    '$msg'"
+test_expect_success 'clone non-existent' "test_remote_error    '$msg' clone nowhere.git    "
+test_expect_success 'push disabled'      "test_remote_error    '$msg' push  repo.git master"
+test_expect_success 'read access denied' "test_remote_error -x '$msg' fetch repo.git       "
+test_expect_success 'not exported'       "test_remote_error -n '$msg' fetch repo.git       "
 
 stop_git_daemon
 start_git_daemon --informative-errors
 
-test_expect_success 'clone non-existent' "test_remote_error    clone nowhere.git 'no such repository'"
-test_expect_success 'push disabled'      "test_remote_error    push  repo.git    'service not enabled'"
-test_expect_success 'read access denied' "test_remote_error -x fetch repo.git    'no such repository'"
-test_expect_success 'not exported'       "test_remote_error -n fetch repo.git    'repository not exported'"
+test_expect_success 'clone non-existent' "test_remote_error    'no such repository'      clone nowhere.git    "
+test_expect_success 'push disabled'      "test_remote_error    'service not enabled'     push  repo.git master"
+test_expect_success 'read access denied' "test_remote_error -x 'no such repository'      fetch repo.git       "
+test_expect_success 'not exported'       "test_remote_error -n 'repository not exported' fetch repo.git       "
 
 stop_git_daemon
 test_done
-- 
1.7.9.6
Junio C Hamano· Apr 14, 2012, 23:40 UTC · re: Clemens Buchacher · lore

Re: [PATCH] t5570: use explicit push refspec

Clemens Buchacher <drizzd@aon.at> writes:
> The default mode for push without arguments will change. Some warnings
> are about to be enabled for such use, which causes some t5570 tests to
> fail because they do not expect this output. Fix this by passing an
> explicit refspec to git push.

I wonder if a better fix is to configure "push.default = matching" in the test repository. Otherwise wouldn't the result of the push change once the default changes?

Show 73 quoted lines
> Signed-off-by: Clemens Buchacher <drizzd@aon.at>
> ---
>
> On Sat, Apr 14, 2012 at 08:29:07PM +0200, Clemens Buchacher wrote:
>> 
>> Note that t5570 fails on current pu, because of the push.default
>> warnings. I am sending an independent patch for that.
>
> Here we go.
>
>  t/t5570-git-daemon.sh |   30 ++++++++++++++----------------
>  1 file changed, 14 insertions(+), 16 deletions(-)
>
> diff --git a/t/t5570-git-daemon.sh b/t/t5570-git-daemon.sh
> index 7cbc999..a3a4e47 100755
> --- a/t/t5570-git-daemon.sh
> +++ b/t/t5570-git-daemon.sh
> @@ -103,14 +103,12 @@ test_remote_error()
>  		esac
>  	done
>  
> -	if test $# -ne 3
> -	then
> -		error "invalid number of arguments"
> -	fi
> -
> +	msg=$1
> +	shift
>  	cmd=$1
> -	repo=$2
> -	msg=$3
> +	shift
> +	repo=$1
> +	shift || error "invalid number of arguments"
>  
>  	if test -x "$GIT_DAEMON_DOCUMENT_ROOT_PATH/$repo"
>  	then
> @@ -122,7 +120,7 @@ test_remote_error()
>  		fi
>  	fi
>  
> -	test_must_fail git "$cmd" "$GIT_DAEMON_URL/$repo" 2>output &&
> +	test_must_fail git "$cmd" "$GIT_DAEMON_URL/$repo" "$@" 2>output &&
>  	echo "fatal: remote error: $msg: /$repo" >expect &&
>  	test_cmp expect output
>  	ret=$?
> @@ -131,18 +129,18 @@ test_remote_error()
>  }
>  
>  msg="access denied or repository not exported"
> -test_expect_success 'clone non-existent' "test_remote_error    clone nowhere.git '$msg'"
> -test_expect_success 'push disabled'      "test_remote_error    push  repo.git    '$msg'"
> -test_expect_success 'read access denied' "test_remote_error -x fetch repo.git    '$msg'"
> -test_expect_success 'not exported'       "test_remote_error -n fetch repo.git    '$msg'"
> +test_expect_success 'clone non-existent' "test_remote_error    '$msg' clone nowhere.git    "
> +test_expect_success 'push disabled'      "test_remote_error    '$msg' push  repo.git master"
> +test_expect_success 'read access denied' "test_remote_error -x '$msg' fetch repo.git       "
> +test_expect_success 'not exported'       "test_remote_error -n '$msg' fetch repo.git       "
>  
>  stop_git_daemon
>  start_git_daemon --informative-errors
>  
> -test_expect_success 'clone non-existent' "test_remote_error    clone nowhere.git 'no such repository'"
> -test_expect_success 'push disabled'      "test_remote_error    push  repo.git    'service not enabled'"
> -test_expect_success 'read access denied' "test_remote_error -x fetch repo.git    'no such repository'"
> -test_expect_success 'not exported'       "test_remote_error -n fetch repo.git    'repository not exported'"
> +test_expect_success 'clone non-existent' "test_remote_error    'no such repository'      clone nowhere.git    "
> +test_expect_success 'push disabled'      "test_remote_error    'service not enabled'     push  repo.git master"
> +test_expect_success 'read access denied' "test_remote_error -x 'no such repository'      fetch repo.git       "
> +test_expect_success 'not exported'       "test_remote_error -n 'repository not exported' fetch repo.git       "
>  
>  stop_git_daemon
>  test_done
Clemens Buchacher· Apr 15, 2012, 00:11 UTC · re: Junio C Hamano · lore

Re: [PATCH] t5570: use explicit push refspec

On Sat, Apr 14, 2012 at 04:40:01PM -0700, Junio C Hamano wrote:
Show 10 quoted lines
> Clemens Buchacher <drizzd@aon.at> writes:
> 
> > The default mode for push without arguments will change. Some warnings
> > are about to be enabled for such use, which causes some t5570 tests to
> > fail because they do not expect this output. Fix this by passing an
> > explicit refspec to git push.
> 
> I wonder if a better fix is to configure "push.default = matching" in the
> test repository.  Otherwise wouldn't the result of the push change once
> the default changes?

The push.default option matters only if a refspec is not specified. By adding a refspec, push.default should not matter any more. Unless that is going to change as well?

Junio C Hamano· Apr 15, 2012, 19:20 UTC · re: Clemens Buchacher · lore

Re: [PATCH] t5570: use explicit push refspec

Clemens Buchacher <drizzd@aon.at> writes:
Show 15 quoted lines
> On Sat, Apr 14, 2012 at 04:40:01PM -0700, Junio C Hamano wrote:
>> Clemens Buchacher <drizzd@aon.at> writes:
>> 
>> > The default mode for push without arguments will change. Some warnings
>> > are about to be enabled for such use, which causes some t5570 tests to
>> > fail because they do not expect this output. Fix this by passing an
>> > explicit refspec to git push.
>> 
>> I wonder if a better fix is to configure "push.default = matching" in the
>> test repository.  Otherwise wouldn't the result of the push change once
>> the default changes?
>
> The push.default option matters only if a refspec is not specified. By
> adding a refspec, push.default should not matter any more. Unless that
> is going to change as well?

No, I was thinking more about testing cases where there is no refspec on the command line, which we used to test, but with your patch we no longer do. In other words, your fix not just squelches the advice message and make them pass, but it changes the way the command behaves, no?

Besides, that way you do not have to swap the parameters to test_remote_error so we do not have scratch our heads wondering why we have changes to test vectors that run clones and fetches.

Clemens Buchacher· Apr 15, 2012, 19:52 UTC · re: Junio C Hamano · lore

Re: [PATCH] t5570: use explicit push refspec

On Sun, Apr 15, 2012 at 12:20:06PM -0700, Junio C Hamano wrote:
Show 22 quoted lines
> Clemens Buchacher <drizzd@aon.at> writes:
> 
> > On Sat, Apr 14, 2012 at 04:40:01PM -0700, Junio C Hamano wrote:
> >> Clemens Buchacher <drizzd@aon.at> writes:
> >> 
> >> > The default mode for push without arguments will change. Some warnings
> >> > are about to be enabled for such use, which causes some t5570 tests to
> >> > fail because they do not expect this output. Fix this by passing an
> >> > explicit refspec to git push.
> >> 
> >> I wonder if a better fix is to configure "push.default = matching" in the
> >> test repository.  Otherwise wouldn't the result of the push change once
> >> the default changes?
> >
> > The push.default option matters only if a refspec is not specified. By
> > adding a refspec, push.default should not matter any more. Unless that
> > is going to change as well?
> 
> No, I was thinking more about testing cases where there is no refspec on
> the command line, which we used to test, but with your patch we no longer
> do.  In other words, your fix not just squelches the advice message and
> make them pass, but it changes the way the command behaves, no?

It does exactly the same thing it did before, since there is only one local branch. The goal of the test is not to check behavior of git push without arguments. Since that is subject to change, and is causing causing the test to fail already for no good reason, I think it better to use the more explicit version of the command.

> Besides, that way you do not have to swap the parameters to test_remote_error
> so we do not have scratch our heads wondering why we have changes to test
> vectors that run clones and fetches.

In my opinion, this change is also an improvement in itself, since now we can more easily pass extra arguments to test_remote_error. Maybe the scratching of heads can be alleviated by amending the commit message like so?

-->o--
Subject: [PATCH] t5570: use explicit push refspec

The default mode for push without arguments will change. Some warnings are about to be enabled for such use, which causes some t5570 tests to fail because they do not expect this output.

Fix this by passing an explicit refspec to git push. To that end, change the calling conventions of test_remote_error in order to accomodate extra command arguments.

Signed-off-by: Clemens Buchacher <drizzd@aon.at>
---
 t/t5570-git-daemon.sh |   30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)
Show changes to t/t5570-git-daemon.sh +14 −16
diff --git a/t/t5570-git-daemon.sh b/t/t5570-git-daemon.sh
index 7cbc999..a3a4e47 100755
--- a/t/t5570-git-daemon.sh
+++ b/t/t5570-git-daemon.sh
@@ -103,14 +103,12 @@ test_remote_error()
 		esac
 	done
 
-	if test $# -ne 3
-	then
-		error "invalid number of arguments"
-	fi
-
+	msg=$1
+	shift
 	cmd=$1
-	repo=$2
-	msg=$3
+	shift
+	repo=$1
+	shift || error "invalid number of arguments"
 
 	if test -x "$GIT_DAEMON_DOCUMENT_ROOT_PATH/$repo"
 	then
@@ -122,7 +120,7 @@ test_remote_error()
 		fi
 	fi
 
-	test_must_fail git "$cmd" "$GIT_DAEMON_URL/$repo" 2>output &&
+	test_must_fail git "$cmd" "$GIT_DAEMON_URL/$repo" "$@" 2>output &&
 	echo "fatal: remote error: $msg: /$repo" >expect &&
 	test_cmp expect output
 	ret=$?
@@ -131,18 +129,18 @@ test_remote_error()
 }
 
 msg="access denied or repository not exported"
-test_expect_success 'clone non-existent' "test_remote_error    clone nowhere.git '$msg'"
-test_expect_success 'push disabled'      "test_remote_error    push  repo.git    '$msg'"
-test_expect_success 'read access denied' "test_remote_error -x fetch repo.git    '$msg'"
-test_expect_success 'not exported'       "test_remote_error -n fetch repo.git    '$msg'"
+test_expect_success 'clone non-existent' "test_remote_error    '$msg' clone nowhere.git    "
+test_expect_success 'push disabled'      "test_remote_error    '$msg' push  repo.git master"
+test_expect_success 'read access denied' "test_remote_error -x '$msg' fetch repo.git       "
+test_expect_success 'not exported'       "test_remote_error -n '$msg' fetch repo.git       "
 
 stop_git_daemon
 start_git_daemon --informative-errors
 
-test_expect_success 'clone non-existent' "test_remote_error    clone nowhere.git 'no such repository'"
-test_expect_success 'push disabled'      "test_remote_error    push  repo.git    'service not enabled'"
-test_expect_success 'read access denied' "test_remote_error -x fetch repo.git    'no such repository'"
-test_expect_success 'not exported'       "test_remote_error -n fetch repo.git    'repository not exported'"
+test_expect_success 'clone non-existent' "test_remote_error    'no such repository'      clone nowhere.git    "
+test_expect_success 'push disabled'      "test_remote_error    'service not enabled'     push  repo.git master"
+test_expect_success 'read access denied' "test_remote_error -x 'no such repository'      fetch repo.git       "
+test_expect_success 'not exported'       "test_remote_error -n 'repository not exported' fetch repo.git       "
 
 stop_git_daemon
 test_done
-- 
1.7.9.6
Junio C Hamano· Apr 15, 2012, 20:18 UTC · re: Clemens Buchacher · lore

Re: [PATCH] t5570: use explicit push refspec

Clemens Buchacher <drizzd@aon.at> writes:
Show 17 quoted lines
> In my opinion, this change is also an improvement in itself, since now
> we can more easily pass extra arguments to test_remote_error. Maybe the
> scratching of heads can be alleviated by amending the commit message
> like so?
>
> -->o--
> Subject: [PATCH] t5570: use explicit push refspec
>
> The default mode for push without arguments will change. Some warnings
> are about to be enabled for such use, which causes some t5570 tests to
> fail because they do not expect this output.
>
> Fix this by passing an explicit refspec to git push. To that end, change
> the calling conventions of test_remote_error in order to accomodate
> extra command arguments.
>
> Signed-off-by: Clemens Buchacher <drizzd@aon.at>
Sounds good.  Thanks.
Ben Walton· Apr 14, 2012, 18:43 UTC · re: Clemens Buchacher · lore

Re: [PATCH] git-daemon wrapper to wait until daemon is ready

Excerpts from Clemens Buchacher's message of Sat Apr 14 14:29:07 -0400 2012:
Hi Clemens,
> The shell script which is currently used to parse git daemon output does
> not seem to work unreliably. In order to work around such issues,
Presumably you mean "work reliably" here?

Thanks -Ben -- Ben Walton Systems Programmer - CHASS University of Toronto C:416.407.5610 | W:416.978.4302

Clemens Buchacher· Apr 14, 2012, 19:00 UTC · re: Ben Walton · lore

Re: [PATCH] git-daemon wrapper to wait until daemon is ready

On Sat, Apr 14, 2012 at 02:43:41PM -0400, Ben Walton wrote:
Show 5 quoted lines
> 
> > The shell script which is currently used to parse git daemon output does
> > not seem to work unreliably. In order to work around such issues,
> 
> Presumably you mean "work reliably" here?
Yes, that's what I meant the commit message to say. Thanks for noticing.

But I would like to wait for more info from the OP before we conclude that this is indeed the case.

Clemens Buchacher· Apr 14, 2012, 19:16 UTC · re: Clemens Buchacher · lore

[PATCH v2] git-daemon wrapper to wait until daemon is ready

The shell script which is currently used to parse git daemon output does not seem to work reliably. In order to work around such issues, re-implement the same procedure in C and write the daemon pid to a file.

This means that we can no longer wait on the daemon process, since it is not a direct child of the shell process.

Signed-off-by: Clemens Buchacher <drizzd@aon.at>
---
> On Sat, Apr 14, 2012 at 02:43:41PM -0400, Ben Walton wrote:
> > 
> > Presumably you mean "work reliably" here?

I fixed the commit message. I also noticed that the buffer returned by read_line had no zero terminator. And the parse_error variable was unnecessary.

 .gitignore          |    1 +
 Makefile            |    1 +
 cache.h             |    1 +
 t/lib-git-daemon.sh |   30 +++-------------------
 test-git-daemon.c   |   69 +++++++++++++++++++++++++++++++++++++++++++++++++++
 wrapper.c           |   21 ++++++++++++++++
 6 files changed, 97 insertions(+), 26 deletions(-)
 create mode 100644 test-git-daemon.c
Show changes to 6 files +97 −26

.gitignore, Makefile, cache.h, t/lib-git-daemon.sh, test-git-daemon.c, wrapper.c

diff --git a/.gitignore b/.gitignore
index 87fcc5f..18a484c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -177,6 +177,7 @@
 /test-dump-cache-tree
 /test-scrap-cache-tree
 /test-genrandom
+/test-git-daemon
 /test-index-version
 /test-line-buffer
 /test-match-trees
diff --git a/Makefile b/Makefile
index be1957a..7317daa 100644
--- a/Makefile
+++ b/Makefile
@@ -477,6 +477,7 @@ TEST_PROGRAMS_NEED_X += test-delta
 TEST_PROGRAMS_NEED_X += test-dump-cache-tree
 TEST_PROGRAMS_NEED_X += test-scrap-cache-tree
 TEST_PROGRAMS_NEED_X += test-genrandom
+TEST_PROGRAMS_NEED_X += test-git-daemon
 TEST_PROGRAMS_NEED_X += test-index-version
 TEST_PROGRAMS_NEED_X += test-line-buffer
 TEST_PROGRAMS_NEED_X += test-match-trees
diff --git a/cache.h b/cache.h
index e5e1aa4..6351c15 100644
--- a/cache.h
+++ b/cache.h
@@ -1176,6 +1176,7 @@ extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg
 extern int write_or_whine_pipe(int fd, const void *buf, size_t count, const char *msg);
 extern void fsync_or_die(int fd, const char *);
 
+extern ssize_t read_line(int fd, void *buf, size_t count);
 extern ssize_t read_in_full(int fd, void *buf, size_t count);
 extern ssize_t write_in_full(int fd, const void *buf, size_t count);
 static inline ssize_t write_str_in_full(int fd, const char *str)
diff --git a/t/lib-git-daemon.sh b/t/lib-git-daemon.sh
index ef2d01f..9fefae1 100644
--- a/t/lib-git-daemon.sh
+++ b/t/lib-git-daemon.sh
@@ -23,27 +23,13 @@ start_git_daemon() {
 	trap 'code=$?; stop_git_daemon; (exit $code); die' EXIT
 
 	say >&3 "Starting git daemon ..."
-	mkfifo git_daemon_output
-	git daemon --listen=127.0.0.1 --port="$LIB_GIT_DAEMON_PORT" \
+	test-git-daemon --listen=127.0.0.1 --port="$LIB_GIT_DAEMON_PORT" \
 		--reuseaddr --verbose \
 		--base-path="$GIT_DAEMON_DOCUMENT_ROOT_PATH" \
 		"$@" "$GIT_DAEMON_DOCUMENT_ROOT_PATH" \
-		>&3 2>git_daemon_output &
-	GIT_DAEMON_PID=$!
-	{
-		read line
-		echo >&4 "$line"
-		cat >&4 &
-
-		# Check expected output
-		if test x"$(expr "$line" : "\[[0-9]*\] \(.*\)")" != x"Ready to rumble"
-		then
-			kill "$GIT_DAEMON_PID"
-			wait "$GIT_DAEMON_PID"
-			trap 'die' EXIT
-			error "git daemon failed to start"
-		fi
-	} <git_daemon_output
+		>&3 2>&4 ||
+		error "git daemon failed to start"
+	GIT_DAEMON_PID=$(cat git-daemon.pid)
 }
 
 stop_git_daemon() {
@@ -57,13 +43,5 @@ stop_git_daemon() {
 	# kill git-daemon child of git
 	say >&3 "Stopping git daemon ..."
 	kill "$GIT_DAEMON_PID"
-	wait "$GIT_DAEMON_PID" >&3 2>&4
-	ret=$?
-	# expect exit with status 143 = 128+15 for signal TERM=15
-	if test $ret -ne 143
-	then
-		error "git daemon exited with status: $ret"
-	fi
 	GIT_DAEMON_PID=
-	rm -f git_daemon_output
 }
diff --git a/test-git-daemon.c b/test-git-daemon.c
new file mode 100644
index 0000000..82a9a8f
--- /dev/null
+++ b/test-git-daemon.c
@@ -0,0 +1,69 @@
+#include "git-compat-util.h"
+#include "run-command.h"
+#include "exec_cmd.h"
+#include "strbuf.h"
+#include "cache.h"
+#include <string.h>
+#include <errno.h>
+
+static int parse_daemon_output(char *s)
+{
+	if (*s++ != '[')
+		return 1;
+	s = strchr(s, ']');
+	if (!s)
+		return 1;
+	if (strcmp(s, "] Ready to rumble\n"))
+		return 1;
+
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	FILE *fp;
+	struct child_process proc, cat;
+	char *cat_argv[] = { "cat", NULL };
+	char buf[PATH_MAX];
+	int r;
+
+	setup_path();
+
+	memset(&proc, 0, sizeof(proc));
+	argv[0] = "git-daemon";
+	proc.argv = (const char **)argv;
+	proc.no_stdin = 1;
+	proc.err = -1;
+
+	if (start_command(&proc) < 0)
+		return 1;
+
+	r = read_line(proc.err, buf, sizeof(buf)-1);
+	if (r < 0) {
+		finish_command(&proc);
+		return 1;
+	}
+	buf[r] = '\0';
+	fprintf(stderr, "%s", buf);
+
+	memset(&cat, 0, sizeof(cat));
+	cat.argv = (const char **)cat_argv;
+	cat.in = proc.err;
+	cat.out = 2;
+
+	if (start_command(&cat) < 0)
+		return 1;
+
+	if (parse_daemon_output(buf)) {
+		kill(proc.pid, SIGTERM);
+		finish_command(&proc);
+		finish_command(&cat);
+		return 1;
+	}
+
+	fp = fopen("git-daemon.pid", "w");
+	fprintf(fp, "%"PRIuMAX"\n", (uintmax_t)proc.pid);
+	fclose(fp);
+
+	return 0;
+}
diff --git a/wrapper.c b/wrapper.c
index 85f09df..7bf6dda 100644
--- a/wrapper.c
+++ b/wrapper.c
@@ -141,6 +141,27 @@ ssize_t xwrite(int fd, const void *buf, size_t len)
 	}
 }
 
+ssize_t read_line(int fd, void *buf, size_t count)
+{
+	char *p = buf;
+	ssize_t total = 0;
+
+	while (count > 0) {
+		ssize_t loaded = xread(fd, p, 1);
+		if (loaded < 0)
+			return -1;
+		if (loaded == 0)
+			return total;
+		count -= loaded;
+		total += loaded;
+		if (*p == '\n')
+			break;
+		p += loaded;
+	}
+
+	return total;
+}
+
 ssize_t read_in_full(int fd, void *buf, size_t count)
 {
 	char *p = buf;
-- 
1.7.9.6
Johannes Sixt· Apr 14, 2012, 19:36 UTC · re: Clemens Buchacher · lore

Re: [PATCH] git-daemon wrapper to wait until daemon is ready

Am 14.04.2012 20:29, schrieb Clemens Buchacher:
> +	r = read_line(proc.err, buf, sizeof(buf));
We have strbuf_getwholeline_fd().
> +	memset(&cat, 0, sizeof(cat));
> +	cat.argv = (const char **)cat_argv;
> +	cat.in = proc.err;
> +	cat.out = 2;
Useless use of cat?
-- Hannes
Clemens Buchacher· Apr 14, 2012, 22:06 UTC · re: Johannes Sixt · lore

Re: [PATCH] git-daemon wrapper to wait until daemon is ready

On Sat, Apr 14, 2012 at 09:36:38PM +0200, Johannes Sixt wrote:
> Am 14.04.2012 20:29, schrieb Clemens Buchacher:
> > +	r = read_line(proc.err, buf, sizeof(buf));
> 
> We have strbuf_getwholeline_fd().
Thanks. Will fix.
Show 6 quoted lines
> > +	memset(&cat, 0, sizeof(cat));
> > +	cat.argv = (const char **)cat_argv;
> > +	cat.in = proc.err;
> > +	cat.out = 2;
> 
> Useless use of cat?

I don't see how I could avoid cat here. I have to create a pipe first so that I can read the first line. And then I have to terminate test-git-daemon in order to start the tests. So I cannot continue reading synchronously.

Clemens Buchacher· Apr 15, 2012, 11:53 UTC · re: Clemens Buchacher · lore

[PATCH v3] git-daemon wrapper to wait until daemon is ready

The shell script which is currently used to parse git daemon output does not seem to work reliably. In order to work around such issues, re-implement the same procedure in C and write the daemon pid to a file.

This means that we can no longer wait on the daemon process, since it is no longer a direct child of the shell process.

Signed-off-by: Clemens Buchacher <drizzd@aon.at>
---
On Sun, Apr 15, 2012 at 12:06:06AM +0200, Clemens Buchacher wrote:
Show 7 quoted lines
> On Sat, Apr 14, 2012 at 09:36:38PM +0200, Johannes Sixt wrote:
> > Am 14.04.2012 20:29, schrieb Clemens Buchacher:
> > > +	r = read_line(proc.err, buf, sizeof(buf));
> > 
> > We have strbuf_getwholeline_fd().
> 
> Thanks. Will fix.
Here's the re-roll for completeness. Still waiting on feedback from the
OP if this actually solves the problem, though.
 
 .gitignore          |    1 +
 Makefile            |    1 +
 t/lib-git-daemon.sh |   30 ++++---------------------
 test-git-daemon.c   |   62 +++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 68 insertions(+), 26 deletions(-)
 create mode 100644 test-git-daemon.c
Show changes to 4 files +68 −26

.gitignore, Makefile, t/lib-git-daemon.sh, test-git-daemon.c

diff --git a/.gitignore b/.gitignore
index 87fcc5f..18a484c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -177,6 +177,7 @@
 /test-dump-cache-tree
 /test-scrap-cache-tree
 /test-genrandom
+/test-git-daemon
 /test-index-version
 /test-line-buffer
 /test-match-trees
diff --git a/Makefile b/Makefile
index be1957a..7317daa 100644
--- a/Makefile
+++ b/Makefile
@@ -477,6 +477,7 @@ TEST_PROGRAMS_NEED_X += test-delta
 TEST_PROGRAMS_NEED_X += test-dump-cache-tree
 TEST_PROGRAMS_NEED_X += test-scrap-cache-tree
 TEST_PROGRAMS_NEED_X += test-genrandom
+TEST_PROGRAMS_NEED_X += test-git-daemon
 TEST_PROGRAMS_NEED_X += test-index-version
 TEST_PROGRAMS_NEED_X += test-line-buffer
 TEST_PROGRAMS_NEED_X += test-match-trees
diff --git a/t/lib-git-daemon.sh b/t/lib-git-daemon.sh
index ef2d01f..9fefae1 100644
--- a/t/lib-git-daemon.sh
+++ b/t/lib-git-daemon.sh
@@ -23,27 +23,13 @@ start_git_daemon() {
 	trap 'code=$?; stop_git_daemon; (exit $code); die' EXIT
 
 	say >&3 "Starting git daemon ..."
-	mkfifo git_daemon_output
-	git daemon --listen=127.0.0.1 --port="$LIB_GIT_DAEMON_PORT" \
+	test-git-daemon --listen=127.0.0.1 --port="$LIB_GIT_DAEMON_PORT" \
 		--reuseaddr --verbose \
 		--base-path="$GIT_DAEMON_DOCUMENT_ROOT_PATH" \
 		"$@" "$GIT_DAEMON_DOCUMENT_ROOT_PATH" \
-		>&3 2>git_daemon_output &
-	GIT_DAEMON_PID=$!
-	{
-		read line
-		echo >&4 "$line"
-		cat >&4 &
-
-		# Check expected output
-		if test x"$(expr "$line" : "\[[0-9]*\] \(.*\)")" != x"Ready to rumble"
-		then
-			kill "$GIT_DAEMON_PID"
-			wait "$GIT_DAEMON_PID"
-			trap 'die' EXIT
-			error "git daemon failed to start"
-		fi
-	} <git_daemon_output
+		>&3 2>&4 ||
+		error "git daemon failed to start"
+	GIT_DAEMON_PID=$(cat git-daemon.pid)
 }
 
 stop_git_daemon() {
@@ -57,13 +43,5 @@ stop_git_daemon() {
 	# kill git-daemon child of git
 	say >&3 "Stopping git daemon ..."
 	kill "$GIT_DAEMON_PID"
-	wait "$GIT_DAEMON_PID" >&3 2>&4
-	ret=$?
-	# expect exit with status 143 = 128+15 for signal TERM=15
-	if test $ret -ne 143
-	then
-		error "git daemon exited with status: $ret"
-	fi
 	GIT_DAEMON_PID=
-	rm -f git_daemon_output
 }
diff --git a/test-git-daemon.c b/test-git-daemon.c
new file mode 100644
index 0000000..f44fa6a
--- /dev/null
+++ b/test-git-daemon.c
@@ -0,0 +1,62 @@
+#include "git-compat-util.h"
+#include "run-command.h"
+#include "exec_cmd.h"
+#include "strbuf.h"
+#include <string.h>
+#include <errno.h>
+
+static int parse_daemon_output(char *s)
+{
+	if (*s++ != '[')
+		return 1;
+	s = strchr(s, ']');
+	if (!s)
+		return 1;
+	if (strcmp(s, "] Ready to rumble\n"))
+		return 1;
+
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	struct strbuf line = STRBUF_INIT;
+	FILE *fp;
+	struct child_process proc, cat;
+	char *cat_argv[] = { "cat", NULL };
+
+	setup_path();
+
+	memset(&proc, 0, sizeof(proc));
+	argv[0] = "git-daemon";
+	proc.argv = (const char **)argv;
+	proc.no_stdin = 1;
+	proc.err = -1;
+
+	if (start_command(&proc) < 0)
+		return 1;
+
+	strbuf_getwholeline_fd(&line, proc.err, '\n');
+	fprintf(stderr, line.buf);
+
+	memset(&cat, 0, sizeof(cat));
+	cat.argv = (const char **)cat_argv;
+	cat.in = proc.err;
+	cat.out = 2;
+
+	if (start_command(&cat) < 0)
+		return 1;
+
+	if (parse_daemon_output(line.buf)) {
+		kill(proc.pid, SIGTERM);
+		finish_command(&proc);
+		finish_command(&cat);
+		return 1;
+	}
+
+	fp = fopen("git-daemon.pid", "w");
+	fprintf(fp, "%"PRIuMAX"\n", (uintmax_t)proc.pid);
+	fclose(fp);
+
+	return 0;
+}
-- 
1.7.9.6
Zbigniew Jędrzejewski-Szmek· Apr 16, 2012, 15:46 UTC · re: Clemens Buchacher · lore

Re: [PATCH v3] git-daemon wrapper to wait until daemon is ready

On 04/15/2012 01:53 PM, Clemens Buchacher wrote:
Show 9 quoted lines
> The shell script which is currently used to parse git daemon output does
> not seem to work reliably. In order to work around such issues,
> re-implement the same procedure in C and write the daemon pid to a file.
>
> This means that we can no longer wait on the daemon process, since it is
> no longer a direct child of the shell process.
>
> Signed-off-by: Clemens Buchacher<drizzd@aon.at>
> ---
> Here's the re-roll for completeness. Still waiting on feedback from the
> OP if this actually solves the problem, though.

I posted a reply in the other thread, but I'm replying here too for completeness: yes, the problem is solved.

Thanks, Zbyszek

Junio C Hamano· Apr 19, 2012, 15:00 UTC · re: Clemens Buchacher · lore

Re: [PATCH v3] git-daemon wrapper to wait until daemon is ready

2012/4/15 Clemens Buchacher <drizzd@aon.at>
Show 7 quoted lines
>
> The shell script which is currently used to parse git daemon output does
> not seem to work reliably. In order to work around such issues,
> re-implement the same procedure in C and write the daemon pid to a file.
> ...
> +       strbuf_getwholeline_fd(&line, proc.err, '\n');
> +       fprintf(stderr, line.buf);
Just a note. I'll update this part with "fputs(line.buf, stderr)".
Johannes Sixt· Apr 15, 2012, 17:11 UTC · re: Clemens Buchacher · lore

Re: [PATCH] git-daemon wrapper to wait until daemon is ready

Am 15.04.2012 00:06, schrieb Clemens Buchacher:
Show 13 quoted lines
> On Sat, Apr 14, 2012 at 09:36:38PM +0200, Johannes Sixt wrote:
>> Am 14.04.2012 20:29, schrieb Clemens Buchacher:
>>> +	memset(&cat, 0, sizeof(cat));
>>> +	cat.argv = (const char **)cat_argv;
>>> +	cat.in = proc.err;
>>> +	cat.out = 2;
>>
>> Useless use of cat?
> 
> I don't see how I could avoid cat here. I have to create a pipe first so
> that I can read the first line. And then I have to terminate
> test-git-daemon in order to start the tests. So I cannot continue
> reading synchronously.
OK, I got it.
But reading the first line in this way needs a few assumptions to be true:
- git-daemon does not write an incomplete line and then waits.
- git-daemon does not write more than one line, because xread() happily
reads everything it can get. Your implementation differs from the old
version because the shell's 'read' is required to read no more than one
line, i.e., to read byte-wise from the pipe until it sees the LF.
-- Hannes
Clemens Buchacher· Apr 15, 2012, 19:32 UTC · re: Johannes Sixt · lore

Re: [PATCH] git-daemon wrapper to wait until daemon is ready

On Sun, Apr 15, 2012 at 07:11:52PM +0200, Johannes Sixt wrote:
> 
> But reading the first line in this way needs a few assumptions to be true:
> 
> - git-daemon does not write an incomplete line and then waits.
Yes. One way to avoid that assumption would be a timeout.
> - git-daemon does not write more than one line, because xread() happily
> reads everything it can get. Your implementation differs from the old
> version because the shell's 'read' is required to read no more than one
> line, i.e., to read byte-wise from the pipe until it sees the LF.

The strbuf_getwholeline_fd implementation calls xread(fd, buf, 1), reading only one byte at a time. It does not try to read beyond the newline.

Johannes Sixt· Apr 15, 2012, 19:57 UTC · re: Clemens Buchacher · lore

Re: [PATCH] git-daemon wrapper to wait until daemon is ready

Am 15.04.2012 21:32, schrieb Clemens Buchacher:
> The strbuf_getwholeline_fd implementation calls xread(fd, buf, 1),
> reading only one byte at a time. It does not try to read beyond the
> newline.

Point taken. I only saw "xread" and didn't look further. Sorry for the noise. I now crawl back under my rock.

-- Hannes

← back to recent threads