# [PATCH 6/6] Dying in an async procedure should only exit the thread, not the process. 6 messages from 2010-03-06 to 2010-03-06. Participants: Johannes Sixt. Thread: https://gitlist.dev/t/22940 ## Johannes Sixt, 2010-03-06 15:40 Subject: [PATCH 1/6] Modernize t5530-upload-pack-error. Message-ID: <40f4b185c3661e5826ec2e58124137cd735e7199.1267889703.git.j6t@kdbg.org> URL: https://gitlist.dev/e/40f4b185c3661e5826ec2e58124137cd735e7199.1267889703.git.j6t%40kdbg.org In-Reply-To: ``` Some tests did not use test_must_fail. Signed-off-by: Johannes Sixt --- t/t5530-upload-pack-error.sh | 18 +++++++++--------- 1 files changed, 9 insertions(+), 9 deletions(-) diff --git a/t/t5530-upload-pack-error.sh b/t/t5530-upload-pack-error.sh index a696b87..044603c 100755 --- a/t/t5530-upload-pack-error.sh +++ b/t/t5530-upload-pack-error.sh @@ -32,9 +32,9 @@ test_expect_success 'fsck fails' ' test_expect_success 'upload-pack fails due to error in pack-objects packing' ' - ! echo "0032want $(git rev-parse HEAD) -00000009done -0000" | git upload-pack . > /dev/null 2> output.err && + printf "0032want %s\n00000009done\n0000" \ + $(git rev-parse HEAD) >input && + test_must_fail git upload-pack . /dev/null 2>output.err && grep "unable to read" output.err && grep "pack-objects died" output.err ' @@ -51,9 +51,9 @@ test_expect_success 'fsck fails' ' ' test_expect_success 'upload-pack fails due to error in rev-list' ' - ! echo "0032want $(git rev-parse HEAD) -0034shallow $(git rev-parse HEAD^)00000009done -0000" | git upload-pack . > /dev/null 2> output.err && + printf "0032want %s\n0034shallow %s00000009done\n0000" \ + $(git rev-parse HEAD) $(git rev-parse HEAD^) >input && + test_must_fail git upload-pack . /dev/null 2>output.err && # pack-objects survived grep "Total.*, reused" output.err && # but there was an error, which must have been in rev-list @@ -62,9 +62,9 @@ test_expect_success 'upload-pack fails due to error in rev-list' ' test_expect_success 'upload-pack fails due to error in pack-objects enumeration' ' - ! echo "0032want $(git rev-parse HEAD) -00000009done -0000" | git upload-pack . > /dev/null 2> output.err && + printf "0032want %s\n00000009done\n0000" \ + $(git rev-parse HEAD) >input && + test_must_fail git upload-pack . /dev/null 2>output.err && grep "bad tree object" output.err && grep "pack-objects died" output.err ' -- 1.7.0.rc2.65.g7b13a ``` ## Johannes Sixt, 2010-03-06 15:40 Subject: [PATCH 2/6] Make report() from usage.c public as vreportf() and use it. Message-ID: URL: https://gitlist.dev/e/dd91671bd5bb1a60a314e71a265ba0068080a47e.1267889703.git.j6t%40kdbg.org In-Reply-To: ``` There exist already a number of static functions named 'report', therefore, the function name was changed. Signed-off-by: Johannes Sixt --- fast-import.c | 8 +++----- git-compat-util.h | 1 + http-backend.c | 5 +---- usage.c | 10 +++++----- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/fast-import.c b/fast-import.c index 74f08bd..5bb3d41 100644 --- a/fast-import.c +++ b/fast-import.c @@ -483,14 +483,12 @@ static void dump_marks(void); static NORETURN void die_nicely(const char *err, va_list params) { static int zombie; - char message[2 * PATH_MAX]; - vsnprintf(message, sizeof(message), err, params); - fputs("fatal: ", stderr); - fputs(message, stderr); - fputc('\n', stderr); + vreportf("fatal: ", err, params); if (!zombie) { + char message[2 * PATH_MAX]; + zombie = 1; write_crash_report(message); end_packfile(); diff --git a/git-compat-util.h b/git-compat-util.h index a3c4537..3cabcdd 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -192,6 +192,7 @@ extern char *gitbasename(char *); #include "compat/bswap.h" /* General helper functions */ +extern void vreportf(const char *prefix, const char *err, va_list params); extern NORETURN void usage(const char *err); extern NORETURN void usagef(const char *err, ...) __attribute__((format (printf, 1, 2))); extern NORETURN void die(const char *err, ...) __attribute__((format (printf, 1, 2))); diff --git a/http-backend.c b/http-backend.c index 345c12b..8c7b7d0 100644 --- a/http-backend.c +++ b/http-backend.c @@ -538,14 +538,11 @@ static void service_rpc(char *service_name) static NORETURN void die_webcgi(const char *err, va_list params) { - char buffer[1000]; - http_status(500, "Internal Server Error"); hdr_nocache(); end_headers(); - vsnprintf(buffer, sizeof(buffer), err, params); - fprintf(stderr, "fatal: %s\n", buffer); + vreportf("fatal: ", err, params); exit(0); } diff --git a/usage.c b/usage.c index 79856a2..ec4cf53 100644 --- a/usage.c +++ b/usage.c @@ -5,7 +5,7 @@ */ #include "git-compat-util.h" -static void report(const char *prefix, const char *err, va_list params) +void vreportf(const char *prefix, const char *err, va_list params) { char msg[4096]; vsnprintf(msg, sizeof(msg), err, params); @@ -14,24 +14,24 @@ static void report(const char *prefix, const char *err, va_list params) static NORETURN void usage_builtin(const char *err, va_list params) { - report("usage: ", err, params); + vreportf("usage: ", err, params); exit(129); } static NORETURN void die_builtin(const char *err, va_list params) { - report("fatal: ", err, params); + vreportf("fatal: ", err, params); exit(128); } static void error_builtin(const char *err, va_list params) { - report("error: ", err, params); + vreportf("error: ", err, params); } static void warn_builtin(const char *warn, va_list params) { - report("warning: ", warn, params); + vreportf("warning: ", warn, params); } /* If we are in a dlopen()ed .so write to a global variable would segfault -- 1.7.0.rc2.65.g7b13a ``` ## Johannes Sixt, 2010-03-06 15:40 Subject: [PATCH 3/6] Fix signature of fcntl() compatibility dummy Message-ID: URL: https://gitlist.dev/e/dc9be33a660733a88e1c2fd8a3d9c190a7779d5d.1267889703.git.j6t%40kdbg.org In-Reply-To: ``` Obviously, this function was never called with two arguments in Windows code sections, but this will be the case in a subsequent patch. Signed-off-by: Johannes Sixt --- compat/mingw.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/compat/mingw.h b/compat/mingw.h index e254fb4..f53bcca 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -89,7 +89,7 @@ static inline int getuid() { return 1; } static inline struct passwd *getpwnam(const char *name) { return NULL; } -static inline int fcntl(int fd, int cmd, long arg) +static inline int fcntl(int fd, int cmd, ...) { if (cmd == F_GETFD || cmd == F_SETFD) return 0; -- 1.7.0.rc2.65.g7b13a ``` ## Johannes Sixt, 2010-03-06 15:40 Subject: [PATCH 4/6] Windows: more pthreads functions Message-ID: <630b17c350a0014d8099576faeaf113d2804bde6.1267889703.git.j6t@kdbg.org> URL: https://gitlist.dev/e/630b17c350a0014d8099576faeaf113d2804bde6.1267889703.git.j6t%40kdbg.org In-Reply-To: ``` This adds: pthread_self pthread_equal pthread_exit pthread_key_create pthread_setspecific pthread_getspecific Signed-off-by: Johannes Sixt --- compat/win32/pthread.c | 8 ++++++++ compat/win32/pthread.h | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 0 deletions(-) diff --git a/compat/win32/pthread.c b/compat/win32/pthread.c index 0f949fc..010e875 100644 --- a/compat/win32/pthread.c +++ b/compat/win32/pthread.c @@ -16,6 +16,7 @@ static unsigned __stdcall win32_start_routine(void *arg) { pthread_t *thread = arg; + thread->tid = GetCurrentThreadId(); thread->arg = thread->start_routine(thread->arg); return 0; } @@ -49,6 +50,13 @@ int win32_pthread_join(pthread_t *thread, void **value_ptr) } } +pthread_t pthread_self(void) +{ + pthread_t t = { 0 }; + t.tid = GetCurrentThreadId(); + return t; +} + int pthread_cond_init(pthread_cond_t *cond, const void *unused) { cond->waiters = 0; diff --git a/compat/win32/pthread.h b/compat/win32/pthread.h index c72f100..c7b8241 100644 --- a/compat/win32/pthread.h +++ b/compat/win32/pthread.h @@ -52,6 +52,7 @@ typedef struct { HANDLE handle; void *(*start_routine)(void*); void *arg; + DWORD tid; } pthread_t; extern int pthread_create(pthread_t *thread, const void *unused, @@ -65,4 +66,28 @@ extern int pthread_create(pthread_t *thread, const void *unused, extern int win32_pthread_join(pthread_t *thread, void **value_ptr); +#define pthread_equal(t1, t2) ((t1).tid == (t2).tid) +extern pthread_t pthread_self(void); + +static inline int pthread_exit(void *ret) +{ + ExitThread((DWORD)ret); +} + +typedef DWORD pthread_key_t; +static inline int pthread_key_create(pthread_key_t *keyp, void (*destructor)(void *value)) +{ + return (*keyp = TlsAlloc()) == TLS_OUT_OF_INDEXES ? EAGAIN : 0; +} + +static inline int pthread_setspecific(pthread_key_t key, const void *value) +{ + return TlsSetValue(key, (void *)value) ? 0 : EINVAL; +} + +static inline void *pthread_getspecific(pthread_key_t key) +{ + return TlsGetValue(key); +} + #endif /* PTHREAD_H */ -- 1.7.0.rc2.65.g7b13a ``` ## Johannes Sixt, 2010-03-06 15:40 Subject: [PATCH 5/6] Reimplement async procedures using pthreads Message-ID: <771cabb3efea5ced5049bdd34c392a218d1f3c4f.1267889703.git.j6t@kdbg.org> URL: https://gitlist.dev/e/771cabb3efea5ced5049bdd34c392a218d1f3c4f.1267889703.git.j6t%40kdbg.org In-Reply-To: ``` On Windows, async procedures have always been run in threads, and the implementation used Windows specific APIs. Rewrite the code to use pthreads. A new configuration option is introduced so that the threaded implementation can also be used on POSIX systems. Since this option is intended only as playground on POSIX, but is mandatory on Windows, the option is not documented. One detail is that on POSIX it is necessary to set FD_CLOEXEC on the pipe handles. On Windows, this is not needed because pipe handles are not inherited to child processes, and the new calls to set_cloexec() are effectively no-ops. Signed-off-by: Johannes Sixt --- Makefile | 5 +++++ run-command.c | 41 +++++++++++++++++++++++------------------ run-command.h | 8 ++++++-- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index afedb54..c5f0f9b 100644 --- a/Makefile +++ b/Makefile @@ -949,6 +949,7 @@ ifeq ($(uname_S),Windows) NO_CURL = YesPlease NO_PYTHON = YesPlease BLK_SHA1 = YesPlease + ASYNC_AS_THREAD = YesPlease CC = compat/vcbuild/scripts/clink.pl AR = compat/vcbuild/scripts/lib.pl @@ -1000,6 +1001,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_REGEX = YesPlease NO_PYTHON = YesPlease BLK_SHA1 = YesPlease + ASYNC_AS_THREAD = YesPlease COMPAT_CFLAGS += -D__USE_MINGW_ACCESS -DNOGDI -Icompat -Icompat/fnmatch -Icompat/win32 COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/fnmatch/fnmatch.o compat/winansi.o \ @@ -1301,6 +1303,9 @@ ifdef NO_PTHREADS else EXTLIBS += $(PTHREAD_LIBS) LIB_OBJS += thread-utils.o +ifdef ASYNC_AS_THREAD + BASIC_CFLAGS += -DASYNC_AS_THREAD +endif endif ifdef DIR_HAS_BSD_GROUP_SEMANTICS diff --git a/run-command.c b/run-command.c index 0cd7f02..77aefff 100644 --- a/run-command.c +++ b/run-command.c @@ -82,6 +82,7 @@ static NORETURN void die_child(const char *err, va_list params) write(child_err, "\n", 1); exit(128); } +#endif static inline void set_cloexec(int fd) { @@ -89,7 +90,6 @@ static inline void set_cloexec(int fd) if (flags >= 0) fcntl(fd, F_SETFD, flags | FD_CLOEXEC); } -#endif static int wait_or_whine(pid_t pid, const char *argv0, int silent_exec_failure) { @@ -447,11 +447,12 @@ int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const return run_command(&cmd); } -#ifdef WIN32 -static unsigned __stdcall run_thread(void *data) +#ifdef ASYNC_AS_THREAD +static void *run_thread(void *data) { struct async *async = data; - return async->proc(async->proc_in, async->proc_out, async->data); + intptr_t ret = async->proc(async->proc_in, async->proc_out, async->data); + return (void *)ret; } #endif @@ -497,7 +498,7 @@ int start_async(struct async *async) else proc_out = -1; -#ifndef WIN32 +#ifndef ASYNC_AS_THREAD /* Flush stdio before fork() to avoid cloning buffers */ fflush(NULL); @@ -524,12 +525,18 @@ int start_async(struct async *async) else if (async->out) close(async->out); #else + if (proc_in >= 0) + set_cloexec(proc_in); + if (proc_out >= 0) + set_cloexec(proc_out); async->proc_in = proc_in; async->proc_out = proc_out; - async->tid = (HANDLE) _beginthreadex(NULL, 0, run_thread, async, 0, NULL); - if (!async->tid) { - error("cannot create thread: %s", strerror(errno)); - goto error; + { + int err = pthread_create(&async->tid, NULL, run_thread, async); + if (err) { + error("cannot create thread: %s", strerror(err)); + goto error; + } } #endif return 0; @@ -549,17 +556,15 @@ error: int finish_async(struct async *async) { -#ifndef WIN32 - int ret = wait_or_whine(async->pid, "child process", 0); +#ifndef ASYNC_AS_THREAD + return wait_or_whine(async->pid, "child process", 0); #else - DWORD ret = 0; - if (WaitForSingleObject(async->tid, INFINITE) != WAIT_OBJECT_0) - ret = error("waiting for thread failed: %lu", GetLastError()); - else if (!GetExitCodeThread(async->tid, &ret)) - ret = error("cannot get thread exit code: %lu", GetLastError()); - CloseHandle(async->tid); + void *ret = (void *)(intptr_t)(-1); + + if (pthread_join(async->tid, &ret)) + error("pthread_join failed"); + return (int)(intptr_t)ret; #endif - return ret; } int run_hook(const char *index_file, const char *name, ...) diff --git a/run-command.h b/run-command.h index 94619f5..40db39c 100644 --- a/run-command.h +++ b/run-command.h @@ -1,6 +1,10 @@ #ifndef RUN_COMMAND_H #define RUN_COMMAND_H +#ifdef ASYNC_AS_THREAD +#include +#endif + struct child_process { const char **argv; pid_t pid; @@ -74,10 +78,10 @@ struct async { void *data; int in; /* caller writes here and closes it */ int out; /* caller reads from here and closes it */ -#ifndef WIN32 +#ifndef ASYNC_AS_THREAD pid_t pid; #else - HANDLE tid; + pthread_t tid; int proc_in; int proc_out; #endif -- 1.7.0.rc2.65.g7b13a ``` ## Johannes Sixt, 2010-03-06 15:40 Subject: [PATCH 6/6] Dying in an async procedure should only exit the thread, not the process. Message-ID: <66dbacb267415a0e2d71c82467093245c06e84e5.1267889703.git.j6t@kdbg.org> URL: https://gitlist.dev/e/66dbacb267415a0e2d71c82467093245c06e84e5.1267889703.git.j6t%40kdbg.org In-Reply-To: ``` Async procedures are intended as helpers that perform a very restricted task, and the caller usually has to manage them in a larger context. Conceptually, the async procedure is not concerned with the "bigger picture" in whose context it is run. When it dies, it is not supposed to destroy this "bigger picture", but rather only its own limit view of the world. On POSIX, the async procedure is run in its own process, and exiting this process naturally had only these limited effects. On Windows (or when ASYNC_AS_THREAD is set), calling die() exited the whole process, destroying the caller (the "big picture") as well. This fixes it to exit only the thread. Without ASYNC_AS_THREAD, one particular effect of exiting the async procedure process is that it automatically closes file descriptors, most notably the writable end of the pipe that the async procedure writes to. The async API already requires that the async procedure closes the pipe ends when it exits normally. But for calls to die() no requirements are imposed. In the non-threaded case the pipe ends are closed implicitly by the exiting process, but in the threaded case, the die routine must take care of closing them. Now t5530-upload-pack-error.sh passes on Windows. Signed-off-by: Johannes Sixt --- run-command.c | 34 ++++++++++++++++++++++++++++++++++ 1 files changed, 34 insertions(+), 0 deletions(-) diff --git a/run-command.c b/run-command.c index 77aefff..66cc4bf 100644 --- a/run-command.c +++ b/run-command.c @@ -448,12 +448,35 @@ int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const } #ifdef ASYNC_AS_THREAD +static pthread_t main_thread; +static int main_thread_set; +static pthread_key_t async_key; + static void *run_thread(void *data) { struct async *async = data; + + pthread_setspecific(async_key, async); + intptr_t ret = async->proc(async->proc_in, async->proc_out, async->data); return (void *)ret; } + +static NORETURN void die_async(const char *err, va_list params) +{ + vreportf("fatal: ", err, params); + + if (!pthread_equal(main_thread, pthread_self())) { + struct async *async = pthread_getspecific(async_key); + if (async->proc_in >= 0) + close(async->proc_in); + if (async->proc_out >= 0) + close(async->proc_out); + pthread_exit((void *)128); + } + + exit(128); +} #endif int start_async(struct async *async) @@ -525,6 +548,17 @@ int start_async(struct async *async) else if (async->out) close(async->out); #else + if (!main_thread_set) { + /* + * We assume that the first time that start_async is called + * it is from the main thread. + */ + main_thread_set = 1; + main_thread = pthread_self(); + pthread_key_create(&async_key, NULL); + set_die_routine(die_async); + } + if (proc_in >= 0) set_cloexec(proc_in); if (proc_out >= 0) -- 1.7.0.rc2.65.g7b13a ```