From: Jeff Hostetler via GitGitGadget Date: Fri, 31 Aug 2018 16:49:55 GMT Subject: [PATCH 1/8] trace2: create new combined trace facility Message-ID: <82885700379efe6d6a83629cac4d943b99b393bf.1535734192.git.gitgitgadget@gmail.com> In-Reply-To: From: Jeff Hostetler Create trace2 API allowing event-based tracing. This will hopefully eventually replace the existing trace API. Create GIT_TR2 trace-key to replace GIT_TRACE, GIT_TR2_PERFORMANCE to replace GIT_TRACE_PERFORMANCE, and a new trace-key GIT_TR2_EVENT to generate JSON data for telemetry purposes. Other structured formats can easily be added later using this new existing model. Define a higher-level event API that selectively writes to all of the new GIT_TR2_* targets (depending on event type) without needing to call different trace_printf*() or trace_performance_*() routines. The API defines both fixed-field and printf-style functions. The trace2 performance tracing includes thread-specific function nesting and timings. Signed-off-by: Jeff Hostetler --- Makefile | 1 + cache.h | 1 + trace2.c | 1592 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ trace2.h | 214 ++++++++ 4 files changed, 1808 insertions(+) create mode 100644 trace2.c create mode 100644 trace2.h diff --git a/Makefile b/Makefile index 5a969f5830..88ae7afdd4 100644 --- a/Makefile +++ b/Makefile @@ -974,6 +974,7 @@ LIB_OBJS += tag.o LIB_OBJS += tempfile.o LIB_OBJS += tmp-objdir.o LIB_OBJS += trace.o +LIB_OBJS += trace2.o LIB_OBJS += trailer.o LIB_OBJS += transport.o LIB_OBJS += transport-helper.o diff --git a/cache.h b/cache.h index 4d014541ab..38f0cd2ba0 100644 --- a/cache.h +++ b/cache.h @@ -9,6 +9,7 @@ #include "gettext.h" #include "convert.h" #include "trace.h" +#include "trace2.h" #include "string-list.h" #include "pack-revindex.h" #include "hash.h" diff --git a/trace2.c b/trace2.c new file mode 100644 index 0000000000..13d5c85366 --- /dev/null +++ b/trace2.c @@ -0,0 +1,1592 @@ +#include "cache.h" +#include "config.h" +#include "json-writer.h" +#include "quote.h" +#include "run-command.h" +#include "sigchain.h" +#include "thread-utils.h" +#include "version.h" + +/***************************************************************** + * TODO remove this section header + *****************************************************************/ + +static struct strbuf tr2sid_buf = STRBUF_INIT; +static int tr2sid_nr_git_parents; + +/* + * Compute a "unique" session id (SID) for the current process. All events + * from this process will have this label. If we were started by another + * git instance, use our parent's SID as a prefix and count the number of + * nested git processes. (This lets us track parent/child relationships + * even if there is an intermediate shell process.) + */ +static void tr2sid_compute(void) +{ + uint64_t us_now; + const char *parent_sid; + + if (tr2sid_buf.len) + return; + + parent_sid = getenv("SLOG_PARENT_SID"); + if (parent_sid && *parent_sid) { + const char *p; + for (p = parent_sid; *p; p++) + if (*p == '/') + tr2sid_nr_git_parents++; + + strbuf_addstr(&tr2sid_buf, parent_sid); + strbuf_addch(&tr2sid_buf, '/'); + tr2sid_nr_git_parents++; + } + + us_now = getnanotime() / 1000; + strbuf_addf(&tr2sid_buf, "%"PRIuMAX"-%"PRIdMAX, + (uintmax_t)us_now, (intmax_t)getpid()); + + setenv("SLOG_PARENT_SID", tr2sid_buf.buf, 1); +} + +/***************************************************************** + * TODO remove this section header + *****************************************************************/ + +/* + * Each thread (including the main thread) has a stack of nested regions. + * This is used to indent messages and provide nested perf times. The + * limit here is for simplicity. Note that the region stack is per-thread + * and not per-trace_key. + */ +#define TR2_MAX_REGION_NESTING (100) +#define TR2_INDENT (2) +#define TR2_INDENT_LENGTH(ctx) (((ctx)->nr_open_regions - 1) * TR2_INDENT) + +#define TR2_MAX_THREAD_NAME (24) + +struct tr2tls_thread_ctx { + struct strbuf thread_name; + uint64_t us_start[TR2_MAX_REGION_NESTING]; + int nr_open_regions; + int thread_id; +}; + +static struct tr2tls_thread_ctx *tr2tls_thread_main; + +static pthread_mutex_t tr2tls_mutex; +static pthread_key_t tr2tls_key; +static int tr2tls_initialized; + +static struct strbuf tr2_dots = STRBUF_INIT; + +static int tr2_next_child_id; +static int tr2_next_thread_id; + +/* + * Create TLS data for the current thread. This gives us a place to + * put per-thread data, such as thread start time, function nesting + * and a per-thread label for our messages. + * + * We assume the first thread is "main". Other threads are given + * non-zero thread-ids to help distinguish messages from concurrent + * threads. + * + * Truncate the thread name if necessary to help with column alignment + * in printf-style messages. + */ +static struct tr2tls_thread_ctx *tr2tls_create_self(const char *thread_name) +{ + struct tr2tls_thread_ctx *ctx = xcalloc(1, sizeof(*ctx)); + + /* + * Implicitly "tr2tls_push_self()" to capture the thread's start + * time in us_start[0]. For the main thread this gives us the + * application run time. + */ + ctx->us_start[ctx->nr_open_regions++] = getnanotime() / 1000; + + pthread_mutex_lock(&tr2tls_mutex); + ctx->thread_id = tr2_next_thread_id++; + pthread_mutex_unlock(&tr2tls_mutex); + + strbuf_init(&ctx->thread_name, 0); + if (ctx->thread_id) + strbuf_addf(&ctx->thread_name, "th%02d:", ctx->thread_id); + strbuf_addstr(&ctx->thread_name, thread_name); + if (ctx->thread_name.len > TR2_MAX_THREAD_NAME) + strbuf_setlen(&ctx->thread_name, TR2_MAX_THREAD_NAME); + + pthread_setspecific(tr2tls_key, ctx); + + return ctx; +} + +static struct tr2tls_thread_ctx *tr2tls_get_self(void) +{ + struct tr2tls_thread_ctx * ctx = pthread_getspecific(tr2tls_key); + + /* + * If the thread-proc did not call trace2_thread_start(), we won't + * have any TLS data associated with the current thread. Fix it + * here and silently continue. + */ + if (!ctx) + ctx = tr2tls_create_self("unknown"); + + return ctx; +} + +static void tr2tls_unset_self(void) +{ + struct tr2tls_thread_ctx * ctx; + + ctx = tr2tls_get_self(); + + pthread_setspecific(tr2tls_key, NULL); + + free(ctx); +} + +/* + * Begin a new nested region and remember the start time. + */ +static void tr2tls_push_self(void) +{ + struct tr2tls_thread_ctx *ctx = tr2tls_get_self(); + + if (ctx->nr_open_regions == TR2_MAX_REGION_NESTING) + BUG("exceeded max region nesting '%d' in thread '%s'", + ctx->nr_open_regions, ctx->thread_name.buf); + + ctx->us_start[ctx->nr_open_regions++] = getnanotime() / 1000; +} + +/* + * End the innermost nested region. + */ +static void tr2tls_pop_self(void) +{ + struct tr2tls_thread_ctx *ctx = tr2tls_get_self(); + + if (!ctx->nr_open_regions) + BUG("no open regions in thread '%s'", ctx->thread_name.buf); + + ctx->nr_open_regions--; +} + +/* + * Compute elapsed time in innermost nested region. + */ +static uint64_t tr2tls_region_elapsed_self(void) +{ + struct tr2tls_thread_ctx *ctx; + uint64_t us_start; + uint64_t us_now; + + ctx = tr2tls_get_self(); + if (!ctx->nr_open_regions) + return 0; + + us_now = getnanotime() / 1000; + us_start = ctx->us_start[ctx->nr_open_regions - 1]; + + return us_now - us_start; +} + +/* + * Compute the elapsed run time since the main thread started. + */ +static uint64_t tr2tls_elapsed_main(void) +{ + uint64_t us_start; + uint64_t us_now; + uint64_t us_elapsed; + + if (!tr2tls_thread_main) + return 0; + + us_now = getnanotime() / 1000; + us_start = tr2tls_thread_main->us_start[0]; + us_elapsed = us_now - us_start; + + return us_elapsed; +} + +static void tr2tls_init(void) +{ + pthread_key_create(&tr2tls_key, NULL); + init_recursive_mutex(&tr2tls_mutex); + + tr2tls_thread_main = tr2tls_create_self("main"); + tr2tls_initialized = 1; +} + +static void tr2tls_release(void) +{ + tr2tls_unset_self(); + tr2tls_thread_main = NULL; + + tr2tls_initialized = 0; + + pthread_mutex_destroy(&tr2tls_mutex); + pthread_key_delete(tr2tls_key); +} + +/***************************************************************** + * TODO remove this section header + *****************************************************************/ + +static struct trace_key tr2key_default = { "GIT_TR2", 0, 0, 0 }; +static struct trace_key tr2key_perf = { "GIT_TR2_PERFORMANCE", 0, 0, 0 }; +static struct trace_key tr2key_event = { "GIT_TR2_EVENT", 0, 0, 0 }; + +/* + * Region depth limit for messages written to the event target. + * + * The "region_enter" and "region_leave" messages (especially recursive + * messages such as those produced while diving the worktree or index) + * are primarily intended for the performance target during debugging. + * + * Some of the outer-most messages, however, may be of interest to the + * event target. Set this environment variable to a larger integer for + * more detail in the event target. + */ +#define GIT_TR2_EVENT_DEPTH "GIT_TR2_EVENT_DEPTH" +static int tr2env_event_depth_wanted = 2; + +/* + * Set this environment variable to true to omit the "