git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 11/19] Add watchman support to reduce index refresh cost

From
David Turner <dturner@twopensource.com>
Date
Mar 9, 2016, 18:36 UTC
Message-ID
<1457548582-28302-12-git-send-email-dturner@twopensource.com>
In-Reply-To
<1457548582-28302-1-git-send-email-dturner@twopensource.com>
From: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>

The previous patch has the logic to clear bits in 'WAMA' bitmap. This patch has logic to set bits as told by watchman. The missing bit, _using_ these bits, are not here yet.

A lot of this code is written by David Turner originally, mostly from [1]. I'm just copying and polishing it a bit.

[1] http://article.gmane.org/gmane.comp.version-control.git/248006
Signed-off-by: David Turner <dturner@twopensource.com>
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 Makefile           |   7 ++++
 cache.h            |   1 +
 config.c           |   5 +++
 configure.ac       |   8 ++++
 environment.c      |   3 ++
 watchman-support.c | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 watchman-support.h |   7 ++++
 7 files changed, 146 insertions(+)
 create mode 100644 watchman-support.c
 create mode 100644 watchman-support.h
diff --git a/Makefile b/Makefile
index a6c668b..e51331c 100644
--- a/Makefile
+++ b/Makefile
@@ -1416,6 +1416,12 @@ else
 	LIB_OBJS += thread-utils.o
 endif
 
+ifdef USE_WATCHMAN
+	LIB_H += watchman-support.h
+	LIB_OBJS += watchman-support.o
+	BASIC_CFLAGS += -DUSE_WATCHMAN
+endif
+
 ifdef HAVE_PATHS_H
 	BASIC_CFLAGS += -DHAVE_PATHS_H
 endif
@@ -2164,6 +2170,7 @@ GIT-BUILD-OPTIONS: FORCE
 	@echo NO_PERL=\''$(subst ','\'',$(subst ','\'',$(NO_PERL)))'\' >>$@+
 	@echo NO_PYTHON=\''$(subst ','\'',$(subst ','\'',$(NO_PYTHON)))'\' >>$@+
 	@echo NO_UNIX_SOCKETS=\''$(subst ','\'',$(subst ','\'',$(NO_UNIX_SOCKETS)))'\' >>$@+
+	@echo USE_WATCHMAN=\''$(subst ','\'',$(subst ','\'',$(USE_WATCHMAN)))'\' >>$@+
 ifdef TEST_OUTPUT_DIRECTORY
 	@echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+
 endif
diff --git a/cache.h b/cache.h
index 8f7b4b1..bf20652 100644
--- a/cache.h
+++ b/cache.h
@@ -688,6 +688,7 @@ extern char *git_replace_ref_base;
 
 extern int fsync_object_files;
 extern int core_preload_index;
+extern int core_watchman_sync_timeout;
 extern int core_apply_sparse_checkout;
 extern int precomposed_unicode;
 extern int protect_hfs;
diff --git a/config.c b/config.c
index 9ba40bc..e6dc141 100644
--- a/config.c
+++ b/config.c
@@ -882,6 +882,11 @@ static int git_default_core_config(const char *var, const char *value)
 		return 0;
 	}
 
+	if (!strcmp(var, "core.watchmansynctimeout")) {
+		core_watchman_sync_timeout = git_config_int(var, value);
+		return 0;
+	}
+
 	if (!strcmp(var, "core.createobject")) {
 		if (!strcmp(value, "rename"))
 			object_creation_mode = OBJECT_CREATION_USES_RENAMES;
diff --git a/configure.ac b/configure.ac
index 89e2590..6f10a15 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1092,6 +1092,14 @@ AC_COMPILE_IFELSE([BSD_SYSCTL_SRC],
 	HAVE_BSD_SYSCTL=])
 GIT_CONF_SUBST([HAVE_BSD_SYSCTL])
 
+#
+# Check for watchman client library
+
+AC_CHECK_LIB([watchman], [watchman_connect],
+	[USE_WATCHMAN=YesPlease],
+	[USE_WATCHMAN=])
+GIT_CONF_SUBST([USE_WATCHMAN])
+
 ## Other checks.
 # Define USE_PIC if you need the main git objects to be built with -fPIC
 # in order to build and link perl/Git.so.  x86-64 seems to need this.
diff --git a/environment.c b/environment.c
index 6dec9d0..35e03c7 100644
--- a/environment.c
+++ b/environment.c
@@ -94,6 +94,9 @@ int core_preload_index = 1;
  */
 int ignore_untracked_cache_config;
 
+int core_watchman_sync_timeout = 300;
+
+
 /* This is set by setup_git_dir_gently() and/or git_default_config() */
 char *git_work_tree_cfg;
 static char *work_tree;
diff --git a/watchman-support.c b/watchman-support.c
new file mode 100644
index 0000000..08e37ae
--- /dev/null
+++ b/watchman-support.c
@@ -0,0 +1,115 @@
+#include "cache.h"
+#include "watchman-support.h"
+#include "strbuf.h"
+#include <watchman.h>
+
+static struct watchman_query *make_query(const char *last_update)
+{
+	struct watchman_query *query = watchman_query();
+	watchman_query_set_fields(query, WATCHMAN_FIELD_NAME |
+					 WATCHMAN_FIELD_EXISTS |
+					 WATCHMAN_FIELD_NEWER);
+	watchman_query_set_empty_on_fresh(query, 1);
+	query->sync_timeout = core_watchman_sync_timeout;
+	if (*last_update)
+		watchman_query_set_since_oclock(query, last_update);
+	return query;
+}
+
+static struct watchman_query_result* query_watchman(
+	struct index_state *istate, struct watchman_connection *connection,
+	const char *fs_path, const char *last_update)
+{
+	struct watchman_error wm_error;
+	struct watchman_query *query;
+	struct watchman_expression *expr;
+	struct watchman_query_result *result;
+
+	query = make_query(last_update);
+	expr = watchman_true_expression();
+	result = watchman_do_query(connection, fs_path, query, expr, &wm_error);
+	watchman_free_query(query);
+	watchman_free_expression(expr);
+
+	if (!result)
+		warning("Watchman query error: %s (at %s)",
+			wm_error.message,
+			*last_update ? last_update : "the beginning");
+
+	return result;
+}
+
+static void update_index(struct index_state *istate,
+			 struct watchman_query_result *result)
+{
+	int i;
+
+	if (result->is_fresh_instance) {
+		/* let refresh clear them later */
+		for (i = 0; i < istate->cache_nr; i++)
+			istate->cache[i]->ce_flags |= CE_WATCHMAN_DIRTY;
+		goto done;
+	}
+
+	for (i = 0; i < result->nr; i++) {
+		struct watchman_stat *wm = result->stats + i;
+		int pos;
+
+		if (!strncmp(wm->name, ".git/", 5) ||
+		    strstr(wm->name, "/.git/"))
+			continue;
+
+		pos = index_name_pos(istate, wm->name, strlen(wm->name));
+		if (pos < 0)
+			continue;
+		/* FIXME: ignore staged entries and gitlinks too? */
+
+		istate->cache[pos]->ce_flags |= CE_WATCHMAN_DIRTY;
+	}
+
+done:
+	free(istate->last_update);
+	istate->last_update    = xstrdup(result->clock);
+	istate->cache_changed |= WATCHMAN_CHANGED;
+}
+
+int check_watchman(struct index_state *istate)
+{
+	struct watchman_error wm_error;
+	struct watchman_connection *connection;
+	struct watchman_query_result *result;
+	const char *fs_path;
+	struct timeval timeout;
+	/*
+	 * Convert core_watchman_sync_timeout, in milliseconds, to
+	 * struct timeval, in seconds and microseconds.
+	 */
+
+	fs_path = get_git_work_tree();
+	if (!fs_path)
+		return -1;
+
+	timeout.tv_sec = core_watchman_sync_timeout / 1000;
+	timeout.tv_usec = (core_watchman_sync_timeout % 1000) * 1000;
+	connection = watchman_connect(timeout, &wm_error);
+
+	if (!connection) {
+		warning("Watchman watch error: %s", wm_error.message);
+		return -1;
+	}
+
+	if (watchman_watch(connection, fs_path, &wm_error)) {
+		warning("Watchman watch error: %s", wm_error.message);
+		watchman_connection_close(connection);
+		return -1;
+	}
+
+
+	result = query_watchman(istate, connection, fs_path, istate->last_update);
+	watchman_connection_close(connection);
+	if (!result)
+		return -1;
+	update_index(istate, result);
+	watchman_free_query_result(result);
+	return 0;
+}
diff --git a/watchman-support.h b/watchman-support.h
new file mode 100644
index 0000000..ee1ef2c
--- /dev/null
+++ b/watchman-support.h
@@ -0,0 +1,7 @@
+#ifndef WATCHMAN_SUPPORT_H
+#define WATCHMAN_SUPPORT_H
+
+struct index_state;
+int check_watchman(struct index_state *index);
+
+#endif /* WATCHMAN_SUPPORT_H */

-- 
2.4.2.767.g62658d5-twtrsrc
Previous: David TurnerNext: David Turner
Message 30 of 60 in “index-helper, watchman”
  1. 00/19 index-helper, watchmanDavid Turner, Mar 9, 2016
  2. 01/19 trace.c: add GIT_TRACE_PACK_STATS for pack usage statisticsDavid Turner, Mar 9, 2016
  3. Junio C HamanoMar 9, 2016
  4. David TurnerMar 10, 2016
  5. Duy NguyenMar 10, 2016
  6. 02/19 read-cache.c: fix constness of verify_hdr()David Turner, Mar 9, 2016
  7. 03/19 read-cache: allow to keep mmap'd memory after readingDavid Turner, Mar 9, 2016
  8. Junio C HamanoMar 9, 2016
  9. David TurnerMar 10, 2016
  10. 04/19 index-helper: new daemon for caching index and related stuffDavid Turner, Mar 9, 2016
  11. Junio C HamanoMar 9, 2016
  12. Junio C HamanoMar 9, 2016
  13. David TurnerMar 10, 2016
  14. Duy NguyenMar 10, 2016
  15. David TurnerMar 10, 2016
  16. Duy NguyenMar 11, 2016
  17. David TurnerMar 10, 2016
  18. Duy NguyenMar 15, 2016
  19. Junio C HamanoMar 15, 2016
  20. Duy NguyenMar 15, 2016
  21. 05/19 trace.c: add GIT_TRACE_INDEX_STATS for index statisticsDavid Turner, Mar 9, 2016
  22. 06/19 index-helper: add --strictDavid Turner, Mar 9, 2016
  23. 07/19 daemonize(): set a flag before exiting the main processDavid Turner, Mar 9, 2016
  24. 08/19 index-helper: add --detachDavid Turner, Mar 9, 2016
  25. 09/19 index-helper: add Windows supportDavid Turner, Mar 9, 2016
  26. Duy NguyenMar 16, 2016
  27. Johannes SchindelinMar 17, 2016
  28. Duy NguyenMar 17, 2016
  29. 10/19 read-cache: add watchman 'WAMA' extensionDavid Turner, Mar 9, 2016
  30. 11/19 Add watchman support to reduce index refresh costDavid Turner, Mar 9, 2016
  31. 12/19 read-cache: allow index-helper to prepare shm before git reads itDavid Turner, Mar 9, 2016
  32. 13/19 index-helper: use watchman to avoid refreshing index with lstat()David Turner, Mar 9, 2016
  33. 14/19 update-index: enable/disable watchman supportDavid Turner, Mar 9, 2016
  34. 15/19 unpack-trees: preserve index extensionsDavid Turner, Mar 9, 2016
  35. 16/19 index-helper: rewrite pidfile after daemonizingDavid Turner, Mar 9, 2016
  36. 17/19 index-helper: process managementDavid Turner, Mar 9, 2016
  37. 18/19 index-helper: autorunDavid Turner, Mar 9, 2016
  38. Duy NguyenMar 15, 2016
  39. Johannes SchindelinMar 15, 2016
  40. Duy NguyenMar 16, 2016
  41. David TurnerMar 16, 2016
  42. Johannes SchindelinMar 16, 2016
  43. Duy NguyenMar 17, 2016
  44. Johannes SchindelinMar 17, 2016
  45. David TurnerMar 17, 2016
  46. Duy NguyenMar 18, 2016
  47. Johannes SchindelinMar 18, 2016
  48. Duy NguyenMar 18, 2016
  49. David TurnerMar 18, 2016
  50. Duy NguyenMar 18, 2016
  51. Johannes SchindelinMar 18, 2016
  52. Duy NguyenMar 18, 2016
  53. Johannes SchindelinMar 18, 2016
  54. 19/19 hack: watchman/untracked cache mashupDavid Turner, Mar 9, 2016
  55. Duy NguyenMar 15, 2016
  56. David TurnerMar 17, 2016
  57. Duy NguyenMar 17, 2016
  58. David TurnerMar 17, 2016
  59. Torsten BögershausenMar 29, 2016
  60. David TurnerMar 29, 2016

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.