git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 2/3] Add read-cache--daemon

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
May 13, 2014, 11:15 UTC
Message-ID
<1399979737-8577-3-git-send-email-pclouds@gmail.com>
In-Reply-To
<1399979737-8577-1-git-send-email-pclouds@gmail.com>
---
 .gitignore                 |   1 +
 Makefile                   |   6 ++
 config.mak.uname           |   1 +
 git-compat-util.h          |   8 +++
 read-cache--daemon.c (new) | 167 +++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 183 insertions(+)
 create mode 100644 read-cache--daemon.c
diff --git a/.gitignore b/.gitignore
index 70992a4..07e0cb6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -110,6 +110,7 @@
 /git-pull
 /git-push
 /git-quiltimport
+/git-read-cache--daemon
 /git-read-tree
 /git-rebase
 /git-rebase--am
diff --git a/Makefile b/Makefile
index 028749b..98d22de 100644
--- a/Makefile
+++ b/Makefile
@@ -1502,6 +1502,12 @@ ifdef HAVE_DEV_TTY
 	BASIC_CFLAGS += -DHAVE_DEV_TTY
 endif
 
+ifdef HAVE_SHM
+	BASIC_CFLAGS += -DHAVE_SHM
+	EXTLIBS += -lrt
+	PROGRAM_OBJS += read-cache--daemon.o
+endif
+
 ifdef DIR_HAS_BSD_GROUP_SEMANTICS
 	COMPAT_CFLAGS += -DDIR_HAS_BSD_GROUP_SEMANTICS
 endif
diff --git a/config.mak.uname b/config.mak.uname
index 23a8803..b6a37e5 100644
--- a/config.mak.uname
+++ b/config.mak.uname
@@ -33,6 +33,7 @@ ifeq ($(uname_S),Linux)
 	HAVE_PATHS_H = YesPlease
 	LIBC_CONTAINS_LIBINTL = YesPlease
 	HAVE_DEV_TTY = YesPlease
+	HAVE_SHM = YesPlease
 endif
 ifeq ($(uname_S),GNU/kFreeBSD)
 	NO_STRLCPY = YesPlease
diff --git a/git-compat-util.h b/git-compat-util.h
index f6d3a46..b2116ab 100644
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@ -723,4 +723,12 @@ struct tm *git_gmtime_r(const time_t *, struct tm *);
 #define gmtime_r git_gmtime_r
 #endif
 
+#ifndef HAVE_SHM
+static inline int shm_open(const char *path, int flags, int mode)
+{
+	errno = ENOSYS;
+	return -1;
+}
+#endif
+
 #endif
diff --git a/read-cache--daemon.c b/read-cache--daemon.c
new file mode 100644
index 0000000..52b4067
--- /dev/null
+++ b/read-cache--daemon.c
@@ -0,0 +1,167 @@
+#include "cache.h"
+#include "sigchain.h"
+#include "unix-socket.h"
+#include "split-index.h"
+#include "pkt-line.h"
+
+static char *socket_path;
+static struct strbuf shm_index = STRBUF_INIT;
+static struct strbuf shm_sharedindex = STRBUF_INIT;
+
+static void cleanup_socket(void)
+{
+	if (socket_path)
+		unlink(socket_path);
+	if (shm_index.len)
+		shm_unlink(shm_index.buf);
+	if (shm_sharedindex.len)
+		shm_unlink(shm_sharedindex.buf);
+}
+
+static void cleanup_socket_on_signal(int sig)
+{
+	cleanup_socket();
+	sigchain_pop(sig);
+	raise(sig);
+}
+
+static void share_index(struct index_state *istate, struct strbuf *shm_path)
+{
+	struct strbuf sb = STRBUF_INIT;
+	void *map;
+	int fd;
+
+	strbuf_addf(&sb, "/git-index-%s", sha1_to_hex(istate->sha1));
+	if (shm_path->len && strcmp(sb.buf, shm_path->buf)) {
+		shm_unlink(shm_path->buf);
+		strbuf_reset(shm_path);
+	}
+	fd = shm_open(sb.buf, O_RDWR | O_CREAT | O_TRUNC, 0700);
+	if (fd < 0)
+		return;
+	/*
+	 * We "lock" the shm in preparation by set its size larger
+	 * than expected. The reader is supposed to check the size and
+	 * ignore if shm size is different than the actual file size
+	 */
+	if (ftruncate(fd, istate->mmap_size + 1)) {
+		close(fd);
+		shm_unlink(shm_path->buf);
+		return;
+	}
+	strbuf_addbuf(shm_path, &sb);
+	map = xmmap(NULL, istate->mmap_size, PROT_READ | PROT_WRITE,
+		    MAP_SHARED, fd, 0);
+	if (map == MAP_FAILED) {
+		close(fd);
+		shm_unlink(shm_path->buf);
+		return;
+	}
+	memcpy(map, istate->mmap, istate->mmap_size);
+	munmap(map, istate->mmap_size);
+	/* Now "unlock" it */
+	if (ftruncate(fd, istate->mmap_size)) {
+		close(fd);
+		shm_unlink(shm_path->buf);
+		return;
+	}
+	close(fd);
+}
+
+static void refresh()
+{
+	the_index.keep_mmap = 1;
+	if (read_cache() < 0)
+		die("could not read index");
+	share_index(&the_index, &shm_index);
+	if (the_index.split_index &&
+	    the_index.split_index->base)
+		share_index(the_index.split_index->base, &shm_sharedindex);
+	discard_index(&the_index);
+}
+
+static unsigned long next;
+static int serve_cache_loop(int fd)
+{
+	struct pollfd pfd;
+	unsigned long now = time(NULL);
+
+	if (now > next)
+		return 0;
+
+	pfd.fd = fd;
+	pfd.events = POLLIN;
+	if (poll(&pfd, 1, 1000 * (next - now)) < 0) {
+		if (errno != EINTR)
+			die_errno("poll failed");
+		return 1;
+	}
+
+	if (pfd.revents & POLLIN) {
+		int client = accept(fd, NULL, NULL);
+		if (client < 0) {
+			warning("accept failed: %s", strerror(errno));
+			return 1;
+		}
+		refresh();
+		close(client);
+		next = now + 600;
+	}
+	return 1;
+}
+
+static void serve_cache(const char *socket_path)
+{
+	int fd;
+
+	fd = unix_stream_listen(socket_path);
+	if (fd < 0)
+		die_errno("unable to bind to '%s'", socket_path);
+
+	refresh();
+
+	printf("ok\n");
+	fclose(stdout);
+
+	next = time(NULL) + 600;
+	while (serve_cache_loop(fd))
+		; /* nothing */
+
+	close(fd);
+	unlink(socket_path);
+}
+
+static void check_socket_directory(const char *path)
+{
+	struct stat st;
+	char *path_copy = xstrdup(path);
+	char *dir = dirname(path_copy);
+
+	if (!stat(dir, &st)) {
+		free(path_copy);
+		return;
+	}
+
+	/*
+	 * We must be sure to create the directory with the correct mode,
+	 * not just chmod it after the fact; otherwise, there is a race
+	 * condition in which somebody can chdir to it, sleep, then try to open
+	 * our protected socket.
+	 */
+	if (safe_create_leading_directories_const(dir) < 0)
+		die_errno("unable to create directories for '%s'", dir);
+	if (mkdir(dir, 0700) < 0)
+		die_errno("unable to mkdir '%s'", dir);
+	free(path_copy);
+}
+
+int main(int argc, const char **argv)
+{
+	setup_git_directory();
+	socket_path = git_pathdup("daemon/index");
+	check_socket_directory(socket_path);
+	atexit(cleanup_socket);
+	sigchain_push_common(cleanup_socket_on_signal);
+	serve_cache(socket_path);
+	return 0;
+}
-- 
1.9.1.346.ga2b5940
Previous: Nguyễn Thái Ngọc DuyNext: Erik Faye-Lund
Message 47 of 75 in “Split index mode for very large indexes”
  1. 00/32 Split index mode for very large indexesNguyễn Thái Ngọc Duy, Apr 28, 2014
  2. 01/32 ewah: fix constness of ewah_read_mmapNguyễn Thái Ngọc Duy, Apr 28, 2014
  3. 02/32 ewah: delete unused ewah_read_mmap_native declarationNguyễn Thái Ngọc Duy, Apr 28, 2014
  4. 03/32 sequencer: do not update/refresh index if the lock cannot be heldNguyễn Thái Ngọc Duy, Apr 28, 2014
  5. 04/32 read-cache: new API write_locked_index instead of write_index/write_cacheNguyễn Thái Ngọc Duy, Apr 28, 2014
  6. 05/32 read-cache: relocate and unexport commit_locked_index()Nguyễn Thái Ngọc Duy, Apr 28, 2014
  7. 06/32 read-cache: store in-memory flags in the first 12 bits of ce_flagsNguyễn Thái Ngọc Duy, Apr 28, 2014
  8. 07/32 read-cache: be strict about "changed" in remove_marked_cache_entries()Nguyễn Thái Ngọc Duy, Apr 28, 2014
  9. 08/32 read-cache: be specific what part of the index has changedNguyễn Thái Ngọc Duy, Apr 28, 2014
  10. 09/32 update-index: be specific what part of the index has changedNguyễn Thái Ngọc Duy, Apr 28, 2014
  11. 10/32 resolve-undo: be specific what part of the index has changedNguyễn Thái Ngọc Duy, Apr 28, 2014
  12. 11/32 unpack-trees: be specific what part of the index has changedNguyễn Thái Ngọc Duy, Apr 28, 2014
  13. 12/32 cache-tree: mark istate->cache_changed on cache tree invalidationNguyễn Thái Ngọc Duy, Apr 28, 2014
  14. 13/32 cache-tree: mark istate->cache_changed on cache tree updateNguyễn Thái Ngọc Duy, Apr 28, 2014
  15. 14/32 cache-tree: mark istate->cache_changed on prime_cache_tree()Nguyễn Thái Ngọc Duy, Apr 28, 2014
  16. 15/32 entry.c: update cache_changed if refresh_cache is set in checkout_entry()Nguyễn Thái Ngọc Duy, Apr 28, 2014
  17. 16/32 read-cache: save index SHA-1 after readingNguyễn Thái Ngọc Duy, Apr 28, 2014
  18. 17/32 read-cache: split-index modeNguyễn Thái Ngọc Duy, Apr 28, 2014
  19. Junio C HamanoApr 28, 2014
  20. Duy NguyenApr 29, 2014
  21. Junio C HamanoApr 29, 2014
  22. Duy NguyenApr 29, 2014
  23. Junio C HamanoApr 30, 2014
  24. 18/32 read-cache: mark new entries for split indexNguyễn Thái Ngọc Duy, Apr 28, 2014
  25. Eric SunshineApr 30, 2014
  26. 19/32 read-cache: save deleted entries in split indexNguyễn Thái Ngọc Duy, Apr 28, 2014
  27. 20/32 read-cache: mark updated entries for split indexNguyễn Thái Ngọc Duy, Apr 28, 2014
  28. 21/32 split-index: the writing partNguyễn Thái Ngọc Duy, Apr 28, 2014
  29. 22/32 split-index: the reading partNguyễn Thái Ngọc Duy, Apr 28, 2014
  30. 23/32 split-index: do not invalidate cache-tree at read timeNguyễn Thái Ngọc Duy, Apr 28, 2014
  31. 24/32 split-index: strip pathname of on-disk replaced entriesNguyễn Thái Ngọc Duy, Apr 28, 2014
  32. Junio C HamanoApr 29, 2014
  33. 25/32 update-index: new options to enable/disable split index modeNguyễn Thái Ngọc Duy, Apr 28, 2014
  34. 26/32 update-index --split-index: do not split if $GIT_DIR is read onlyNguyễn Thái Ngọc Duy, Apr 28, 2014
  35. 27/32 rev-parse: add --shared-index-path to get shared index pathNguyễn Thái Ngọc Duy, Apr 28, 2014
  36. 28/32 read-tree: force split-index mode off on --index-outputNguyễn Thái Ngọc Duy, Apr 28, 2014
  37. 29/32 read-tree: note about dropping split-index mode or index versionNguyễn Thái Ngọc Duy, Apr 28, 2014
  38. 30/32 read-cache: force split index mode with GIT_TEST_SPLIT_INDEXNguyễn Thái Ngọc Duy, Apr 28, 2014
  39. 31/32 t2104: make sure split index mode is off for the version testNguyễn Thái Ngọc Duy, Apr 28, 2014
  40. 32/32 t1700: new tests for split-index modeNguyễn Thái Ngọc Duy, Apr 28, 2014
  41. Shawn PearceApr 28, 2014
  42. Duy NguyenApr 29, 2014
  43. Duy NguyenMay 9, 2014
  44. Junio C HamanoMay 9, 2014
  45. 0/8 Speed up cache loading timeNguyễn Thái Ngọc Duy, May 13, 2014
  46. 1/8 read-cache: allow to keep mmap'd memory after readingNguyễn Thái Ngọc Duy, May 13, 2014
  47. 2/3 Add read-cache--daemonNguyễn Thái Ngọc Duy, May 13, 2014
  48. Erik Faye-LundMay 13, 2014
  49. Duy NguyenMay 13, 2014
  50. Duy NguyenMay 13, 2014
  51. Erik Faye-LundMay 13, 2014
  52. Duy NguyenMay 13, 2014
  53. Erik Faye-LundMay 13, 2014
  54. Duy NguyenMay 13, 2014
  55. Erik Faye-LundMay 13, 2014
  56. 2/8 unix-socket: stub impl. for platforms with no unix socket supportNguyễn Thái Ngọc Duy, May 13, 2014
  57. Erik Faye-LundMay 13, 2014
  58. Erik Faye-LundMay 13, 2014
  59. 3/8 daemonize: set a flag before exiting the main processNguyễn Thái Ngọc Duy, May 13, 2014
  60. 3/3 read-cache: try index data from shared memoryNguyễn Thái Ngọc Duy, May 13, 2014
  61. 4/8 Add read-cache--daemon for caching index and related stuffNguyễn Thái Ngọc Duy, May 13, 2014
  62. Erik Faye-LundMay 13, 2014
  63. 5/8 read-cache: try index data from shared memoryNguyễn Thái Ngọc Duy, May 13, 2014
  64. Erik Faye-LundMay 13, 2014
  65. 6/8 read-cache--daemon: do not read index from shared memoryNguyễn Thái Ngọc Duy, May 13, 2014
  66. 7/8 read-cache: skip verifying trailing SHA-1 on cached indexNguyễn Thái Ngọc Duy, May 13, 2014
  67. 8/8 read-cache: inform the daemon that the index has been updatedNguyễn Thái Ngọc Duy, May 13, 2014
  68. Erik Faye-LundMay 13, 2014
  69. David TurnerMay 22, 2014
  70. Stefan BellerMay 13, 2014
  71. Duy NguyenMay 13, 2014
  72. 9/8 even faster loading time with index version 254Nguyễn Thái Ngọc Duy, May 13, 2014
  73. Junio C HamanoApr 28, 2014
  74. Richard HansenApr 30, 2014
  75. Duy NguyenMay 1, 2014

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.