git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v5 7/7] fsmonitor: add a performance test

From
Ben Peart <peartben@gmail.com>
Date
Jun 10, 2017, 13:40 UTC
Message-ID
<20170610134026.104552-8-benpeart@microsoft.com>
In-Reply-To
<20170610134026.104552-1-benpeart@microsoft.com>

Add a test utility (test-drop-caches) that enables dropping the file system cache on Windows.

Add a perf test (p7519-fsmonitor.sh) for fsmonitor.
Signed-off-by: Ben Peart <benpeart@microsoft.com>
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
---
 Makefile                    |   1 +
 t/helper/test-drop-caches.c | 107 +++++++++++++++++++++++++++++
 t/perf/p7519-fsmonitor.sh   | 161 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 269 insertions(+)
 create mode 100644 t/helper/test-drop-caches.c
 create mode 100755 t/perf/p7519-fsmonitor.sh
diff --git a/Makefile b/Makefile
index 992dd58801..893947839f 100644
--- a/Makefile
+++ b/Makefile
@@ -648,6 +648,7 @@ TEST_PROGRAMS_NEED_X += test-subprocess
 TEST_PROGRAMS_NEED_X += test-svn-fe
 TEST_PROGRAMS_NEED_X += test-urlmatch-normalization
 TEST_PROGRAMS_NEED_X += test-wildmatch
+TEST_PROGRAMS_NEED_X += test-drop-caches
 
 TEST_PROGRAMS = $(patsubst %,t/helper/%$X,$(TEST_PROGRAMS_NEED_X))
 
diff --git a/t/helper/test-drop-caches.c b/t/helper/test-drop-caches.c
new file mode 100644
index 0000000000..80830d920b
--- /dev/null
+++ b/t/helper/test-drop-caches.c
@@ -0,0 +1,107 @@
+#include "git-compat-util.h"
+#include <stdio.h>
+
+typedef DWORD NTSTATUS;
+
+#ifdef GIT_WINDOWS_NATIVE
+#include <tchar.h>
+
+#define STATUS_SUCCESS			(0x00000000L)
+#define STATUS_PRIVILEGE_NOT_HELD	(0xC0000061L)
+
+typedef enum _SYSTEM_INFORMATION_CLASS {
+	SystemMemoryListInformation = 80, // 80, q: SYSTEM_MEMORY_LIST_INFORMATION; s: SYSTEM_MEMORY_LIST_COMMAND (requires SeProfileSingleProcessPrivilege)
+} SYSTEM_INFORMATION_CLASS;
+
+// private
+typedef enum _SYSTEM_MEMORY_LIST_COMMAND
+{
+	MemoryCaptureAccessedBits,
+	MemoryCaptureAndResetAccessedBits,
+	MemoryEmptyWorkingSets,
+	MemoryFlushModifiedList,
+	MemoryPurgeStandbyList,
+	MemoryPurgeLowPriorityStandbyList,
+	MemoryCommandMax
+} SYSTEM_MEMORY_LIST_COMMAND;
+
+BOOL GetPrivilege(HANDLE TokenHandle, LPCSTR lpName, int flags)
+{
+	BOOL bResult;
+	DWORD dwBufferLength;
+	LUID luid;
+	TOKEN_PRIVILEGES tpPreviousState;
+	TOKEN_PRIVILEGES tpNewState;
+
+	dwBufferLength = 16;
+	bResult = LookupPrivilegeValueA(0, lpName, &luid);
+	if (bResult)
+	{
+		tpNewState.PrivilegeCount = 1;
+		tpNewState.Privileges[0].Luid = luid;
+		tpNewState.Privileges[0].Attributes = 0;
+		bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpNewState, (DWORD)((LPBYTE)&(tpNewState.Privileges[1]) - (LPBYTE)&tpNewState), &tpPreviousState, &dwBufferLength);
+		if (bResult)
+		{
+			tpPreviousState.PrivilegeCount = 1;
+			tpPreviousState.Privileges[0].Luid = luid;
+			tpPreviousState.Privileges[0].Attributes = flags != 0 ? 2 : 0;
+			bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpPreviousState, dwBufferLength, 0, 0);
+		}
+	}
+	return bResult;
+}
+#endif
+
+int cmd_main(int argc, const char **argv)
+{
+	NTSTATUS status = 1;
+#ifdef GIT_WINDOWS_NATIVE
+	HANDLE hProcess = GetCurrentProcess();
+	HANDLE hToken;
+	if (!OpenProcessToken(hProcess, TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken))
+	{
+		_ftprintf(stderr, _T("Can't open current process token\n"));
+		return 1;
+	}
+
+	if (!GetPrivilege(hToken, "SeProfileSingleProcessPrivilege", 1))
+	{
+		_ftprintf(stderr, _T("Can't get SeProfileSingleProcessPrivilege\n"));
+		return 1;
+	}
+
+	CloseHandle(hToken);
+
+	HMODULE ntdll = LoadLibrary(_T("ntdll.dll"));
+	if (!ntdll)
+	{
+		_ftprintf(stderr, _T("Can't load ntdll.dll, wrong Windows version?\n"));
+		return 1;
+	}
+
+	NTSTATUS(WINAPI *NtSetSystemInformation)(INT, PVOID, ULONG) = (NTSTATUS(WINAPI *)(INT, PVOID, ULONG))GetProcAddress(ntdll, "NtSetSystemInformation");
+	if (!NtSetSystemInformation)
+	{
+		_ftprintf(stderr, _T("Can't get function addresses, wrong Windows version?\n"));
+		return 1;
+	}
+
+	SYSTEM_MEMORY_LIST_COMMAND command = MemoryPurgeStandbyList;
+	status = NtSetSystemInformation(
+		SystemMemoryListInformation,
+		&command,
+		sizeof(SYSTEM_MEMORY_LIST_COMMAND)
+	);
+	if (status == STATUS_PRIVILEGE_NOT_HELD)
+	{
+		_ftprintf(stderr, _T("Insufficient privileges to execute the memory list command"));
+	}
+	else if (status != STATUS_SUCCESS)
+	{
+		_ftprintf(stderr, _T("Unable to execute the memory list command %lX"), status);
+	}
+#endif
+
+	return status;
+}
diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh
new file mode 100755
index 0000000000..e41905cb9b
--- /dev/null
+++ b/t/perf/p7519-fsmonitor.sh
@@ -0,0 +1,161 @@
+#!/bin/sh
+
+test_description="Test core.fsmonitor"
+
+. ./perf-lib.sh
+
+# This has to be run with GIT_PERF_REPEAT_COUNT=1 to generate valid results.
+# Otherwise the caching that happens for the nth run will negate the validity
+# of the comparisons.
+if [ "$GIT_PERF_REPEAT_COUNT" -ne 1 ]
+then
+	echo "warning: This test must be run with GIT_PERF_REPEAT_COUNT=1 to generate valid results." >&2
+	echo "warning: Setting GIT_PERF_REPEAT_COUNT=1" >&2
+	GIT_PERF_REPEAT_COUNT=1
+fi
+
+test_perf_large_repo
+test_checkout_worktree
+
+# Convert unix style paths to what Watchman expects
+case "$(uname -s)" in
+MINGW*|MSYS_NT*)
+  GIT_WORK_TREE="$(cygpath -aw "$PWD" | sed 's,\\,/,g')"
+  ;;
+*)
+  GIT_WORK_TREE="$PWD"
+  ;;
+esac
+
+# The big win for using fsmonitor is the elimination of the need to scan
+# the working directory looking for changed files and untracked files. If
+# the file information is all cached in RAM, the benefits are reduced.
+
+flush_disk_cache () {
+	case "$(uname -s)" in
+	MINGW*|MSYS_NT*)
+	  sync && test-drop-caches
+	  ;;
+	*)
+	  sudo sync && echo 3 | sudo tee /proc/sys/vm/drop_caches
+	  ;;
+	esac
+
+}
+
+test_lazy_prereq UNTRACKED_CACHE '
+	{ git update-index --test-untracked-cache; ret=$?; } &&
+	test $ret -ne 1
+'
+
+test_expect_success "setup" '
+	# Maybe set untrackedCache & splitIndex depending on the environment
+	if test -n "$GIT_PERF_7519_UNTRACKED_CACHE"
+	then
+		git config core.untrackedCache "$GIT_PERF_7519_UNTRACKED_CACHE"
+	else
+		if test_have_prereq UNTRACKED_CACHE
+		then
+			git config core.untrackedCache true
+		else
+			git config core.untrackedCache false
+		fi
+	fi &&
+
+	if test -n "$GIT_PERF_7519_SPLIT_INDEX"
+	then
+		git config core.splitIndex "$GIT_PERF_7519_SPLIT_INDEX"
+	fi &&
+
+	# Hook scaffolding
+	mkdir .git/hooks &&
+	cp ../../../templates/hooks--query-fsmonitor.sample .git/hooks/query-fsmonitor &&
+
+	# have Watchman monitor the test folder
+	watchman watch "$GIT_WORK_TREE" &&
+	watchman watch-list | grep -q -F "$GIT_WORK_TREE"
+'
+
+# Worst case without fsmonitor
+test_expect_success "clear fs cache" '
+	git config core.fsmonitor false &&
+	flush_disk_cache
+'
+test_perf "status (fsmonitor=false, cold fs cache)" '
+	git status
+'
+
+# Best case without fsmonitor
+test_perf "status (fsmonitor=false, warm fs cache)" '
+	git status
+'
+
+# Let's see if -uno & -uall make any difference
+test_expect_success "clear fs cache" '
+	flush_disk_cache
+'
+test_perf "status -uno (fsmonitor=false, cold fs cache)" '
+	git status -uno
+'
+
+test_expect_success "clear fs cache" '
+	flush_disk_cache
+'
+test_perf "status -uall (fsmonitor=false, cold fs cache)" '
+	git status -uall
+'
+
+# The first run with core.fsmonitor=true has to do a normal scan and write
+# out the index extension.
+test_expect_success "populate extension" '
+	# core.preloadIndex defeats the benefits of core.fsMonitor as it
+	# calls lstat for the index entries. Turn it off as _not_ doing
+	# the work is faster than doing the work across multiple threads.
+	git config core.fsmonitor true &&
+	git config core.preloadIndex false &&
+	git status
+'
+
+# Worst case with fsmonitor
+test_expect_success "shutdown fsmonitor, clear fs cache" '
+	watchman shutdown-server &&
+	flush_disk_cache
+'
+test_perf "status (fsmonitor=true, cold fs cache, cold fsmonitor)" '
+	git status
+'
+
+# Best case with fsmonitor
+test_perf "status (fsmonitor=true, warm fs cache, warm fsmonitor)" '
+	git status
+'
+
+# Best improved with fsmonitor (compare to worst case without fsmonitor)
+test_expect_success "clear fs cache" '
+	flush_disk_cache
+'
+test_perf "status (fsmonitor=true, cold fs cache, warm fsmonitor)" '
+	git status
+'
+
+# Let's see if -uno & -uall make any difference
+test_expect_success "clear fs cache" '
+	flush_disk_cache
+'
+test_perf "status -uno (fsmonitor=true, cold fs cache)" '
+	git status -uno
+'
+
+test_expect_success "clear fs cache" '
+	flush_disk_cache
+'
+test_perf "status -uall (fsmonitor=true, cold fs cache)" '
+	git status -uall
+'
+
+test_expect_success "cleanup" '
+	watchman watch-del "$GIT_WORK_TREE" &&
+	watchman shutdown-server
+'
+
+test_done

-- 
2.13.0
Previous: Ben PeartNext: Ben Peart
Message 11 of 137 in “Fast git status via a file system watcher”
  1. 0/7 Fast git status via a file system watcherBen Peart, Jun 10, 2017
  2. 2/7 dir: make lookup_untracked() available outside of dir.cBen Peart, Jun 10, 2017
  3. 1/7 bswap: add 64 bit endianness helper get_be64Ben Peart, Jun 10, 2017
  4. 4/7 fsmonitor: add test cases for fsmonitor extensionBen Peart, Jun 10, 2017
  5. Christian CouderJun 27, 2017
  6. Ben PeartJul 7, 2017
  7. 3/7 fsmonitor: teach git to optionally utilize a file system monitor to speed up detecting new or changed files.Ben Peart, Jun 10, 2017
  8. Christian CouderJun 27, 2017
  9. Ben PeartJul 3, 2017
  10. 6/7 fsmonitor: add a sample query-fsmonitor hook script for WatchmanBen Peart, Jun 10, 2017
  11. 7/7 fsmonitor: add a performance testBen Peart, Jun 10, 2017
  12. Ben PeartJun 10, 2017
  13. Junio C HamanoJun 12, 2017
  14. Ben PeartJun 14, 2017
  15. Junio C HamanoJun 14, 2017
  16. Ben PeartJul 7, 2017
  17. Junio C HamanoJul 7, 2017
  18. Ben PeartJul 7, 2017
  19. David TurnerJul 7, 2017
  20. Christian CouderJul 8, 2017
  21. 5/7 fsmonitor: add documentation for the fsmonitor extension.Ben Peart, Jun 10, 2017
  22. Christian CouderJun 28, 2017
  23. Ben PeartJul 10, 2017
  24. Ben PeartJul 10, 2017
  25. 00/12 Fast git status via a file system watcherBen Peart, Sep 15, 2017
  26. 01/12 bswap: add 64 bit endianness helper get_be64Ben Peart, Sep 15, 2017
  27. 04/12 fsmonitor: teach git to optionally utilize a file system monitor to speed up detecting new or changed files.Ben Peart, Sep 15, 2017
  28. David TurnerSep 15, 2017
  29. Ben PeartSep 18, 2017
  30. David TurnerSep 18, 2017
  31. Ben PeartSep 18, 2017
  32. 05/12 fsmonitor: add documentation for the fsmonitor extension.Ben Peart, Sep 15, 2017
  33. David TurnerSep 15, 2017
  34. Ben PeartSep 18, 2017
  35. Junio C HamanoSep 17, 2017
  36. Ben PeartSep 18, 2017
  37. 07/12 update-index: add fsmonitor support to update-indexBen Peart, Sep 15, 2017
  38. 06/12 ls-files: Add support in ls-files to display the fsmonitor valid bitBen Peart, Sep 15, 2017
  39. David TurnerSep 15, 2017
  40. 10/12 fsmonitor: add test cases for fsmonitor extensionBen Peart, Sep 15, 2017
  41. David TurnerSep 15, 2017
  42. David TurnerSep 19, 2017
  43. Ben PeartSep 19, 2017
  44. Torsten BögershausenSep 16, 2017
  45. 1/1 test-lint: echo -e (or -E) is not portabletboegi@web.de, Sep 17, 2017
  46. Jonathan NiederSep 19, 2017
  47. Torsten BögershausenSep 20, 2017
  48. Junio C HamanoSep 22, 2017
  49. Ben PeartSep 18, 2017
  50. Junio C HamanoSep 17, 2017
  51. Ben PeartSep 18, 2017
  52. Jonathan NiederSep 19, 2017
  53. 12/12 fsmonitor: add a performance testBen Peart, Sep 15, 2017
  54. David TurnerSep 15, 2017
  55. Johannes SchindelinSep 18, 2017
  56. Ben PeartSep 18, 2017
  57. Johannes SchindelinSep 19, 2017
  58. 11/12 fsmonitor: add a sample integration script for WatchmanBen Peart, Sep 15, 2017
  59. 08/12 fsmonitor: add a test tool to dump the index extensionBen Peart, Sep 15, 2017
  60. Junio C HamanoSep 17, 2017
  61. Ben PeartSep 18, 2017
  62. Torsten BögershausenSep 18, 2017
  63. Ben PeartSep 18, 2017
  64. Torsten BögershausenSep 19, 2017
  65. Ben PeartSep 19, 2017
  66. 09/12 split-index: disable the fsmonitor extension when running the split index testBen Peart, Sep 15, 2017
  67. Jonathan NiederSep 19, 2017
  68. Ben PeartSep 20, 2017
  69. Jonathan NiederSep 20, 2017
  70. Ben PeartSep 21, 2017
  71. 03/12 update-index: add a new --force-write-index optionBen Peart, Sep 15, 2017
  72. 02/12 preload-index: add override to enable testing preload-indexBen Peart, Sep 15, 2017
  73. 00/12 Fast git status via a file system watcherBen Peart, Sep 19, 2017
  74. 01/12 bswap: add 64 bit endianness helper get_be64Ben Peart, Sep 19, 2017
  75. 02/12 preload-index: add override to enable testing preload-indexBen Peart, Sep 19, 2017
  76. Stefan BellerSep 20, 2017
  77. Ben PeartSep 21, 2017
  78. Stefan BellerSep 21, 2017
  79. 05/12 fsmonitor: add documentation for the fsmonitor extension.Ben Peart, Sep 19, 2017
  80. Martin ÅgrenSep 20, 2017
  81. Ben PeartSep 20, 2017
  82. Martin ÅgrenSep 20, 2017
  83. 04/12 fsmonitor: teach git to optionally utilize a file system monitor to speed up detecting new or changed files.Ben Peart, Sep 19, 2017
  84. Junio C HamanoSep 20, 2017
  85. Ben PeartSep 20, 2017
  86. Junio C HamanoSep 21, 2017
  87. Ben PeartSep 21, 2017
  88. Ben PeartSep 21, 2017
  89. Junio C HamanoSep 22, 2017
  90. Junio C HamanoSep 20, 2017
  91. Ben PeartSep 20, 2017
  92. 08/12 fsmonitor: add a test tool to dump the index extensionBen Peart, Sep 19, 2017
  93. 09/12 split-index: disable the fsmonitor extension when running the split index testBen Peart, Sep 19, 2017
  94. 07/12 update-index: add fsmonitor support to update-indexBen Peart, Sep 19, 2017
  95. 10/12 fsmonitor: add test cases for fsmonitor extensionBen Peart, Sep 19, 2017
  96. 03/12 update-index: add a new --force-write-index optionBen Peart, Sep 19, 2017
  97. Junio C HamanoSep 20, 2017
  98. Ben PeartSep 20, 2017
  99. Junio C HamanoSep 21, 2017
  100. Ben PeartSep 21, 2017
  101. Junio C HamanoSep 21, 2017
  102. Junio C HamanoSep 21, 2017
  103. 12/12 fsmonitor: add a performance testBen Peart, Sep 19, 2017
  104. 11/12 fsmonitor: add a sample integration script for WatchmanBen Peart, Sep 19, 2017
  105. 06/12 ls-files: Add support in ls-files to display the fsmonitor valid bitBen Peart, Sep 19, 2017
  106. David TurnerSep 19, 2017
  107. Ben PeartSep 19, 2017
  108. David TurnerSep 19, 2017
  109. Ben PeartSep 19, 2017
  110. 00/12 Fast git status via a file system watcherBen Peart, Sep 22, 2017
  111. 02/12 preload-index: add override to enable testing preload-indexBen Peart, Sep 22, 2017
  112. 01/12 bswap: add 64 bit endianness helper get_be64Ben Peart, Sep 22, 2017
  113. Martin ÅgrenSep 22, 2017
  114. Ben PeartSep 23, 2017
  115. Jeff KingSep 24, 2017
  116. Junio C HamanoSep 24, 2017
  117. 03/12 update-index: add a new --force-write-index optionBen Peart, Sep 22, 2017
  118. 11/12 fsmonitor: add a sample integration script for WatchmanBen Peart, Sep 22, 2017
  119. 04/12 fsmonitor: teach git to optionally utilize a file system monitor to speed up detecting new or changed files.Ben Peart, Sep 22, 2017
  120. 10/12 fsmonitor: add test cases for fsmonitor extensionBen Peart, Sep 22, 2017
  121. 09/12 split-index: disable the fsmonitor extension when running the split index testBen Peart, Sep 22, 2017
  122. 12/12 fsmonitor: add a performance testBen Peart, Sep 22, 2017
  123. 06/12 ls-files: Add support in ls-files to display the fsmonitor valid bitBen Peart, Sep 22, 2017
  124. 05/12 fsmonitor: add documentation for the fsmonitor extension.Ben Peart, Sep 22, 2017
  125. 07/12 update-index: add fsmonitor support to update-indexBen Peart, Sep 22, 2017
  126. 08/12 fsmonitor: add a test tool to dump the index extensionBen Peart, Sep 22, 2017
  127. Martin ÅgrenSep 22, 2017
  128. Ben PeartSep 23, 2017
  129. Junio C HamanoSep 24, 2017
  130. Junio C HamanoSep 29, 2017
  131. Ben PeartSep 29, 2017
  132. Junio C HamanoOct 1, 2017
  133. Ben PeartOct 3, 2017
  134. Junio C HamanoOct 4, 2017
  135. Alex VandiverOct 4, 2017
  136. Ben PeartOct 4, 2017
  137. Ben PeartOct 4, 2017

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.