[PATCH v4 0/6] Fast git status via a file system watcher
Changes from V3 include:
- update test script based on feedback
- update template hook proc with better post-processing code and make
it executable
Ben Peart (6):
bswap: add 64 bit endianness helper get_be64
dir: make lookup_untracked() available outside of dir.c
fsmonitor: teach git to optionally utilize a file system monitor to
speed up detecting new or changed files.
fsmonitor: add test cases for fsmonitor extension
fsmonitor: add documentation for the fsmonitor extension.
fsmonitor: add a sample query-fsmonitor hook script for Watchman Documentation/config.txt | 7 +
Documentation/githooks.txt | 23 +++
Documentation/technical/index-format.txt | 19 +++
Makefile | 1 +
builtin/update-index.c | 1 +
cache.h | 5 +
compat/bswap.h | 4 +
config.c | 5 +
dir.c | 16 ++-
dir.h | 5 +
entry.c | 1 +
environment.c | 1 +
fsmonitor.c | 238 +++++++++++++++++++++++++++++++
fsmonitor.h | 9 ++
read-cache.c | 28 +++-
t/t7519-status-fsmonitor.sh | 173 ++++++++++++++++++++++
templates/hooks--query-fsmonitor.sample | 60 ++++++++
unpack-trees.c | 1 +
18 files changed, 594 insertions(+), 3 deletions(-)
create mode 100644 fsmonitor.c
create mode 100644 fsmonitor.h
create mode 100755 t/t7519-status-fsmonitor.sh
create mode 100755 templates/hooks--query-fsmonitor.sample
Interdiff (v3..v4):
diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh
index 395db46d55..458eabe6dc 100755
--- a/t/t7519-status-fsmonitor.sh
+++ b/t/t7519-status-fsmonitor.sh
@@ -5,48 +5,46 @@ test_description='git status with file system watcher'
. ./test-lib.sh
clean_repo () {
- git reset --hard HEAD
- git clean -fd
+ git reset --hard HEAD &&
+ git clean -fd &&
rm -f marker
}
dirty_repo () {
- : >untracked
- : >dir1/untracked
- : >dir2/untracked
- echo 1 >modified
- echo 2 >dir1/modified
- echo 3 >dir2/modified
- echo 4 >new
- echo 5 >dir1/new
- echo 6 >dir2/new
- git add new
- git add dir1/new
+ : >untracked &&
+ : >dir1/untracked &&
+ : >dir2/untracked &&
+ echo 1 >modified &&
+ echo 2 >dir1/modified &&
+ echo 3 >dir2/modified &&
+ echo 4 >new &&
+ echo 5 >dir1/new &&
+ echo 6 >dir2/new &&
+ git add new &&
+ git add dir1/new &&
git add dir2/new
}
# The test query-fsmonitor hook proc will output a marker file we can use to
# ensure the hook was actually used to generate the correct results.
+# fsmonitor works correctly with or without the untracked cache
+# but if it is available, we'll turn it on to ensure we test that
+# codepath as well.
+
+test_lazy_prereq UNTRACKED_CACHE '
+ { git update-index --test-untracked-cache; ret=$?; } &&
+ test $ret -ne 1
+'
+
+if test_have_prereq UNTRACKED_CACHE; then
+ git config core.untrackedcache true
+else
+ git config core.untrackedcache false
+fi
+
test_expect_success 'setup' '
mkdir -p .git/hooks &&
- write_script .git/hooks/query-fsmonitor<<-\EOF &&
- if [ $1 -ne 1 ]
- then
- echo -e "Unsupported query-fsmonitor hook version.\n" >&2
- exit 1;
- fi
- : >marker
- printf "untracked\0"
- printf "dir1/untracked\0"
- printf "dir2/untracked\0"
- printf "modified\0"
- printf "dir1/modified\0"
- printf "dir2/modified\0"
- printf "new\0""
- printf "dir1/new\0"
- printf "dir2/new\0"
- EOF
: >tracked &&
: >modified &&
mkdir dir1 &&
@@ -58,55 +56,19 @@ test_expect_success 'setup' '
git add . &&
test_tick &&
git commit -m initial &&
- dirty_repo
-'
-
-cat >.gitignore <<\EOF
-.gitignore
-expect*
-output*
-marker*
-EOF
-
-# Status is well tested elsewhere so we'll just ensure that the results are
-# the same when using core.fsmonitor. First call after turning on the option
-# does a complete scan so need to do two calls to ensure we test the new
-# codepath.
-
-test_expect_success 'status with core.untrackedcache true' '
- git config core.fsmonitor true &&
- git config core.untrackedcache true &&
- git -c core.fsmonitor=false -c core.untrackedcache=true status >expect &&
- clean_repo &&
- git status &&
- test_path_is_missing marker &&
- dirty_repo &&
- git status >output &&
- test_path_is_file marker &&
- test_i18ncmp expect output
-'
-
-
-test_expect_success 'status with core.untrackedcache false' '
git config core.fsmonitor true &&
- git config core.untrackedcache false &&
- git -c core.fsmonitor=false -c core.untrackedcache=false status >expect &&
- clean_repo &&
- git status &&
- test_path_is_missing marker &&
- dirty_repo &&
- git status >output &&
- test_path_is_file marker &&
- test_i18ncmp expect output
+ cat >.gitignore <<-\EOF
+ .gitignore
+ expect*
+ output*
+ marker*
+ EOF
'
# Ensure commands that call refresh_index() to move the index back in time
# properly invalidate the fsmonitor cache
test_expect_success 'refresh_index() invalidates fsmonitor cache' '
- git config core.fsmonitor true &&
- git config core.untrackedcache true &&
- clean_repo &&
git status &&
test_path_is_missing marker &&
dirty_repo &&
@@ -118,6 +80,7 @@ test_expect_success 'refresh_index() invalidates fsmonitor cache' '
git status &&
test_path_is_file marker &&
git reset HEAD~1 &&
+ rm -f marker &&
git status >output &&
test_path_is_file marker &&
git -c core.fsmonitor=false status >expect &&
@@ -129,9 +92,7 @@ test_expect_success 'refresh_index() invalidates fsmonitor cache' '
# extensions exist other than 'TREE' so do a "git status" to get the extension
# written before testing the results.
-test_expect_success 'status doesnt detect unreported modifications' '
- git config core.fsmonitor true &&
- git config core.untrackedcache true &&
+test_expect_success "status doesn't detect unreported modifications" '
write_script .git/hooks/query-fsmonitor<<-\EOF &&
:>marker
EOF
@@ -146,13 +107,67 @@ test_expect_success 'status doesnt detect unreported modifications' '
test_i18ngrep ! "Untracked files:" output &&
write_script .git/hooks/query-fsmonitor<<-\EOF &&
:>marker
- printf "untracked%s\0"
+ printf "untracked\0"
printf "dir1/modified\0"
EOF
+ rm -f marker &&
git status >output &&
test_path_is_file marker &&
test_i18ngrep "Changes not staged for commit:" output &&
test_i18ngrep "Untracked files:" output
'
+# Status is well tested elsewhere so we'll just ensure that the results are
+# the same when using core.fsmonitor. First call after turning on the option
+# does a complete scan so we need to do two calls to ensure we test the new
+# codepath.
+
+test_expect_success 'status with core.untrackedcache false' '
+ git config core.untrackedcache false &&
+ write_script .git/hooks/query-fsmonitor<<-\EOF &&
+ if [ $1 -ne 1 ]
+ then
+ echo -e "Unsupported query-fsmonitor hook version.\n" >&2
+ exit 1;
+ fi
+ : >marker
+ printf "untracked\0"
+ printf "dir1/untracked\0"
+ printf "dir2/untracked\0"
+ printf "modified\0"
+ printf "dir1/modified\0"
+ printf "dir2/modified\0"
+ printf "new\0""
+ printf "dir1/new\0"
+ printf "dir2/new\0"
+ EOF
+ clean_repo &&
+ dirty_repo &&
+ git -c core.fsmonitor=false status >expect &&
+ clean_repo &&
+ git status &&
+ test_path_is_missing marker &&
+ dirty_repo &&
+ git status >output &&
+ test_path_is_file marker &&
+ test_i18ncmp expect output
+'
+
+if ! test_have_prereq UNTRACKED_CACHE; then
+ skip_all='This system does not support untracked cache'
+ test_done
+fi
+
+test_expect_success 'status with core.untrackedcache true' '
+ git config core.untrackedcache true &&
+ git -c core.fsmonitor=false status >expect &&
+ clean_repo &&
+ git status &&
+ test_path_is_missing marker &&
+ dirty_repo &&
+ git status >output &&
+ test_path_is_file marker &&
+ test_i18ncmp expect output
+'
+
test_done
diff --git a/templates/hooks--query-fsmonitor.sample b/templates/hooks--query-fsmonitor.sample
old mode 100644
new mode 100755
index 615f3332fa..941c4c5b57
--- a/templates/hooks--query-fsmonitor.sample
+++ b/templates/hooks--query-fsmonitor.sample
@@ -4,10 +4,10 @@
# (https://facebook.github.io/watchman/) with git to provide fast
# git status.
#
-# The hook is passed a time in nanoseconds formatted as a string and
-# outputs to stdout all files that have been modified since the given
-# time. Paths must be relative to the root of the working tree and
-# separated by a single NUL.
+# The hook is passed a version (currently 1) and a time in nanoseconds
+# formatted as a string and outputs to stdout all files that have been
+# modified since the given time. Paths must be relative to the root of
+# the working tree and separated by a single NUL.
#
# To enable this hook, rename this file to "query-fsmonitor"
@@ -33,5 +33,28 @@ esac
# Query Watchman for all the changes since the requested time
echo "[\"query\", \"$GIT_WORK_TREE\", {\"since\": $time_t, \"fields\":[\"name\"]}]" | \
-watchman -j | \
-perl -e 'use JSON::PP; my $o = JSON::PP->new->utf8->decode(join("", <>)); die "Watchman: $o->{'error'}.\nFalling back to scanning...\n" if defined($o->{"error"}); print(join("\0", @{$o->{"files"}}));'
+ watchman -j |
+ perl -0666 -e '
+ use strict;
+ use warnings;
+
+ my $stdin = <>;
+ die "Watchman: command returned no output.\nFalling back to scanning...\n" if $stdin eq "";
+ die "Watchman: command returned invalid output: $stdin\nFalling back to scanning...\n" unless $stdin =~ /^\{/;
+
+ my $json_pkg;
+ eval {
+ require JSON::XS;
+ $json_pkg = "JSON::XS";
+ 1;
+ } or do {
+ require JSON::PP;
+ $json_pkg = "JSON::PP";
+ };
+
+ my $o = $json_pkg->new->utf8->decode($stdin);
+ die "Watchman: $o->{error}.\nFalling back to scanning...\n" if $o->{error};
+
+ local $, = "\0";
+ print @{$o->{files}};
+ '
--
2.13.0.windows.1.9.gc201c67b71
Ben Peart <peartben@gmail.com>