v2, 6 partsFast git status via a file system watcher

> After sending this, I noticed that using a different compiler
> generated a couple of warnings I should fix.  I'm assuming I'll need
> another re-roll but if not, here are the changes that will be squashed
> in.
>
> Ben

> diff --git a/cache.h b/cache.h
> index 188811920c..36423c77cc 100644
> --- a/cache.h
> +++ b/cache.h
> @@ -201,6 +201,7 @@ struct cache_entry {
>  #define CE_ADDED             (1 << 19)
>
>  #define CE_HASHED            (1 << 20)
> +#define CE_FSMONITOR_DIRTY   (1 << 21)

>  #define CE_WT_REMOVE         (1 << 22) /* remove in work directory */
>  #define CE_CONFLICTED        (1 << 23)
>
>  struct split_index;
>  struct untracked_cache;
> @@ -342,6 +344,8 @@ struct index_state {
>         struct hashmap dir_hash;
>         unsigned char sha1[20];
>         struct untracked_cache *untracked;
> +       time_t fsmonitor_last_update;
> +       struct ewah_bitmap *fsmonitor_dirty_bitmap;

> diff --git a/dir.c b/dir.c
> index 1b5558fdf9..da428489e2 100644
> --- a/dir.c
> +++ b/dir.c
> @@ -1652,6 +1652,18 @@ static int valid_cached_dir(struct dir_struct *dir,
>         if (!untracked)
>                 return 0;
>
> +       refresh_by_fsmonitor(&the_index);
> +       if (dir->untracked->use_fsmonitor) {
> +               /*
> +                * With fsmonitor, we can trust the untracked cache's
> +                * valid field.
> +                */
> +               if (untracked->valid)
> +                       goto skip_stat;

> +               else
> +                       invalidate_directory(dir->untracked, untracked);
> +       }
> +
>         if (stat(path->len ? path->buf : ".", &st)) {
>                 invalidate_directory(dir->untracked, untracked);
>                 memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
> @@ -1665,6 +1677,7 @@ static int valid_cached_dir(struct dir_struct *dir,
>                 return 0;
>         }
>
> +skip_stat:
>         if (untracked->check_only != !!check_only) {
>                 invalidate_directory(dir->untracked, untracked);
>                 return 0;

> +void refresh_by_fsmonitor(struct index_state *istate)
> +{
> +       static has_run_once = FALSE;
> +       struct strbuf query_result = STRBUF_INIT;
> +       int query_success = 0;
> +       size_t bol = 0; /* beginning of line */
> +       time_t last_update;
> +       char *buf, *entry;
> +       int i;
> +
> +       if (!core_fsmonitor || has_run_once)
> +               return;
> +       has_run_once = TRUE;
> +
> +       /*
> +        * This could be racy so save the date/time now and the hook
> +        * should be inclusive to ensure we don't miss potential changes.
> +        */
> +       last_update = time(NULL);
> +
> +       /* If we have a last update time, call query-monitor for the set of changes since that time */
> +       if (istate->fsmonitor_last_update) {
> +               query_success = !query_fsmonitor(istate->fsmonitor_last_update, &query_result);
> +       }

> +       if (query_success) {
> +               /* Mark all entries returned by the monitor as dirty */
> +               buf = entry = query_result.buf;
> +               for (i = 0; i < query_result.len; i++) {
> +                       if (buf[i] != '\0')
> +                               continue;
> +                       mark_file_dirty(istate, buf + bol);
> +                       bol = i + 1;
> +               }
> +               if (bol < query_result.len)
> +                       mark_file_dirty(istate, buf + bol);
> +
> +               /* Mark all clean entries up-to-date */
> +               for (i = 0; i < istate->cache_nr; i++) {
> +                       struct cache_entry *ce = istate->cache[i];
> +                       if (ce_stage(ce) || (ce->ce_flags & CE_FSMONITOR_DIRTY))
> +                               continue;
> +                       ce_mark_uptodate(ce);
> +               }
> +
> +               /*
> +                * Now that we've marked the invalid entries in the
> +                * untracked-cache itself, we can mark the untracked cache for
> +                * fsmonitor usage.
> +                */
> +               if (istate->untracked) {
> +                       istate->untracked->use_fsmonitor = 1;
> +               }

> +       }
> +       else {
> +               /* if we can't update the cache, fall back to checking them all */
> +               for (i = 0; i < istate->cache_nr; i++)
> +                       istate->cache[i]->ce_flags |= CE_FSMONITOR_DIRTY;
> +
> +               /* mark the untracked cache as unusable for fsmonitor */
> +               if (istate->untracked)
> +                       istate->untracked->use_fsmonitor = 0;
> +       }
> +       strbuf_release(&query_result);
> +
> +       /* Now that we've updated istate, save the last_update time */
> +       istate->fsmonitor_last_update = last_update;
> +       istate->cache_changed |= FSMONITOR_CHANGED;
> +}

> diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh
> new file mode 100755
> index 0000000000..d3cffc758f
> --- /dev/null
> +++ b/t/t7519-status-fsmonitor.sh
> @@ -0,0 +1,153 @@
> +#!/bin/sh
> +
> +test_description='git status with file system watcher'
> +
> +. ./test-lib.sh
> +
> +clean_repo () {
> +	git reset --hard HEAD
> +	git clean -fd
> +	rm marker -f

> This hook script integrates the new fsmonitor capabilities of git with
> the cross platform Watchman file watching service. To use the script:
>
> Download and install Watchman from https://facebook.github.io/watchman/
> and instruct Watchman to watch your working directory for changes
> ('watchman watch-project /usr/src/git').
>
> Rename the sample integration hook from query-fsmonitor.sample to
> query-fsmonitor.
>
> Configure git to use the extension ('git config core.fsmonitor true')
> and optionally turn on the untracked cache for optimal performance
> ('git config core.untrackedcache true').

> Signed-off-by: Ben Peart <benpeart@microsoft.com>
> Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
> ---
>  templates/hooks--query-fsmonitor.sample | 27 +++++++++++++++++++++++++++
>  1 file changed, 27 insertions(+)
>  create mode 100644 templates/hooks--query-fsmonitor.sample
>
> diff --git a/templates/hooks--query-fsmonitor.sample b/templates/hooks--query-fsmonitor.sample
> new file mode 100644
> index 0000000000..4bd22f21d8
> --- /dev/null
> +++ b/templates/hooks--query-fsmonitor.sample
> @@ -0,0 +1,27 @@
> +#!/bin/sh
> +#
> +# An example hook script to integrate Watchman
> +# (https://facebook.github.io/watchman/) with git to provide fast
> +# git status.
> +#
> +# The hook is passed a time_t formatted as a string and outputs to
> +# stdout all files that have been modified since the given time.
> +# Paths must be relative to the root of the working tree and
> +# separated by a single NUL.
> +#
> +# To enable this hook, rename this file to "query-fsmonitor"
> +
> +# Convert unix style paths to escaped Windows style paths
> +case "$(uname -s)" in
> +MINGW*|MSYS_NT*)
> +  GIT_WORK_TREE="$(cygpath -aw "$PWD" | sed 's,\\,\\\\,g')"
> +  ;;
> +*)
> +  GIT_WORK_TREE="$PWD"
> +  ;;
> +esac
> +
> +# Query Watchman for all the changes since the requested time
> +echo "[\"query\", \"$GIT_WORK_TREE\", {\"since\": $1, \"fields\":[\"name\"]}]" | \
> +watchman -j | \
> +perl -e 'use JSON::PP; my $o = JSON::PP->new->utf8->decode(join("", <>)); die "Watchman: $o->{'error'}.\nFalling back to scanning...\n" if defined($o->{"error"}); print(join("\0", @{$o->{"files"}}));'

> On Thu, May 18, 2017 at 10:13 PM, Ben Peart <peartben@gmail.com> wrote:
>> This hook script integrates the new fsmonitor capabilities of git with
>> the cross platform Watchman file watching service. To use the script:
>>
>> Download and install Watchman from https://facebook.github.io/watchman/
>> and instruct Watchman to watch your working directory for changes
>> ('watchman watch-project /usr/src/git').
>>
>> Rename the sample integration hook from query-fsmonitor.sample to
>> query-fsmonitor.
>>
>> Configure git to use the extension ('git config core.fsmonitor true')
>> and optionally turn on the untracked cache for optimal performance
>> ('git config core.untrackedcache true').
>
> Ok, it looks like the untracked cache can be used, but it could work without it.
>
>> Signed-off-by: Ben Peart <benpeart@microsoft.com>
>> Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
>> ---
>>  templates/hooks--query-fsmonitor.sample | 27 +++++++++++++++++++++++++++
>>  1 file changed, 27 insertions(+)
>>  create mode 100644 templates/hooks--query-fsmonitor.sample
>>
>> diff --git a/templates/hooks--query-fsmonitor.sample b/templates/hooks--query-fsmonitor.sample
>> new file mode 100644
>> index 0000000000..4bd22f21d8
>> --- /dev/null
>> +++ b/templates/hooks--query-fsmonitor.sample
>> @@ -0,0 +1,27 @@
>> +#!/bin/sh
>> +#
>> +# An example hook script to integrate Watchman
>> +# (https://facebook.github.io/watchman/) with git to provide fast
>> +# git status.
>> +#
>> +# The hook is passed a time_t formatted as a string and outputs to
>> +# stdout all files that have been modified since the given time.
>> +# Paths must be relative to the root of the working tree and
>> +# separated by a single NUL.
>> +#
>> +# To enable this hook, rename this file to "query-fsmonitor"
>> +
>> +# Convert unix style paths to escaped Windows style paths
>> +case "$(uname -s)" in
>> +MINGW*|MSYS_NT*)
>> +  GIT_WORK_TREE="$(cygpath -aw "$PWD" | sed 's,\\,\\\\,g')"
>> +  ;;
>> +*)
>> +  GIT_WORK_TREE="$PWD"
>> +  ;;
>> +esac
>> +
>> +# Query Watchman for all the changes since the requested time
>> +echo "[\"query\", \"$GIT_WORK_TREE\", {\"since\": $1, \"fields\":[\"name\"]}]" | \
>> +watchman -j | \
>> +perl -e 'use JSON::PP; my $o = JSON::PP->new->utf8->decode(join("", <>)); die "Watchman: $o->{'error'}.\nFalling back to scanning...\n" if defined($o->{"error"}); print(join("\0", @{$o->{"files"}}));'
>
> Maybe put the above small perl script on multiple lines for improved
> readability.
>
> Is JSON::PP always available in Perl >= 5.8?

> On Wed, May 24, 2017 at 3:12 PM, Christian Couder
> <christian.couder@gmail.com> wrote:
>> On Thu, May 18, 2017 at 10:13 PM, Ben Peart <peartben@gmail.com> wrote:
>>> This hook script integrates the new fsmonitor capabilities of git with
>>> the cross platform Watchman file watching service. To use the script:
>>>
>>> Download and install Watchman from https://facebook.github.io/watchman/
>>> and instruct Watchman to watch your working directory for changes
>>> ('watchman watch-project /usr/src/git').
>>>
>>> Rename the sample integration hook from query-fsmonitor.sample to
>>> query-fsmonitor.
>>>
>>> Configure git to use the extension ('git config core.fsmonitor true')
>>> and optionally turn on the untracked cache for optimal performance
>>> ('git config core.untrackedcache true').
>>
>> Ok, it looks like the untracked cache can be used, but it could work without it.
>>
>>> Signed-off-by: Ben Peart <benpeart@microsoft.com>
>>> Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
>>> ---
>>>   templates/hooks--query-fsmonitor.sample | 27 +++++++++++++++++++++++++++
>>>   1 file changed, 27 insertions(+)
>>>   create mode 100644 templates/hooks--query-fsmonitor.sample
>>>
>>> diff --git a/templates/hooks--query-fsmonitor.sample b/templates/hooks--query-fsmonitor.sample
>>> new file mode 100644
>>> index 0000000000..4bd22f21d8
>>> --- /dev/null
>>> +++ b/templates/hooks--query-fsmonitor.sample
>>> @@ -0,0 +1,27 @@
>>> +#!/bin/sh
>>> +#
>>> +# An example hook script to integrate Watchman
>>> +# (https://facebook.github.io/watchman/) with git to provide fast
>>> +# git status.
>>> +#
>>> +# The hook is passed a time_t formatted as a string and outputs to
>>> +# stdout all files that have been modified since the given time.
>>> +# Paths must be relative to the root of the working tree and
>>> +# separated by a single NUL.
>>> +#
>>> +# To enable this hook, rename this file to "query-fsmonitor"
>>> +
>>> +# Convert unix style paths to escaped Windows style paths
>>> +case "$(uname -s)" in
>>> +MINGW*|MSYS_NT*)
>>> +  GIT_WORK_TREE="$(cygpath -aw "$PWD" | sed 's,\\,\\\\,g')"
>>> +  ;;
>>> +*)
>>> +  GIT_WORK_TREE="$PWD"
>>> +  ;;
>>> +esac
>>> +
>>> +# Query Watchman for all the changes since the requested time
>>> +echo "[\"query\", \"$GIT_WORK_TREE\", {\"since\": $1, \"fields\":[\"name\"]}]" | \
>>> +watchman -j | \
>>> +perl -e 'use JSON::PP; my $o = JSON::PP->new->utf8->decode(join("", <>)); die "Watchman: $o->{'error'}.\nFalling back to scanning...\n" if defined($o->{"error"}); print(join("\0", @{$o->{"files"}}));'
>>
>> Maybe put the above small perl script on multiple lines for improved
>> readability.
>>

>> Is JSON::PP always available in Perl >= 5.8?
> 
> No, it's shipped with perl as of 5.14.0, which came out in May 2011.
> 
> I think depending on that is fine. FWIW we bumped the required core
> dependency (but you might still need to install modules) in 2010 in my
> d48b284183 ("perl: bump the required Perl version to 5.8 from
> 5.6.[21]", 2010-09-24). Maybe we should be bumping that again...
> 
>> What happens if watchman is not installed or not in the PATH?
>> It seems to me that the git process will not die, and will just work
>> as if the hook does not exist, except that it will call the hook which
>> will probably output error messages.
> 
> I think a good solution to this is to just add "set -euo pipefail" to
> that script.
> 
> Then we'll print out on stderr that we couldn't find the watchman
> command. Right now (with my patch) it'll make its way to the perl
> program and get empty input.
> 

> This hook script integrates the new fsmonitor capabilities of git with
> the cross platform Watchman file watching service. To use the script:
>
> Download and install Watchman from https://facebook.github.io/watchman/
> and instruct Watchman to watch your working directory for changes
> ('watchman watch-project /usr/src/git').
>
> Rename the sample integration hook from query-fsmonitor.sample to
> query-fsmonitor.
>
> Configure git to use the extension ('git config core.fsmonitor true')
> and optionally turn on the untracked cache for optimal performance
> ('git config core.untrackedcache true').
>
> Signed-off-by: Ben Peart <benpeart@microsoft.com>
> Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
> ---
>  templates/hooks--query-fsmonitor.sample | 27 +++++++++++++++++++++++++++
>  1 file changed, 27 insertions(+)
>  create mode 100644 templates/hooks--query-fsmonitor.sample
>
> diff --git a/templates/hooks--query-fsmonitor.sample b/templates/hooks--query-fsmonitor.sample
> new file mode 100644
> index 0000000000..4bd22f21d8
> --- /dev/null
> +++ b/templates/hooks--query-fsmonitor.sample
> @@ -0,0 +1,27 @@
> +#!/bin/sh
> +#
> +# An example hook script to integrate Watchman
> +# (https://facebook.github.io/watchman/) with git to provide fast
> +# git status.
> +#
> +# The hook is passed a time_t formatted as a string and outputs to
> +# stdout all files that have been modified since the given time.
> +# Paths must be relative to the root of the working tree and
> +# separated by a single NUL.
> +#
> +# To enable this hook, rename this file to "query-fsmonitor"
> +
> +# Convert unix style paths to escaped Windows style paths
> +case "$(uname -s)" in
> +MINGW*|MSYS_NT*)
> +  GIT_WORK_TREE="$(cygpath -aw "$PWD" | sed 's,\\,\\\\,g')"
> +  ;;
> +*)
> +  GIT_WORK_TREE="$PWD"
> +  ;;
> +esac
> +
> +# Query Watchman for all the changes since the requested time
> +echo "[\"query\", \"$GIT_WORK_TREE\", {\"since\": $1, \"fields\":[\"name\"]}]" | \
> +watchman -j | \
> +perl -e 'use JSON::PP; my $o = JSON::PP->new->utf8->decode(join("", <>)); die "Watchman: $o->{'error'}.\nFalling back to scanning...\n" if defined($o->{"error"}); print(join("\0", @{$o->{"files"}}));'

> diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt
> index 706091a569..f7b4b4a844 100644
> --- a/Documentation/githooks.txt
> +++ b/Documentation/githooks.txt
> @@ -448,6 +448,29 @@ The commits are guaranteed to be listed in the order that they were
>  processed by rebase.
>  
>  
> +[[query-fsmonitor]]
> +query-fsmonitor
> +~~~~~~~~~~~~

> This includes the core.fsmonitor setting, the query-fsmonitor hook,
> and the fsmonitor index extension.
>
> Signed-off-by: Ben Peart <benpeart@microsoft.com>
> ---
>  Documentation/config.txt                 |  7 +++++++
>  Documentation/githooks.txt               | 23 +++++++++++++++++++++++
>  Documentation/technical/index-format.txt | 18 ++++++++++++++++++
>  3 files changed, 48 insertions(+)
>
> diff --git a/Documentation/config.txt b/Documentation/config.txt
> index 96e9cf8b73..4ffbf0d4c2 100644
> --- a/Documentation/config.txt
> +++ b/Documentation/config.txt
> @@ -389,6 +389,13 @@ core.protectNTFS::
>         8.3 "short" names.
>         Defaults to `true` on Windows, and `false` elsewhere.
>
> +core.fsmonitor::
> +       If set to true, call the query-fsmonitor hook proc which will
> +       identify all files that may have had changes since the last
> +       request. This information is used to speed up operations like
> +       'git commit' and 'git status' by limiting what git must scan to
> +       detect changes.
> +
>  core.trustctime::
>         If false, the ctime differences between the index and the
>         working tree are ignored; useful when the inode change time
> diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt
> index 706091a569..f7b4b4a844 100644
> --- a/Documentation/githooks.txt
> +++ b/Documentation/githooks.txt
> @@ -448,6 +448,29 @@ The commits are guaranteed to be listed in the order that they were
>  processed by rebase.
>
>
> +[[query-fsmonitor]]
> +query-fsmonitor
> +~~~~~~~~~~~~
> +
> +This hook is invoked when the configuration option core.fsmonitor is
> +set and git needs to identify changed or untracked files.  It takes
> +a single argument which is the time in elapsed seconds since midnight,
> +January 1, 1970.
> +
> +The hook should output to stdout the list of all files in the working
> +directory that may have changed since the requested time.  The logic
> +should be inclusive so that it does not miss any potential changes.
> +The paths should be relative to the root of the working directory
> +and be separated by a single NUL.
> +
> +Git will limit what files it checks for changes as well as which
> +directories are checked for untracked files based on the path names
> +given.
> +
> +The exit status determines whether git will use the data from the
> +hook to limit its search.  On error, it will fall back to verifying
> +all files and folders.
> +
>  GIT
>  ---
>  Part of the linkgit:git[1] suite
> diff --git a/Documentation/technical/index-format.txt b/Documentation/technical/index-format.txt
> index ade0b0c445..b002d23c05 100644
> --- a/Documentation/technical/index-format.txt
> +++ b/Documentation/technical/index-format.txt
> @@ -295,3 +295,21 @@ The remaining data of each directory block is grouped by type:
>      in the previous ewah bitmap.
>
>    - One NUL.
> +
> +== File System Monitor cache
> +
> +  The file system monitor cache tracks files for which the query-fsmonitor
> +  hook has told us about changes.  The signature for this extension is
> +  { 'F', 'S', 'M', 'N' }.
> +
> +  The extension starts with
> +
> +  - 32-bit version number: the current supported version is 1.
> +
> +  - 64-bit time: the extension data reflects all changes through the given
> +       time which is stored as the seconds elapsed since midnight, January 1, 1970.
> +
> +  - 32-bit bitmap size: the size of the CE_FSMONITOR_DIRTY bitmap.
> +
> +  - An ewah bitmap, the n-th bit indicates whether the n-th index entry
> +    is CE_FSMONITOR_DIRTY.

>> +== File System Monitor cache
>> +
>> +  The file system monitor cache tracks files for which the query-fsmonitor
>> +  hook has told us about changes.  The signature for this extension is
>> +  { 'F', 'S', 'M', 'N' }.
>> +
>> +  The extension starts with
>> +
>> +  - 32-bit version number: the current supported version is 1.
>> +
>> +  - 64-bit time: the extension data reflects all changes through the given
>> +       time which is stored as the seconds elapsed since midnight, January 1, 1970.
>> +
>> +  - 32-bit bitmap size: the size of the CE_FSMONITOR_DIRTY bitmap.
>> +
>> +  - An ewah bitmap, the n-th bit indicates whether the n-th index entry
>> +    is CE_FSMONITOR_DIRTY.
>
> We already have a uint64_t in one place in the codebase (getnanotime)
> which uses a 64 bit time for nanosecond accuracy, and numerous
> filesystems already support nanosecond timestamps (ext4, that new
> Apple thingy...).
>
> I don't know if any of the inotify/fsmonitor APIs support that yet,
> but it seems inevitable that that'll be added if not, in some
> pathological cases we can have a lot of files modified in 1 second, so
> using nanosecond accuracy means there'll be a lot less data to
> consider in some cases.
>
> It does mean this'll only work until the year ~2500, but that seems
> like an acceptable trade-off.
>

> On 5/20/2017 8:10 AM, Ævar Arnfjörð Bjarmason wrote:
>>>
>>> +== File System Monitor cache
>>> +
>>> +  The file system monitor cache tracks files for which the
>>> query-fsmonitor
>>> +  hook has told us about changes.  The signature for this extension is
>>> +  { 'F', 'S', 'M', 'N' }.
>>> +
>>> +  The extension starts with
>>> +
>>> +  - 32-bit version number: the current supported version is 1.
>>> +
>>> +  - 64-bit time: the extension data reflects all changes through the
>>> given
>>> +       time which is stored as the seconds elapsed since midnight,
>>> January 1, 1970.
>>> +
>>> +  - 32-bit bitmap size: the size of the CE_FSMONITOR_DIRTY bitmap.
>>> +
>>> +  - An ewah bitmap, the n-th bit indicates whether the n-th index entry
>>> +    is CE_FSMONITOR_DIRTY.
>>
>>
>> We already have a uint64_t in one place in the codebase (getnanotime)
>> which uses a 64 bit time for nanosecond accuracy, and numerous
>> filesystems already support nanosecond timestamps (ext4, that new
>> Apple thingy...).
>>
>> I don't know if any of the inotify/fsmonitor APIs support that yet,
>> but it seems inevitable that that'll be added if not, in some
>> pathological cases we can have a lot of files modified in 1 second, so
>> using nanosecond accuracy means there'll be a lot less data to
>> consider in some cases.
>>
>> It does mean this'll only work until the year ~2500, but that seems
>> like an acceptable trade-off.
>>
>
> I really don't think nano-second resolution is needed in this case for a few
> reasons.
>
> The number of files that can change within a given second is limited by the
> IO throughput of the underlying device. Even assuming a very fast device and
> very small files and changes, this won't be that many files.
>
> Without this patch, git would have scanned all those files every time. With
> this patch, git will only scan those files a 2nd time that are modified in
> the same second that it did the first scan *that came before the first scan
> started* (the "lots of files modified" section in the 1 second timeline
> below).
>
> |------------------------- one second ---------------------|
> |-lots of files modified - git status - more file modified-|
>
> Yes, some duplicate status checks can be made but its still a significant
> win in any reasonable scenario. Especially when you consider that it is
> pretty unusual to do git status/add/commit calls in the middle of making
> lots of changes to files.

> On Mon, May 22, 2017 at 6:18 PM, Ben Peart <peartben@gmail.com> wrote:
>> On 5/20/2017 8:10 AM, Ævar Arnfjörð Bjarmason wrote:
>>>>
>>>> +== File System Monitor cache
>>>> +
>>>> +  The file system monitor cache tracks files for which the
>>>> query-fsmonitor
>>>> +  hook has told us about changes.  The signature for this extension is
>>>> +  { 'F', 'S', 'M', 'N' }.
>>>> +
>>>> +  The extension starts with
>>>> +
>>>> +  - 32-bit version number: the current supported version is 1.
>>>> +
>>>> +  - 64-bit time: the extension data reflects all changes through the
>>>> given
>>>> +       time which is stored as the seconds elapsed since midnight,
>>>> January 1, 1970.
>>>> +
>>>> +  - 32-bit bitmap size: the size of the CE_FSMONITOR_DIRTY bitmap.
>>>> +
>>>> +  - An ewah bitmap, the n-th bit indicates whether the n-th index entry
>>>> +    is CE_FSMONITOR_DIRTY.
>>>
>>>
>>> We already have a uint64_t in one place in the codebase (getnanotime)
>>> which uses a 64 bit time for nanosecond accuracy, and numerous
>>> filesystems already support nanosecond timestamps (ext4, that new
>>> Apple thingy...).
>>>
>>> I don't know if any of the inotify/fsmonitor APIs support that yet,
>>> but it seems inevitable that that'll be added if not, in some
>>> pathological cases we can have a lot of files modified in 1 second, so
>>> using nanosecond accuracy means there'll be a lot less data to
>>> consider in some cases.
>>>
>>> It does mean this'll only work until the year ~2500, but that seems
>>> like an acceptable trade-off.
>>>
>>
>> I really don't think nano-second resolution is needed in this case for a few
>> reasons.
>>
>> The number of files that can change within a given second is limited by the
>> IO throughput of the underlying device. Even assuming a very fast device and
>> very small files and changes, this won't be that many files.
>>
>> Without this patch, git would have scanned all those files every time. With
>> this patch, git will only scan those files a 2nd time that are modified in
>> the same second that it did the first scan *that came before the first scan
>> started* (the "lots of files modified" section in the 1 second timeline
>> below).
>>
>> |------------------------- one second ---------------------|
>> |-lots of files modified - git status - more file modified-|
>>
>> Yes, some duplicate status checks can be made but its still a significant
>> win in any reasonable scenario. Especially when you consider that it is
>> pretty unusual to do git status/add/commit calls in the middle of making
>> lots of changes to files.
> 
> I understand that we get most of the wins either way.
> 
> I'm just wondering why not make it nanosecond-resolution now from the
> beginning since that's where major filesystems are heading already,
> and changing stuff like this can be a hassle once it's initially out
> there, whereas just dividing by 10^9 for APIs that need seconds from
> the beginning is easy & future-proof.
> 
> There are some uses of git where this would probably matter in practice.
> 
> E.g. consider a git-annex backed fileserver using ext4, currently
> git-annex comes with its own FS watcher as a daemon invoked via `git
> annex watch` to constantly add new files without killing performance
> via a status/add in a loop, with this a daemon like that could just
> run status/add in a loop, but on a busy repo the 1s interval size
> might start to matter as you're constantly inspecting larger
> intervals.
> 
> More importantly though, I can't think of any case where having it in
> nanoseconds to begin with would do any harm.

> 
>> In addition, the backing file system monitor (Watchman) supports number of
>> seconds since the unix epoch (unix time_t style).  This means any support of
>> nano seconds by git is academic until someone provides a file system watcher
>> that does support nano second granularity.
> 
> I haven't used watchman for anything non-trivial, but the
> documentation for the query API you seem to be using says you can
> request a {ctime,mtime}_ns field:
> 
> https://github.com/facebook/watchman/blob/master/website/_docs/cmd.query.markdown#user-content-available-fields
> 
> And isn't this the documentation for the "since" query you're using,
> saying you can specify any arbitrary fs timing field, such as a _ns
> time field:
> 
> https://github.com/facebook/watchman/blob/master/website/_docs/expr.since.md
> 
> ?

> 
>> Finally, the fsmonitor index extension is versioned so that we can
>> seamlessly upgrade to nano second resolution later if we desire.
> 
> Aside from my nanosecond bikeshedding, let's say we change the
> semantics of any of this in the future: The index has the version, but
> there's one argument passed to the hook without a version. Is the hook
> expected to potentially be reading the version from the index to make
> sense of whether (in this case) the argument is a mtime or mtime_ns?
> 
> Wouldn't this make more sense than that on top, i.e. pass the version
> to the hook, untested (and probably whines about the sprintf
> format...):

> 
> $ git diff -U1
> diff --git a/cache.h b/cache.h
> index 6eafd34fff..3c63f179f8 100644
> --- a/cache.h
> +++ b/cache.h
> @@ -346,2 +346,3 @@ struct index_state {
>          struct untracked_cache *untracked;
> +       uint32_t fsmonitor_version;
>          time_t fsmonitor_last_update;
> diff --git a/fsmonitor.c b/fsmonitor.c
> index f5a9f818e8..7236b538ac 100644
> --- a/fsmonitor.c
> +++ b/fsmonitor.c
> @@ -60,2 +60,4 @@ int read_fsmonitor_extension(struct index_state
> *istate, const void *data,
>                  return error("bad fsmonitor version %d", hdr_version);
> +       else
> +               istate->fsmonitor_version = hdr_version;
> 
> @@ -137,2 +139,3 @@ static int query_fsmonitor(time_t last_update,
> struct strbuf *query_result)
>          struct child_process cp = CHILD_PROCESS_INIT;
> +       char version[1];
>          char date[64];
> @@ -143,2 +146,3 @@ static int query_fsmonitor(time_t last_update,
> struct strbuf *query_result)
> 
> +       snprintf(version, sizeof(version), "%d", istate->fsmonitor_version);
>          snprintf(date, sizeof(date), "%" PRIuMAX, (uintmax_t)last_update)
> 

> Design
> ~~~~~~
>
> A new git hook (query-fsmonitor) must exist and be enabled
> (core.fsmonitor=true) that takes a time_t formatted as a string and
> outputs to stdout all files that have been modified since the requested
> time.

> A new 'fsmonitor' index extension has been added to store the time the
> fsmonitor hook was last queried and a ewah bitmap of the current
> 'fsmonitor-dirty' files. Unmarked entries are 'fsmonitor-clean', marked
> entries are 'fsmonitor-dirty.'
>
> As needed, git will call the query-fsmonitor hook proc for the set of
> changes since the index was last updated. Git then uses this set of
> files along with the list saved in the fsmonitor index extension to flag
> the potentially dirty index and untracked cache entries.

>> Design
>> ~~~~~~
>>
>> A new git hook (query-fsmonitor) must exist and be enabled
>> (core.fsmonitor=true) that takes a time_t formatted as a string and
>> outputs to stdout all files that have been modified since the requested
>> time.
> 
> Is there a reason why there is a new hook, instead of a
> "core.fsmonitorquery" config option to which you could pass whatever
> command line with options?

> 
>> A new 'fsmonitor' index extension has been added to store the time the
>> fsmonitor hook was last queried and a ewah bitmap of the current
>> 'fsmonitor-dirty' files. Unmarked entries are 'fsmonitor-clean', marked
>> entries are 'fsmonitor-dirty.'
>>
>> As needed, git will call the query-fsmonitor hook proc for the set of
>> changes since the index was last updated. Git then uses this set of
>> files along with the list saved in the fsmonitor index extension to flag
>> the potentially dirty index and untracked cache entries.
> 
> So this can work only if "core.untrackedCache" is set to true?
> 

>
>
> On 5/24/2017 6:54 AM, Christian Couder wrote:
>>>
>>> Design
>>> ~~~~~~
>>>
>>> A new git hook (query-fsmonitor) must exist and be enabled
>>> (core.fsmonitor=true) that takes a time_t formatted as a string and
>>> outputs to stdout all files that have been modified since the requested
>>> time.
>>
>>
>> Is there a reason why there is a new hook, instead of a
>> "core.fsmonitorquery" config option to which you could pass whatever
>> command line with options?
>
>
> A hook is a simple and well defined way to integrate git with another
> process.  If there is some fixed set of arguments that need to be passed to
> a file system monitor (beyond the timestamp stored in the index extension),
> they can be encoded in the integration script like I've done in the Watchman
> integration sample hook.

>>> A new 'fsmonitor' index extension has been added to store the time the
>>> fsmonitor hook was last queried and a ewah bitmap of the current
>>> 'fsmonitor-dirty' files. Unmarked entries are 'fsmonitor-clean', marked
>>> entries are 'fsmonitor-dirty.'
>>>
>>> As needed, git will call the query-fsmonitor hook proc for the set of
>>> changes since the index was last updated. Git then uses this set of
>>> files along with the list saved in the fsmonitor index extension to flag
>>> the potentially dirty index and untracked cache entries.
>>
>>
>> So this can work only if "core.untrackedCache" is set to true?
>>
>
> This works with core.untrackedCache set to true or false.  If it is set to
> false, you get valid results, you just don't get the speed up when checking
> for untracked files.

> On Thu, May 25, 2017 at 3:55 PM, Ben Peart <peartben@gmail.com> wrote:
>>
>>
>> On 5/24/2017 6:54 AM, Christian Couder wrote:
>>>>
>>>> Design
>>>> ~~~~~~
>>>>
>>>> A new git hook (query-fsmonitor) must exist and be enabled
>>>> (core.fsmonitor=true) that takes a time_t formatted as a string and
>>>> outputs to stdout all files that have been modified since the requested
>>>> time.
>>>
>>>
>>> Is there a reason why there is a new hook, instead of a
>>> "core.fsmonitorquery" config option to which you could pass whatever
>>> command line with options?
>>
>>
>> A hook is a simple and well defined way to integrate git with another
>> process.  If there is some fixed set of arguments that need to be passed to
>> a file system monitor (beyond the timestamp stored in the index extension),
>> they can be encoded in the integration script like I've done in the Watchman
>> integration sample hook.
> 
> Yeah, they could be encoded in the integration script, but it could be
> better if it was possible to just configure a generic command line.
> 
> For example if the directory that should be watched for filesystem
> changes could be passed as well as the time since the last changes,
> perhaps only a generic command line would be need.

> 
> I am also wondering about sparse checkout, as we might want to pass
> all the directories we are interested in.
> How is it supposed to work with sparse checkout?
> 

>>>> A new 'fsmonitor' index extension has been added to store the time the
>>>> fsmonitor hook was last queried and a ewah bitmap of the current
>>>> 'fsmonitor-dirty' files. Unmarked entries are 'fsmonitor-clean', marked
>>>> entries are 'fsmonitor-dirty.'
>>>>
>>>> As needed, git will call the query-fsmonitor hook proc for the set of
>>>> changes since the index was last updated. Git then uses this set of
>>>> files along with the list saved in the fsmonitor index extension to flag
>>>> the potentially dirty index and untracked cache entries.
>>>
>>>
>>> So this can work only if "core.untrackedCache" is set to true?
>>>
>>
>> This works with core.untrackedCache set to true or false.  If it is set to
>> false, you get valid results, you just don't get the speed up when checking
>> for untracked files.
> 
> Great!
> 

>
>
> On 5/27/2017 2:57 AM, Christian Couder wrote:
>>
>> On Thu, May 25, 2017 at 3:55 PM, Ben Peart <peartben@gmail.com> wrote:
>>>
>>>
>>>
>>> On 5/24/2017 6:54 AM, Christian Couder wrote:
>>>>>
>>>>>
>>>>> Design
>>>>> ~~~~~~
>>>>>
>>>>> A new git hook (query-fsmonitor) must exist and be enabled
>>>>> (core.fsmonitor=true) that takes a time_t formatted as a string and
>>>>> outputs to stdout all files that have been modified since the requested
>>>>> time.
>>>>
>>>>
>>>>
>>>> Is there a reason why there is a new hook, instead of a
>>>> "core.fsmonitorquery" config option to which you could pass whatever
>>>> command line with options?
>>>
>>>
>>>
>>> A hook is a simple and well defined way to integrate git with another
>>> process.  If there is some fixed set of arguments that need to be passed
>>> to
>>> a file system monitor (beyond the timestamp stored in the index
>>> extension),
>>> they can be encoded in the integration script like I've done in the
>>> Watchman
>>> integration sample hook.
>>
>>
>> Yeah, they could be encoded in the integration script, but it could be
>> better if it was possible to just configure a generic command line.
>>
>> For example if the directory that should be watched for filesystem
>> changes could be passed as well as the time since the last changes,
>> perhaps only a generic command line would be need.
>
>
> Maybe I'm not understanding what you have in mind but I haven't found this
> to be the case in the two integrations I've done with file system watchers
> (one internal and Watchman).  They require you download, install, and
> configure them by telling them about the folders you want monitored.  Then
> you can start querying them for changes and processing the output to match
> what git expects.  While the download and install steps vary, having that
> query + process and return results wrapped up in an integration hook has
> worked well.

>> I am also wondering about sparse checkout, as we might want to pass
>> all the directories we are interested in.
>> How is it supposed to work with sparse checkout?
>
> The fsmonitor code works well with or without a sparse-checkout.  The file
> system monitor is unaware of the sparse checkout so will notify git about
> any change irrespective of whether git will eventually ignore it because the
> skip worktree bit is set.

> On Tue, May 30, 2017 at 8:05 PM, Ben Peart <peartben@gmail.com> wrote:
>>
>>
>> On 5/27/2017 2:57 AM, Christian Couder wrote:
>>>
>>> On Thu, May 25, 2017 at 3:55 PM, Ben Peart <peartben@gmail.com> wrote:
>>>>
>>>>
>>>>
>>>> On 5/24/2017 6:54 AM, Christian Couder wrote:
>>>>>>
>>>>>>
>>>>>> Design
>>>>>> ~~~~~~
>>>>>>
>>>>>> A new git hook (query-fsmonitor) must exist and be enabled
>>>>>> (core.fsmonitor=true) that takes a time_t formatted as a string and
>>>>>> outputs to stdout all files that have been modified since the requested
>>>>>> time.
>>>>>
>>>>>
>>>>>
>>>>> Is there a reason why there is a new hook, instead of a
>>>>> "core.fsmonitorquery" config option to which you could pass whatever
>>>>> command line with options?
>>>>
>>>>
>>>>
>>>> A hook is a simple and well defined way to integrate git with another
>>>> process.  If there is some fixed set of arguments that need to be passed
>>>> to
>>>> a file system monitor (beyond the timestamp stored in the index
>>>> extension),
>>>> they can be encoded in the integration script like I've done in the
>>>> Watchman
>>>> integration sample hook.
>>>
>>>
>>> Yeah, they could be encoded in the integration script, but it could be
>>> better if it was possible to just configure a generic command line.
>>>
>>> For example if the directory that should be watched for filesystem
>>> changes could be passed as well as the time since the last changes,
>>> perhaps only a generic command line would be need.
>>
>>
>> Maybe I'm not understanding what you have in mind but I haven't found this
>> to be the case in the two integrations I've done with file system watchers
>> (one internal and Watchman).  They require you download, install, and
>> configure them by telling them about the folders you want monitored.  Then
>> you can start querying them for changes and processing the output to match
>> what git expects.  While the download and install steps vary, having that
>> query + process and return results wrapped up in an integration hook has
>> worked well.
> 
> It looks like one can also just ask watchman to monitor a directory with:
> 
> watchman watch /path/to/dir
> 
> or:
> 
> echo '["watch", "/path/to/dir"]' | watchman -j
> 
> Also for example on Linux people might want to use command line tools like:
> 
> https://linux.die.net/man/1/inotifywait
> 
> and you can pass the directories you want to be watched as arguments
> to this kind of tools.
> 
> So it would be nice, if we didn't require the user to configure
> anything and we could just configure the watching of what we need in
> the hook (or a command configured using a config option). If the hook
> (or configured command) could be passed the directory by git, it could
> also be generic.
> 

>>> I am also wondering about sparse checkout, as we might want to pass
>>> all the directories we are interested in.
>>> How is it supposed to work with sparse checkout?
>>
>> The fsmonitor code works well with or without a sparse-checkout.  The file
>> system monitor is unaware of the sparse checkout so will notify git about
>> any change irrespective of whether git will eventually ignore it because the
>> skip worktree bit is set.
> 
> I was wondering if it could ease the job for the monitoring service
> and perhaps improve performance to just ask to watch the directories
> we are interested in when using sparse checkout.
> On Linux it looks like a separate inotify watch is created for every
> subdirectory and there is maximum amount of inotify watches per user.
> This can be increased by writing in
> /proc/sys/fs/inotify/max_user_watches, but it is not nice to have to
> ask admins to increase this.
> 

>>>> Yeah, they could be encoded in the integration script, but it could be
>>>> better if it was possible to just configure a generic command line.
>>>>
>>>> For example if the directory that should be watched for filesystem
>>>> changes could be passed as well as the time since the last changes,
>>>> perhaps only a generic command line would be need.
>>>
>>> Maybe I'm not understanding what you have in mind but I haven't found
>>> this
>>> to be the case in the two integrations I've done with file system
>>> watchers
>>> (one internal and Watchman).  They require you download, install, and
>>> configure them by telling them about the folders you want monitored.
>>> Then
>>> you can start querying them for changes and processing the output to
>>> match
>>> what git expects.  While the download and install steps vary, having that
>>> query + process and return results wrapped up in an integration hook has
>>> worked well.
>>
>> It looks like one can also just ask watchman to monitor a directory with:
>>
>> watchman watch /path/to/dir
>>
>> or:
>>
>> echo '["watch", "/path/to/dir"]' | watchman -j
>>
>> Also for example on Linux people might want to use command line tools
>> like:
>>
>> https://linux.die.net/man/1/inotifywait
>>
>> and you can pass the directories you want to be watched as arguments
>> to this kind of tools.
>>
>> So it would be nice, if we didn't require the user to configure
>> anything and we could just configure the watching of what we need in
>> the hook (or a command configured using a config option). If the hook
>> (or configured command) could be passed the directory by git, it could
>> also be generic.
>
> OK, I think I understand what you're attempting to accomplish now. Often,
> Watchman (and other similar tools) are used to do much more than speed up
> git (in fact, _all_ use cases today are not used for that since this patch
> series hasn't been accepted yet :)).  They trigger builds, run verification
> tools, test passes, or other tasks.

>>>> I am also wondering about sparse checkout, as we might want to pass
>>>> all the directories we are interested in.
>>>> How is it supposed to work with sparse checkout?
>>>
>>>
>>> The fsmonitor code works well with or without a sparse-checkout.  The
>>> file
>>> system monitor is unaware of the sparse checkout so will notify git about
>>> any change irrespective of whether git will eventually ignore it because
>>> the
>>> skip worktree bit is set.
>>
>> I was wondering if it could ease the job for the monitoring service
>> and perhaps improve performance to just ask to watch the directories
>> we are interested in when using sparse checkout.
>> On Linux it looks like a separate inotify watch is created for every
>> subdirectory and there is maximum amount of inotify watches per user.
>> This can be increased by writing in
>> /proc/sys/fs/inotify/max_user_watches, but it is not nice to have to
>> ask admins to increase this.
>
> Having a single instance that watches the root of the working directory is
> the simplest model and minimizes use of system resources like inotify as
> there is only one needed per clone.

>
> On 5/24/2017 6:54 AM, Christian Couder wrote:
>>>
>>> A new git hook (query-fsmonitor) must exist and be enabled
>>> (core.fsmonitor=true) that takes a time_t formatted as a string and
>>> outputs to stdout all files that have been modified since the requested
>>> time.
>>
>> Is there a reason why there is a new hook, instead of a
>> "core.fsmonitorquery" config option to which you could pass whatever
>> command line with options?
>
> A hook is a simple and well defined way to integrate git with another
> process.  If there is some fixed set of arguments that need to be passed to
> a file system monitor (beyond the timestamp stored in the index extension),
> they can be encoded in the integration script like I've done in the Watchman
> integration sample hook.

> On Thu, May 25, 2017 at 3:55 PM, Ben Peart <peartben@gmail.com> wrote:
>>
>> On 5/24/2017 6:54 AM, Christian Couder wrote:
>>>>
>>>> A new git hook (query-fsmonitor) must exist and be enabled
>>>> (core.fsmonitor=true) that takes a time_t formatted as a string and
>>>> outputs to stdout all files that have been modified since the requested
>>>> time.
>>>
>>> Is there a reason why there is a new hook, instead of a
>>> "core.fsmonitorquery" config option to which you could pass whatever
>>> command line with options?
>>
>> A hook is a simple and well defined way to integrate git with another
>> process.  If there is some fixed set of arguments that need to be passed to
>> a file system monitor (beyond the timestamp stored in the index extension),
>> they can be encoded in the integration script like I've done in the Watchman
>> integration sample hook.
> 
> Yeah, but a hook must also be called everytime git wants to
> communicate with the file system monitor. And we could perhaps get a
> speed up if we could have only one long running process to communicate
> with the file system monitor.
> 

> On 5/31/2017 3:59 AM, Christian Couder wrote:
>>
>> On Thu, May 25, 2017 at 3:55 PM, Ben Peart <peartben@gmail.com> wrote:
>>>
>>>
>>> On 5/24/2017 6:54 AM, Christian Couder wrote:
>>>>>
>>>>>
>>>>> A new git hook (query-fsmonitor) must exist and be enabled
>>>>> (core.fsmonitor=true) that takes a time_t formatted as a string and
>>>>> outputs to stdout all files that have been modified since the requested
>>>>> time.
>>>>
>>>>
>>>> Is there a reason why there is a new hook, instead of a
>>>> "core.fsmonitorquery" config option to which you could pass whatever
>>>> command line with options?
>>>
>>>
>>> A hook is a simple and well defined way to integrate git with another
>>> process.  If there is some fixed set of arguments that need to be passed
>>> to
>>> a file system monitor (beyond the timestamp stored in the index
>>> extension),
>>> they can be encoded in the integration script like I've done in the
>>> Watchman
>>> integration sample hook.
>>
>>
>> Yeah, but a hook must also be called everytime git wants to
>> communicate with the file system monitor. And we could perhaps get a
>> speed up if we could have only one long running process to communicate
>> with the file system monitor.
>>
>
> In this particular case a long running background processes isn't helpful
> because refresh_by_fsmonitor() already has logic to ensure the file system
> monitor is only called once per git process.
>
> The overhead of that one call isn't significant as demonstrated by the
> performance numbers I sent out in the cover letter.  Even with the cost of
> the hook and all the associated post-processing, this patch series still
> results in a nice performance win (even on Windows where spawning processes
> is typically more expensive than on other platforms).
>
> I appreciate the close look at this patch series!  Even when I'm pushing
> back, I'm glad someone is asking questions and making sure I've thought
> through things.