{"thread":{"id":"47381","subject":"[PATCH v6 00/14] Parial clone part 3: clone, fetch, fetch-pack, upload-pack, and tests","startedAt":"2017-12-05T17:03:16Z","lastAt":"2017-12-05T17:04:04Z","messageCount":15,"participants":["Jeff Hostetler"],"isPatch":true,"patchVersion":6,"patchTotal":14},"messages":[{"id":"334182","messageId":"20171205170254.65293-1-git@jeffhostetler.com","threadId":"47381","inReplyTo":null,"subject":"[PATCH v6 00/14] Parial clone part 3: clone, fetch, fetch-pack, upload-pack, and tests","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:40Z","receivedAt":"2017-12-05T17:03:16Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jeff Hostetler <jeffhost@microsoft.com>\n\nThis is V6 of part 3 of partial clone.  It assumes that V6\nof parts 1 and 2 are already present.\n\nThis version is refactors and reorders commits to better combine\nnew functionality and tests.  It contains 3 fixup commits that\nshould be squashed after some discussion.\n\nJeff Hostetler (9):\n  upload-pack: add object filtering for partial clone\n  fetch-pack, index-pack, transport: partial clone\n  fetch-pack: add --no-filter\n  fetch: support filters\n  fixup: fetch: update error messages from --blob-max-bytes to --filter\n  fixup: fetch: update --blob-max-bytes to --fitler\n  fixup: connected: conditionally pass --exclude-promisor-objects to\n    rev-list\n  partial-clone: define partial clone settings in config\n  t5616: end-to-end tests for partial clone\n\nJonathan Tan (5):\n  fetch-pack: test support excluding large blobs\n  fetch: refactor calculation of remote list\n  clone: partial clone\n  unpack-trees: batch fetching of missing blobs\n  fetch-pack: restore save_commit_buffer after use\n\n Documentation/config.txt                          |   4 +\n Documentation/technical/pack-protocol.txt         |   8 ++\n Documentation/technical/protocol-capabilities.txt |   8 ++\n builtin/clone.c                                   |  22 +++-\n builtin/fetch-pack.c                              |   8 ++\n builtin/fetch.c                                   |  33 ++++--\n cache.h                                           |   1 +\n config.c                                          |   5 +\n connected.c                                       |   2 +\n environment.c                                     |   1 +\n fetch-object.c                                    |  26 ++++-\n fetch-object.h                                    |   5 +\n fetch-pack.c                                      |  17 +++\n fetch-pack.h                                      |   2 +\n list-objects-filter-options.c                     |  90 +++++++++++++---\n list-objects-filter-options.h                     |   6 ++\n remote-curl.c                                     |   6 ++\n t/t5500-fetch-pack.sh                             |  63 +++++++++++\n t/t5601-clone.sh                                  | 101 +++++++++++++++++\n t/t5616-partial-clone.sh                          | 125 ++++++++++++++++++++++\n transport-helper.c                                |   5 +\n transport.c                                       |   4 +\n transport.h                                       |   5 +\n unpack-trees.c                                    |  22 ++++\n upload-pack.c                                     |  31 +++++-\n 25 files changed, 572 insertions(+), 28 deletions(-)\n create mode 100755 t/t5616-partial-clone.sh\n\n-- \n2.9.3\n\n"},{"id":"334183","messageId":"20171205170254.65293-3-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 02/14] fetch-pack, index-pack, transport: partial clone","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:42Z","receivedAt":"2017-12-05T17:03:20Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jeff Hostetler <jeffhost@microsoft.com>\n\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n builtin/fetch-pack.c |  4 ++++\n fetch-pack.c         | 13 +++++++++++++\n fetch-pack.h         |  2 ++\n transport-helper.c   |  5 +++++\n transport.c          |  4 ++++\n transport.h          |  5 +++++\n 6 files changed, 33 insertions(+)\n\ndiff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c\nindex 15eeed7..7957807 100644\n--- a/builtin/fetch-pack.c\n+++ b/builtin/fetch-pack.c\n@@ -153,6 +153,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)\n \t\t\targs.no_dependents = 1;\n \t\t\tcontinue;\n \t\t}\n+\t\tif (skip_prefix(arg, (\"--\" CL_ARG__FILTER \"=\"), &arg)) {\n+\t\t\tparse_list_objects_filter(&args.filter_options, arg);\n+\t\t\tcontinue;\n+\t\t}\n \t\tusage(fetch_pack_usage);\n \t}\n \tif (deepen_not.nr)\ndiff --git a/fetch-pack.c b/fetch-pack.c\nindex 0798e0b..3c5f064 100644\n--- a/fetch-pack.c\n+++ b/fetch-pack.c\n@@ -29,6 +29,7 @@ static int deepen_not_ok;\n static int fetch_fsck_objects = -1;\n static int transfer_fsck_objects = -1;\n static int agent_supported;\n+static int server_supports_filtering;\n static struct lock_file shallow_lock;\n static const char *alternate_shallow_file;\n \n@@ -379,6 +380,8 @@ static int find_common(struct fetch_pack_args *args,\n \t\t\tif (deepen_not_ok)      strbuf_addstr(&c, \" deepen-not\");\n \t\t\tif (agent_supported)    strbuf_addf(&c, \" agent=%s\",\n \t\t\t\t\t\t\t    git_user_agent_sanitized());\n+\t\t\tif (args->filter_options.choice)\n+\t\t\t\tstrbuf_addstr(&c, \" filter\");\n \t\t\tpacket_buf_write(&req_buf, \"want %s%s\\n\", remote_hex, c.buf);\n \t\t\tstrbuf_release(&c);\n \t\t} else\n@@ -407,6 +410,9 @@ static int find_common(struct fetch_pack_args *args,\n \t\t\tpacket_buf_write(&req_buf, \"deepen-not %s\", s->string);\n \t\t}\n \t}\n+\tif (server_supports_filtering && args->filter_options.choice)\n+\t\tpacket_buf_write(&req_buf, \"filter %s\",\n+\t\t\t\t args->filter_options.filter_spec);\n \tpacket_buf_flush(&req_buf);\n \tstate_len = req_buf.len;\n \n@@ -969,6 +975,13 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,\n \telse\n \t\tprefer_ofs_delta = 0;\n \n+\tif (server_supports(\"filter\")) {\n+\t\tserver_supports_filtering = 1;\n+\t\tprint_verbose(args, _(\"Server supports filter\"));\n+\t} else if (args->filter_options.choice) {\n+\t\twarning(\"filtering not recognized by server, ignoring\");\n+\t}\n+\n \tif ((agent_feature = server_feature_value(\"agent\", &agent_len))) {\n \t\tagent_supported = 1;\n \t\tif (agent_len)\ndiff --git a/fetch-pack.h b/fetch-pack.h\nindex aeac152..3e224a1 100644\n--- a/fetch-pack.h\n+++ b/fetch-pack.h\n@@ -3,6 +3,7 @@\n \n #include \"string-list.h\"\n #include \"run-command.h\"\n+#include \"list-objects-filter-options.h\"\n \n struct oid_array;\n \n@@ -12,6 +13,7 @@ struct fetch_pack_args {\n \tint depth;\n \tconst char *deepen_since;\n \tconst struct string_list *deepen_not;\n+\tstruct list_objects_filter_options filter_options;\n \tunsigned deepen_relative:1;\n \tunsigned quiet:1;\n \tunsigned keep_pack:1;\ndiff --git a/transport-helper.c b/transport-helper.c\nindex c948d52..0650ca0 100644\n--- a/transport-helper.c\n+++ b/transport-helper.c\n@@ -671,6 +671,11 @@ static int fetch(struct transport *transport,\n \tif (data->transport_options.update_shallow)\n \t\tset_helper_option(transport, \"update-shallow\", \"true\");\n \n+\tif (data->transport_options.filter_options.choice)\n+\t\tset_helper_option(\n+\t\t\ttransport, \"filter\",\n+\t\t\tdata->transport_options.filter_options.filter_spec);\n+\n \tif (data->fetch)\n \t\treturn fetch_with_fetch(transport, nr_heads, to_fetch);\n \ndiff --git a/transport.c b/transport.c\nindex f2fbc6f..cce50f5 100644\n--- a/transport.c\n+++ b/transport.c\n@@ -166,6 +166,9 @@ static int set_git_option(struct git_transport_options *opts,\n \t} else if (!strcmp(name, TRANS_OPT_NO_DEPENDENTS)) {\n \t\topts->no_dependents = !!value;\n \t\treturn 0;\n+\t} else if (!strcmp(name, TRANS_OPT_LIST_OBJECTS_FILTER)) {\n+\t\tparse_list_objects_filter(&opts->filter_options, value);\n+\t\treturn 0;\n \t}\n \treturn 1;\n }\n@@ -236,6 +239,7 @@ static int fetch_refs_via_pack(struct transport *transport,\n \targs.update_shallow = data->options.update_shallow;\n \targs.from_promisor = data->options.from_promisor;\n \targs.no_dependents = data->options.no_dependents;\n+\targs.filter_options = data->options.filter_options;\n \n \tif (!data->got_remote_heads) {\n \t\tconnect_setup(transport, 0);\ndiff --git a/transport.h b/transport.h\nindex c49a8ed..31b1936 100644\n--- a/transport.h\n+++ b/transport.h\n@@ -4,6 +4,7 @@\n #include \"cache.h\"\n #include \"run-command.h\"\n #include \"remote.h\"\n+#include \"list-objects-filter-options.h\"\n \n struct string_list;\n \n@@ -23,6 +24,7 @@ struct git_transport_options {\n \tconst char *uploadpack;\n \tconst char *receivepack;\n \tstruct push_cas_option *cas;\n+\tstruct list_objects_filter_options filter_options;\n };\n \n enum transport_family {\n@@ -221,6 +223,9 @@ void transport_check_allowed(const char *type);\n  */\n #define TRANS_OPT_NO_DEPENDENTS \"no-dependents\"\n \n+/* Filter objects for partial clone and fetch */\n+#define TRANS_OPT_LIST_OBJECTS_FILTER \"filter\"\n+\n /**\n  * Returns 0 if the option was used, non-zero otherwise. Prints a\n  * message to stderr if the option is not used.\n-- \n2.9.3\n\n"},{"id":"334184","messageId":"20171205170254.65293-4-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 03/14] fetch-pack: add --no-filter","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:43Z","receivedAt":"2017-12-05T17:03:22Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jeff Hostetler <jeffhost@microsoft.com>\n\nFixup fetch-pack to accept --no-filter to be consistent with\nrev-list and pack-objects.\n\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n builtin/fetch-pack.c | 4 ++++\n 1 file changed, 4 insertions(+)\n\ndiff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c\nindex 7957807..cbf5035 100644\n--- a/builtin/fetch-pack.c\n+++ b/builtin/fetch-pack.c\n@@ -157,6 +157,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)\n \t\t\tparse_list_objects_filter(&args.filter_options, arg);\n \t\t\tcontinue;\n \t\t}\n+\t\tif (!strcmp(arg, (\"--no-\" CL_ARG__FILTER))) {\n+\t\t\tlist_objects_filter_release(&args.filter_options);\n+\t\t\tcontinue;\n+\t\t}\n \t\tusage(fetch_pack_usage);\n \t}\n \tif (deepen_not.nr)\n-- \n2.9.3\n\n"},{"id":"334185","messageId":"20171205170254.65293-6-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 05/14] fetch: refactor calculation of remote list","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:45Z","receivedAt":"2017-12-05T17:03:23Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jonathan Tan <jonathantanmy@google.com>\n\nSeparate out the calculation of remotes to be fetched from and the\nactual fetching. This will allow us to include an additional step before\nthe actual fetching in a subsequent commit.\n\nSigned-off-by: Jonathan Tan <jonathantanmy@google.com>\n---\n builtin/fetch.c | 14 ++++++++------\n 1 file changed, 8 insertions(+), 6 deletions(-)\n\ndiff --git a/builtin/fetch.c b/builtin/fetch.c\nindex 225c734..1b1f039 100644\n--- a/builtin/fetch.c\n+++ b/builtin/fetch.c\n@@ -1322,7 +1322,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)\n {\n \tint i;\n \tstruct string_list list = STRING_LIST_INIT_DUP;\n-\tstruct remote *remote;\n+\tstruct remote *remote = NULL;\n \tint result = 0;\n \tstruct argv_array argv_gc_auto = ARGV_ARRAY_INIT;\n \n@@ -1367,17 +1367,14 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)\n \t\telse if (argc > 1)\n \t\t\tdie(_(\"fetch --all does not make sense with refspecs\"));\n \t\t(void) for_each_remote(get_one_remote_for_fetch, &list);\n-\t\tresult = fetch_multiple(&list);\n \t} else if (argc == 0) {\n \t\t/* No arguments -- use default remote */\n \t\tremote = remote_get(NULL);\n-\t\tresult = fetch_one(remote, argc, argv);\n \t} else if (multiple) {\n \t\t/* All arguments are assumed to be remotes or groups */\n \t\tfor (i = 0; i < argc; i++)\n \t\t\tif (!add_remote_or_group(argv[i], &list))\n \t\t\t\tdie(_(\"No such remote or remote group: %s\"), argv[i]);\n-\t\tresult = fetch_multiple(&list);\n \t} else {\n \t\t/* Single remote or group */\n \t\t(void) add_remote_or_group(argv[0], &list);\n@@ -1385,14 +1382,19 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)\n \t\t\t/* More than one remote */\n \t\t\tif (argc > 1)\n \t\t\t\tdie(_(\"Fetching a group and specifying refspecs does not make sense\"));\n-\t\t\tresult = fetch_multiple(&list);\n \t\t} else {\n \t\t\t/* Zero or one remotes */\n \t\t\tremote = remote_get(argv[0]);\n-\t\t\tresult = fetch_one(remote, argc-1, argv+1);\n+\t\t\targc--;\n+\t\t\targv++;\n \t\t}\n \t}\n \n+\tif (remote)\n+\t\tresult = fetch_one(remote, argc, argv);\n+\telse\n+\t\tresult = fetch_multiple(&list);\n+\n \tif (!result && (recurse_submodules != RECURSE_SUBMODULES_OFF)) {\n \t\tstruct argv_array options = ARGV_ARRAY_INIT;\n \n-- \n2.9.3\n\n"},{"id":"334186","messageId":"20171205170254.65293-7-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 06/14] fetch: support filters","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:46Z","receivedAt":"2017-12-05T17:03:27Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jeff Hostetler <jeffhost@microsoft.com>\n\nTeach fetch to support filters. This is only allowed for the remote\nconfigured in extensions.partialcloneremote.\n\nSigned-off-by: Jonathan Tan <jonathantanmy@google.com>\n---\n builtin/fetch.c       | 23 +++++++++++++++++++++--\n connected.c           |  1 +\n remote-curl.c         |  6 ++++++\n t/t5500-fetch-pack.sh | 36 ++++++++++++++++++++++++++++++++++++\n 4 files changed, 64 insertions(+), 2 deletions(-)\n\ndiff --git a/builtin/fetch.c b/builtin/fetch.c\nindex 1b1f039..05d0b1a 100644\n--- a/builtin/fetch.c\n+++ b/builtin/fetch.c\n@@ -18,6 +18,7 @@\n #include \"argv-array.h\"\n #include \"utf8.h\"\n #include \"packfile.h\"\n+#include \"list-objects-filter-options.h\"\n \n static const char * const builtin_fetch_usage[] = {\n \tN_(\"git fetch [<options>] [<repository> [<refspec>...]]\"),\n@@ -55,6 +56,7 @@ static int recurse_submodules_default = RECURSE_SUBMODULES_ON_DEMAND;\n static int shown_url = 0;\n static int refmap_alloc, refmap_nr;\n static const char **refmap_array;\n+static struct list_objects_filter_options filter_options;\n \n static int git_fetch_config(const char *k, const char *v, void *cb)\n {\n@@ -160,6 +162,7 @@ static struct option builtin_fetch_options[] = {\n \t\t\tTRANSPORT_FAMILY_IPV4),\n \tOPT_SET_INT('6', \"ipv6\", &family, N_(\"use IPv6 addresses only\"),\n \t\t\tTRANSPORT_FAMILY_IPV6),\n+\tOPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),\n \tOPT_END()\n };\n \n@@ -1044,6 +1047,11 @@ static struct transport *prepare_transport(struct remote *remote, int deepen)\n \t\tset_option(transport, TRANS_OPT_DEEPEN_RELATIVE, \"yes\");\n \tif (update_shallow)\n \t\tset_option(transport, TRANS_OPT_UPDATE_SHALLOW, \"yes\");\n+\tif (filter_options.choice) {\n+\t\tset_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER,\n+\t\t\t   filter_options.filter_spec);\n+\t\tset_option(transport, TRANS_OPT_FROM_PROMISOR, \"1\");\n+\t}\n \treturn transport;\n }\n \n@@ -1328,6 +1336,8 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)\n \n \tpacket_trace_identity(\"fetch\");\n \n+\tfetch_if_missing = 0;\n+\n \t/* Record the command line for the reflog */\n \tstrbuf_addstr(&default_rla, \"fetch\");\n \tfor (i = 1; i < argc; i++)\n@@ -1361,6 +1371,9 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)\n \tif (depth || deepen_since || deepen_not.nr)\n \t\tdeepen = 1;\n \n+\tif (filter_options.choice && !repository_format_partial_clone)\n+\t\tdie(\"--blob-max-bytes can only be used when extensions.partialClone is set\");\n+\n \tif (all) {\n \t\tif (argc == 1)\n \t\t\tdie(_(\"fetch --all does not take a repository argument\"));\n@@ -1390,10 +1403,16 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)\n \t\t}\n \t}\n \n-\tif (remote)\n+\tif (remote) {\n+\t\tif (filter_options.choice &&\n+\t\t    strcmp(remote->name, repository_format_partial_clone))\n+\t\t\tdie(_(\"--blob-max-bytes can only be used with the remote configured in core.partialClone\"));\n \t\tresult = fetch_one(remote, argc, argv);\n-\telse\n+\t} else {\n+\t\tif (filter_options.choice)\n+\t\t\tdie(_(\"--blob-max-bytes can only be used with the remote configured in core.partialClone\"));\n \t\tresult = fetch_multiple(&list);\n+\t}\n \n \tif (!result && (recurse_submodules != RECURSE_SUBMODULES_OFF)) {\n \t\tstruct argv_array options = ARGV_ARRAY_INIT;\ndiff --git a/connected.c b/connected.c\nindex f416b05..a51c01d 100644\n--- a/connected.c\n+++ b/connected.c\n@@ -56,6 +56,7 @@ int check_connected(sha1_iterate_fn fn, void *cb_data,\n \targv_array_push(&rev_list.args,\"rev-list\");\n \targv_array_push(&rev_list.args, \"--objects\");\n \targv_array_push(&rev_list.args, \"--stdin\");\n+\targv_array_push(&rev_list.args, \"--exclude-promisor-objects\");\n \targv_array_push(&rev_list.args, \"--not\");\n \targv_array_push(&rev_list.args, \"--all\");\n \targv_array_push(&rev_list.args, \"--quiet\");\ndiff --git a/remote-curl.c b/remote-curl.c\nindex 4318391..6ec5352 100644\n--- a/remote-curl.c\n+++ b/remote-curl.c\n@@ -24,6 +24,7 @@ struct options {\n \tchar *deepen_since;\n \tstruct string_list deepen_not;\n \tstruct string_list push_options;\n+\tchar *filter;\n \tunsigned progress : 1,\n \t\tcheck_self_contained_and_connected : 1,\n \t\tcloning : 1,\n@@ -165,6 +166,9 @@ static int set_option(const char *name, const char *value)\n \t} else if (!strcmp(name, \"no-dependents\")) {\n \t\toptions.no_dependents = 1;\n \t\treturn 0;\n+\t} else if (!strcmp(name, \"filter\")) {\n+\t\toptions.filter = xstrdup(value);;\n+\t\treturn 0;\n \t} else {\n \t\treturn 1 /* unsupported */;\n \t}\n@@ -834,6 +838,8 @@ static int fetch_git(struct discovery *heads,\n \t\targv_array_push(&args, \"--from-promisor\");\n \tif (options.no_dependents)\n \t\targv_array_push(&args, \"--no-dependents\");\n+\tif (options.filter)\n+\t\targv_array_pushf(&args, \"--filter=%s\", options.filter);\n \targv_array_push(&args, url.buf);\n \n \tfor (i = 0; i < nr_heads; i++) {\ndiff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh\nindex c57916b..65fc7bb 100755\n--- a/t/t5500-fetch-pack.sh\n+++ b/t/t5500-fetch-pack.sh\n@@ -782,4 +782,40 @@ test_expect_success 'filtering by size has no effect if support for it is not ad\n \ttest_i18ngrep \"filtering not recognized by server\" err\n '\n \n+fetch_blob_max_bytes () {\n+\tSERVER=\"$1\"\n+\tURL=\"$2\"\n+\n+\trm -rf \"$SERVER\" client &&\n+\ttest_create_repo \"$SERVER\" &&\n+\ttest_commit -C \"$SERVER\" one &&\n+\ttest_config -C \"$SERVER\" uploadpack.allowfilter 1 &&\n+\n+\tgit clone \"$URL\" client &&\n+\ttest_config -C client extensions.partialclone origin &&\n+\n+\ttest_commit -C \"$SERVER\" two &&\n+\n+\tgit -C client fetch --filter=blob:limit=0 origin HEAD:somewhere &&\n+\n+\t# Ensure that commit is fetched, but blob is not\n+\ttest_config -C client extensions.partialclone \"arbitrary string\" &&\n+\tgit -C client cat-file -e $(git -C \"$SERVER\" rev-parse two) &&\n+\ttest_must_fail git -C client cat-file -e $(git hash-object \"$SERVER/two.t\")\n+}\n+\n+test_expect_success 'fetch with --blob-max-bytes' '\n+\tfetch_blob_max_bytes server server\n+'\n+\n+. \"$TEST_DIRECTORY\"/lib-httpd.sh\n+start_httpd\n+\n+test_expect_success 'fetch with --blob-max-bytes and HTTP' '\n+\tfetch_blob_max_bytes \"$HTTPD_DOCUMENT_ROOT_PATH/server\" \"$HTTPD_URL/smart/server\"\n+'\n+\n+stop_httpd\n+\n+\n test_done\n-- \n2.9.3\n\n"},{"id":"334187","messageId":"20171205170254.65293-13-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 12/14] unpack-trees: batch fetching of missing blobs","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:52Z","receivedAt":"2017-12-05T17:03:36Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jonathan Tan <jonathantanmy@google.com>\n\nWhen running checkout, first prefetch all blobs that are to be updated\nbut are missing. This means that only one pack is downloaded during such\noperations, instead of one per missing blob.\n\nThis operates only on the blob level - if a repository has a missing\ntree, they are still fetched one at a time.\n\nThis does not use the delayed checkout mechanism introduced in commit\n2841e8f (\"convert: add \"status=delayed\" to filter process protocol\",\n2017-06-30) due to significant conceptual differences - in particular,\nfor partial clones, we already know what needs to be fetched based on\nthe contents of the local repo alone, whereas for status=delayed, it is\nthe filter process that tells us what needs to be checked in the end.\n\nSigned-off-by: Jonathan Tan <jonathantanmy@google.com>\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n fetch-object.c   | 26 ++++++++++++++++++++++----\n fetch-object.h   |  5 +++++\n t/t5601-clone.sh | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++\n unpack-trees.c   | 22 ++++++++++++++++++++++\n 4 files changed, 101 insertions(+), 4 deletions(-)\n\ndiff --git a/fetch-object.c b/fetch-object.c\nindex 258fcfa..853624f 100644\n--- a/fetch-object.c\n+++ b/fetch-object.c\n@@ -5,11 +5,10 @@\n #include \"transport.h\"\n #include \"fetch-object.h\"\n \n-void fetch_object(const char *remote_name, const unsigned char *sha1)\n+static void fetch_refs(const char *remote_name, struct ref *ref)\n {\n \tstruct remote *remote;\n \tstruct transport *transport;\n-\tstruct ref *ref;\n \tint original_fetch_if_missing = fetch_if_missing;\n \n \tfetch_if_missing = 0;\n@@ -18,10 +17,29 @@ void fetch_object(const char *remote_name, const unsigned char *sha1)\n \t\tdie(_(\"Remote with no URL\"));\n \ttransport = transport_get(remote, remote->url[0]);\n \n-\tref = alloc_ref(sha1_to_hex(sha1));\n-\thashcpy(ref->old_oid.hash, sha1);\n \ttransport_set_option(transport, TRANS_OPT_FROM_PROMISOR, \"1\");\n \ttransport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, \"1\");\n \ttransport_fetch_refs(transport, ref);\n \tfetch_if_missing = original_fetch_if_missing;\n }\n+\n+void fetch_object(const char *remote_name, const unsigned char *sha1)\n+{\n+\tstruct ref *ref = alloc_ref(sha1_to_hex(sha1));\n+\thashcpy(ref->old_oid.hash, sha1);\n+\tfetch_refs(remote_name, ref);\n+}\n+\n+void fetch_objects(const char *remote_name, const struct oid_array *to_fetch)\n+{\n+\tstruct ref *ref = NULL;\n+\tint i;\n+\n+\tfor (i = 0; i < to_fetch->nr; i++) {\n+\t\tstruct ref *new_ref = alloc_ref(oid_to_hex(&to_fetch->oid[i]));\n+\t\toidcpy(&new_ref->old_oid, &to_fetch->oid[i]);\n+\t\tnew_ref->next = ref;\n+\t\tref = new_ref;\n+\t}\n+\tfetch_refs(remote_name, ref);\n+}\ndiff --git a/fetch-object.h b/fetch-object.h\nindex f371300..4b269d0 100644\n--- a/fetch-object.h\n+++ b/fetch-object.h\n@@ -1,6 +1,11 @@\n #ifndef FETCH_OBJECT_H\n #define FETCH_OBJECT_H\n \n+#include \"sha1-array.h\"\n+\n extern void fetch_object(const char *remote_name, const unsigned char *sha1);\n \n+extern void fetch_objects(const char *remote_name,\n+\t\t\t  const struct oid_array *to_fetch);\n+\n #endif\ndiff --git a/t/t5601-clone.sh b/t/t5601-clone.sh\nindex 6d37c6d..13610b7 100755\n--- a/t/t5601-clone.sh\n+++ b/t/t5601-clone.sh\n@@ -611,6 +611,58 @@ test_expect_success 'partial clone: warn if server does not support object filte\n \ttest_i18ngrep \"filtering not recognized by server\" err\n '\n \n+test_expect_success 'batch missing blob request during checkout' '\n+\trm -rf server client &&\n+\n+\ttest_create_repo server &&\n+\techo a >server/a &&\n+\techo b >server/b &&\n+\tgit -C server add a b &&\n+\n+\tgit -C server commit -m x &&\n+\techo aa >server/a &&\n+\techo bb >server/b &&\n+\tgit -C server add a b &&\n+\tgit -C server commit -m x &&\n+\n+\ttest_config -C server uploadpack.allowfilter 1 &&\n+\ttest_config -C server uploadpack.allowanysha1inwant 1 &&\n+\n+\tgit clone --filter=blob:limit=0 \"file://$(pwd)/server\" client &&\n+\n+\t# Ensure that there is only one negotiation by checking that there is\n+\t# only \"done\" line sent. (\"done\" marks the end of negotiation.)\n+\tGIT_TRACE_PACKET=\"$(pwd)/trace\" git -C client checkout HEAD^ &&\n+\tgrep \"git> done\" trace >done_lines &&\n+\ttest_line_count = 1 done_lines\n+'\n+\n+test_expect_success 'batch missing blob request does not inadvertently try to fetch gitlinks' '\n+\trm -rf server client &&\n+\n+\ttest_create_repo repo_for_submodule &&\n+\ttest_commit -C repo_for_submodule x &&\n+\n+\ttest_create_repo server &&\n+\techo a >server/a &&\n+\techo b >server/b &&\n+\tgit -C server add a b &&\n+\tgit -C server commit -m x &&\n+\n+\techo aa >server/a &&\n+\techo bb >server/b &&\n+\t# Also add a gitlink pointing to an arbitrary repository\n+\tgit -C server submodule add \"$(pwd)/repo_for_submodule\" c &&\n+\tgit -C server add a b c &&\n+\tgit -C server commit -m x &&\n+\n+\ttest_config -C server uploadpack.allowfilter 1 &&\n+\ttest_config -C server uploadpack.allowanysha1inwant 1 &&\n+\n+\t# Make sure that it succeeds\n+\tgit clone --filter=blob:limit=0 \"file://$(pwd)/server\" client\n+'\n+\n . \"$TEST_DIRECTORY\"/lib-httpd.sh\n start_httpd\n \ndiff --git a/unpack-trees.c b/unpack-trees.c\nindex 71b70cc..73a1cdb 100644\n--- a/unpack-trees.c\n+++ b/unpack-trees.c\n@@ -14,6 +14,7 @@\n #include \"dir.h\"\n #include \"submodule.h\"\n #include \"submodule-config.h\"\n+#include \"fetch-object.h\"\n \n /*\n  * Error messages expected by scripts out of plumbing commands such as\n@@ -369,6 +370,27 @@ static int check_updates(struct unpack_trees_options *o)\n \t\tload_gitmodules_file(index, &state);\n \n \tenable_delayed_checkout(&state);\n+\tif (repository_format_partial_clone && o->update && !o->dry_run) {\n+\t\t/*\n+\t\t * Prefetch the objects that are to be checked out in the loop\n+\t\t * below.\n+\t\t */\n+\t\tstruct oid_array to_fetch = OID_ARRAY_INIT;\n+\t\tint fetch_if_missing_store = fetch_if_missing;\n+\t\tfetch_if_missing = 0;\n+\t\tfor (i = 0; i < index->cache_nr; i++) {\n+\t\t\tstruct cache_entry *ce = index->cache[i];\n+\t\t\tif ((ce->ce_flags & CE_UPDATE) &&\n+\t\t\t    !S_ISGITLINK(ce->ce_mode)) {\n+\t\t\t\tif (!has_object_file(&ce->oid))\n+\t\t\t\t\toid_array_append(&to_fetch, &ce->oid);\n+\t\t\t}\n+\t\t}\n+\t\tif (to_fetch.nr)\n+\t\t\tfetch_objects(repository_format_partial_clone,\n+\t\t\t\t      &to_fetch);\n+\t\tfetch_if_missing = fetch_if_missing_store;\n+\t}\n \tfor (i = 0; i < index->cache_nr; i++) {\n \t\tstruct cache_entry *ce = index->cache[i];\n \n-- \n2.9.3\n\n"},{"id":"334188","messageId":"20171205170254.65293-15-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 14/14] t5616: end-to-end tests for partial clone","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:54Z","receivedAt":"2017-12-05T17:03:38Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jeff Hostetler <jeffhost@microsoft.com>\n\nAdditional end-to-end tests for partial clone.\n\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n t/t5616-partial-clone.sh | 125 +++++++++++++++++++++++++++++++++++++++++++++++\n 1 file changed, 125 insertions(+)\n create mode 100755 t/t5616-partial-clone.sh\n\ndiff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh\nnew file mode 100755\nindex 0000000..fa573f8\n--- /dev/null\n+++ b/t/t5616-partial-clone.sh\n@@ -0,0 +1,125 @@\n+#!/bin/sh\n+\n+test_description='git partial clone'\n+\n+. ./test-lib.sh\n+\n+# create a normal \"src\" repo where we can later create new commits.\n+# expect_1.oids will contain a list of the OIDs of all blobs.\n+test_expect_success 'setup normal src repo' '\n+\techo \"{print \\$1}\" >print_1.awk &&\n+\techo \"{print \\$2}\" >print_2.awk &&\n+\n+\tgit init src &&\n+\tfor n in 1 2 3 4\n+\tdo\n+\t\techo \"This is file: $n\" > src/file.$n.txt\n+\t\tgit -C src add file.$n.txt\n+\t\tgit -C src commit -m \"file $n\"\n+\t\tgit -C src ls-files -s file.$n.txt >>temp\n+\tdone &&\n+\tawk -f print_2.awk <temp | sort >expect_1.oids &&\n+\ttest \"$(wc -l <expect_1.oids)\" = \"4\"\n+'\n+\n+# bare clone \"src\" giving \"srv.bare\" for use as our server.\n+test_expect_success 'setup bare clone for server' '\n+\tgit clone --bare \"file://$(pwd)/src\" srv.bare &&\n+\tgit -C srv.bare config --local uploadpack.allowfilter 1 &&\n+\tgit -C srv.bare config --local uploadpack.allowanysha1inwant 1\n+'\n+\n+# do basic partial clone from \"srv.bare\"\n+# confirm we are missing all of the known blobs.\n+# confirm partial clone was registered in the local config.\n+test_expect_success 'do partial clone 1' '\n+\tgit clone --no-checkout --filter=blob:none \"file://$(pwd)/srv.bare\" pc1 &&\n+\tgit -C pc1 rev-list HEAD --quiet --objects --missing=print \\\n+\t\t| awk -f print_1.awk \\\n+\t\t| sed \"s/?//\" \\\n+\t\t| sort >observed.oids &&\n+\ttest_cmp expect_1.oids observed.oids &&\n+\ttest \"$(git -C pc1 config --local core.repositoryformatversion)\" = \"1\" &&\n+\ttest \"$(git -C pc1 config --local extensions.partialclone)\" = \"origin\" &&\n+\ttest \"$(git -C pc1 config --local core.partialclonefilter)\" = \"blob:none\"\n+'\n+\n+# checkout master to force dynamic object fetch of blobs at HEAD.\n+# confirm we now have the expected blobs in a new packfile.\n+test_expect_success 'verify checkout with dynamic object fetch' '\n+\tgit -C pc1 checkout master &&\n+\t(\tcd pc1/.git/objects/pack;\n+\t\tgit verify-pack -v *.pack\n+\t) >temp &&\n+\tgrep blob <temp \\\n+\t\t| awk -f print_1.awk \\\n+\t\t| sort >observed.oids &&\n+\ttest_cmp expect_1.oids observed.oids\n+'\n+\n+# create new commits in \"src\" repo and push to \"srv.bare\".\n+# repack srv.bare just to make it easy to count the blobs.\n+# expect_2.oids will contain a list of the OIDs of all blobs.\n+test_expect_success 'push new commits to server' '\n+\tgit -C src remote add srv \"file://$(pwd)/srv.bare\" &&\n+\tfor x in a b c d\n+\tdo\n+\t\techo \"Mod $x\" >>src/file.1.txt\n+\t\tgit -C src add file.1.txt\n+\t\tgit -C src commit -m \"mod $x\"\n+\tdone &&\n+\tgit -C src push -u srv master &&\n+\tgit -C srv.bare repack &&\n+\t(\tcd srv.bare/objects/pack;\n+\t\tgit verify-pack -v *.pack\n+\t) >temp &&\n+\tgrep blob <temp \\\n+\t\t| awk -f print_1.awk \\\n+\t\t| sort >expect_2.oids &&\n+\ttest \"$(wc -l <expect_2.oids)\" = \"8\" &&\n+\tgit -C src blame master -- file.1.txt >expect.blame\n+'\n+\n+# fetch in the partial clone repo from the server (the promisor remote).\n+# verify that fetch was a \"partial fetch\".\n+# [] that it inherited the filter settings\n+# [] that is DOES NOT have the new blobs.\n+test_expect_success 'partial fetch inherits filter settings' '\n+\tgit -C pc1 fetch origin &&\n+\t(\tcd pc1/.git/objects/pack;\n+\t\tgit verify-pack -v *.pack\n+\t) >temp &&\n+\tgrep blob <temp \\\n+\t\t| awk -f print_1.awk \\\n+\t\t| sort >observed.oids &&\n+\ttest_cmp expect_1.oids observed.oids\n+'\n+\n+# force dynamic object fetch using diff.\n+# we should only get 1 new blob (for the file in origin/master).\n+# it should be in a new packfile (since the promisor boundary is\n+# currently a packfile, it should not get unpacked upon receipt.)\n+test_expect_success 'verify diff causes dynamic object fetch' '\n+\ttest \"$(wc -l <observed.oids)\" = \"4\" &&\n+\t\tcat observed.oids &&\n+\tgit -C pc1 diff master..origin/master -- file.1.txt &&\n+\t(\tcd pc1/.git/objects/pack;\n+\t\tgit verify-pack -v *.pack\n+\t) >temp &&\n+\tgrep blob <temp \\\n+\t\t| awk -f print_1.awk \\\n+\t\t| sort >observed.oids &&\n+\t\tcat observed.oids &&\n+\ttest \"$(wc -l <observed.oids)\" = \"4\"\n+'\n+\n+# force dynamic object fetch using blame.\n+# we should get the intermediate blobs for the file.\n+# we may get multiple packfiles (one for each blob/commit) or\n+# we may get a single new packfile, but we don't care.\n+test_expect_success 'verify blame causes dynamic object fetch' '\n+\tgit -C pc1 blame origin/master -- file.1.txt >observed.blame &&\n+\ttest_cmp expect.blame observed.blame\n+'\n+\n+test_done\n-- \n2.9.3\n\n"},{"id":"334189","messageId":"20171205170254.65293-14-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 13/14] fetch-pack: restore save_commit_buffer after use","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:53Z","receivedAt":"2017-12-05T17:03:40Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jonathan Tan <jonathantanmy@google.com>\n\nIn fetch-pack, the global variable save_commit_buffer is set to 0, but\nnot restored to its original value after use.\n\nIn particular, if show_log() (in log-tree.c) is invoked after\nfetch_pack() in the same process, show_log() will return before printing\nout the commit message (because the invocation to\nget_cached_commit_buffer() returns NULL, because the commit buffer was\nnot saved). I discovered this when attempting to run \"git log -S\" in a\npartial clone, triggering the case where revision walking lazily loads\nmissing objects.\n\nTherefore, restore save_commit_buffer to its original value after use.\n\nAn alternative to solve the problem I had is to replace\nget_cached_commit_buffer() with get_commit_buffer(). That invocation was\nintroduced in commit a97934d (\"use get_cached_commit_buffer where\nappropriate\", 2014-06-13) to replace \"commit->buffer\" introduced in\ncommit 3131b71 (\"Add \"--show-all\" revision walker flag for debugging\",\n2008-02-13). In the latter commit, the commit author seems to be\ndeciding between not showing an unparsed commit at all and showing an\nunparsed commit without the message (which is what the commit does), and\ndid not mention parsing the unparsed commit, so I prefer to preserve the\nexisting behavior.\n\nSigned-off-by: Jonathan Tan <jonathantanmy@google.com>\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n fetch-pack.c | 4 ++++\n 1 file changed, 4 insertions(+)\n\ndiff --git a/fetch-pack.c b/fetch-pack.c\nindex 3c5f064..773453c 100644\n--- a/fetch-pack.c\n+++ b/fetch-pack.c\n@@ -717,6 +717,7 @@ static int everything_local(struct fetch_pack_args *args,\n {\n \tstruct ref *ref;\n \tint retval;\n+\tint old_save_commit_buffer = save_commit_buffer;\n \ttimestamp_t cutoff = 0;\n \n \tsave_commit_buffer = 0;\n@@ -786,6 +787,9 @@ static int everything_local(struct fetch_pack_args *args,\n \t\tprint_verbose(args, _(\"already have %s (%s)\"), oid_to_hex(remote),\n \t\t\t      ref->name);\n \t}\n+\n+\tsave_commit_buffer = old_save_commit_buffer;\n+\n \treturn retval;\n }\n \n-- \n2.9.3\n\n"},{"id":"334190","messageId":"20171205170254.65293-12-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 11/14] clone: partial clone","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:51Z","receivedAt":"2017-12-05T17:03:43Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jonathan Tan <jonathantanmy@google.com>\n\nSigned-off-by: Jonathan Tan <jonathantanmy@google.com>\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n builtin/clone.c  | 22 ++++++++++++++++++++--\n t/t5601-clone.sh | 49 +++++++++++++++++++++++++++++++++++++++++++++++++\n 2 files changed, 69 insertions(+), 2 deletions(-)\n\ndiff --git a/builtin/clone.c b/builtin/clone.c\nindex dbddd98..f519bd4 100644\n--- a/builtin/clone.c\n+++ b/builtin/clone.c\n@@ -26,6 +26,7 @@\n #include \"run-command.h\"\n #include \"connected.h\"\n #include \"packfile.h\"\n+#include \"list-objects-filter-options.h\"\n \n /*\n  * Overall FIXMEs:\n@@ -60,6 +61,7 @@ static struct string_list option_optional_reference = STRING_LIST_INIT_NODUP;\n static int option_dissociate;\n static int max_jobs = -1;\n static struct string_list option_recurse_submodules = STRING_LIST_INIT_NODUP;\n+static struct list_objects_filter_options filter_options;\n \n static int recurse_submodules_cb(const struct option *opt,\n \t\t\t\t const char *arg, int unset)\n@@ -135,6 +137,7 @@ static struct option builtin_clone_options[] = {\n \t\t\tTRANSPORT_FAMILY_IPV4),\n \tOPT_SET_INT('6', \"ipv6\", &family, N_(\"use IPv6 addresses only\"),\n \t\t\tTRANSPORT_FAMILY_IPV6),\n+\tOPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),\n \tOPT_END()\n };\n \n@@ -886,6 +889,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix)\n \tstruct refspec *refspec;\n \tconst char *fetch_pattern;\n \n+\tfetch_if_missing = 0;\n+\n \tpacket_trace_identity(\"clone\");\n \targc = parse_options(argc, argv, prefix, builtin_clone_options,\n \t\t\t     builtin_clone_usage, 0);\n@@ -1073,6 +1078,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix)\n \t\t\twarning(_(\"--shallow-since is ignored in local clones; use file:// instead.\"));\n \t\tif (option_not.nr)\n \t\t\twarning(_(\"--shallow-exclude is ignored in local clones; use file:// instead.\"));\n+\t\tif (filter_options.choice)\n+\t\t\twarning(_(\"--filter is ignored in local clones; use file:// instead.\"));\n \t\tif (!access(mkpath(\"%s/shallow\", path), F_OK)) {\n \t\t\tif (option_local > 0)\n \t\t\t\twarning(_(\"source repository is shallow, ignoring --local\"));\n@@ -1104,7 +1111,13 @@ int cmd_clone(int argc, const char **argv, const char *prefix)\n \t\ttransport_set_option(transport, TRANS_OPT_UPLOADPACK,\n \t\t\t\t     option_upload_pack);\n \n-\tif (transport->smart_options && !deepen)\n+\tif (filter_options.choice) {\n+\t\ttransport_set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER,\n+\t\t\t\t     filter_options.filter_spec);\n+\t\ttransport_set_option(transport, TRANS_OPT_FROM_PROMISOR, \"1\");\n+\t}\n+\n+\tif (transport->smart_options && !deepen && !filter_options.choice)\n \t\ttransport->smart_options->check_self_contained_and_connected = 1;\n \n \trefs = transport_get_remote_refs(transport);\n@@ -1164,13 +1177,17 @@ int cmd_clone(int argc, const char **argv, const char *prefix)\n \twrite_refspec_config(src_ref_prefix, our_head_points_at,\n \t\t\tremote_head_points_at, &branch_top);\n \n+\tif (filter_options.choice)\n+\t\tpartial_clone_register(\"origin\", &filter_options);\n+\n \tif (is_local)\n \t\tclone_local(path, git_dir);\n \telse if (refs && complete_refs_before_fetch)\n \t\ttransport_fetch_refs(transport, mapped_refs);\n \n \tupdate_remote_refs(refs, mapped_refs, remote_head_points_at,\n-\t\t\t   branch_top.buf, reflog_msg.buf, transport, !is_local);\n+\t\t\t   branch_top.buf, reflog_msg.buf, transport,\n+\t\t\t   !is_local && !filter_options.choice);\n \n \tupdate_head(our_head_points_at, remote_head, reflog_msg.buf);\n \n@@ -1191,6 +1208,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)\n \t}\n \n \tjunk_mode = JUNK_LEAVE_REPO;\n+\tfetch_if_missing = 1;\n \terr = checkout(submodule_progress);\n \n \tstrbuf_release(&reflog_msg);\ndiff --git a/t/t5601-clone.sh b/t/t5601-clone.sh\nindex 9c56f77..6d37c6d 100755\n--- a/t/t5601-clone.sh\n+++ b/t/t5601-clone.sh\n@@ -571,4 +571,53 @@ test_expect_success 'GIT_TRACE_PACKFILE produces a usable pack' '\n \tgit -C replay.git index-pack -v --stdin <tmp.pack\n '\n \n+partial_clone () {\n+\t       SERVER=\"$1\" &&\n+\t       URL=\"$2\" &&\n+\n+\trm -rf \"$SERVER\" client &&\n+\ttest_create_repo \"$SERVER\" &&\n+\ttest_commit -C \"$SERVER\" one &&\n+\tHASH1=$(git hash-object \"$SERVER/one.t\") &&\n+\tgit -C \"$SERVER\" revert HEAD &&\n+\ttest_commit -C \"$SERVER\" two &&\n+\tHASH2=$(git hash-object \"$SERVER/two.t\") &&\n+\ttest_config -C \"$SERVER\" uploadpack.allowfilter 1 &&\n+\ttest_config -C \"$SERVER\" uploadpack.allowanysha1inwant 1 &&\n+\n+\tgit clone --filter=blob:limit=0 \"$URL\" client &&\n+\n+\tgit -C client fsck &&\n+\n+\t# Ensure that unneeded blobs are not inadvertently fetched.\n+\ttest_config -C client extensions.partialclone \"not a remote\" &&\n+\ttest_must_fail git -C client cat-file -e \"$HASH1\" &&\n+\n+\t# But this blob was fetched, because clone performs an initial checkout\n+\tgit -C client cat-file -e \"$HASH2\"\n+}\n+\n+test_expect_success 'partial clone' '\n+\tpartial_clone server \"file://$(pwd)/server\"\n+'\n+\n+test_expect_success 'partial clone: warn if server does not support object filtering' '\n+\trm -rf server client &&\n+\ttest_create_repo server &&\n+\ttest_commit -C server one &&\n+\n+\tgit clone --filter=blob:limit=0 \"file://$(pwd)/server\" client 2> err &&\n+\n+\ttest_i18ngrep \"filtering not recognized by server\" err\n+'\n+\n+. \"$TEST_DIRECTORY\"/lib-httpd.sh\n+start_httpd\n+\n+test_expect_success 'partial clone using HTTP' '\n+\tpartial_clone \"$HTTPD_DOCUMENT_ROOT_PATH/server\" \"$HTTPD_URL/smart/server\"\n+'\n+\n+stop_httpd\n+\n test_done\n-- \n2.9.3\n\n"},{"id":"334191","messageId":"20171205170254.65293-11-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 10/14] partial-clone: define partial clone settings in config","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:50Z","receivedAt":"2017-12-05T17:03:45Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jeff Hostetler <jeffhost@microsoft.com>\n\nCreate get and set routines for \"partial clone\" config settings.\nThese will be used in a future commit by clone and fetch to\nremember the promisor remote and the default filter-spec.\n\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n cache.h                       |  1 +\n config.c                      |  5 +++\n environment.c                 |  1 +\n list-objects-filter-options.c | 90 +++++++++++++++++++++++++++++++++++--------\n list-objects-filter-options.h |  6 +++\n 5 files changed, 88 insertions(+), 15 deletions(-)\n\ndiff --git a/cache.h b/cache.h\nindex 6980072..bccc510 100644\n--- a/cache.h\n+++ b/cache.h\n@@ -861,6 +861,7 @@ extern int grafts_replace_parents;\n #define GIT_REPO_VERSION_READ 1\n extern int repository_format_precious_objects;\n extern char *repository_format_partial_clone;\n+extern const char *core_partial_clone_filter_default;\n \n struct repository_format {\n \tint version;\ndiff --git a/config.c b/config.c\nindex adb7d7a..adeee04 100644\n--- a/config.c\n+++ b/config.c\n@@ -1241,6 +1241,11 @@ static int git_default_core_config(const char *var, const char *value)\n \t\treturn 0;\n \t}\n \n+\tif (!strcmp(var, \"core.partialclonefilter\")) {\n+\t\treturn git_config_string(&core_partial_clone_filter_default,\n+\t\t\t\t\t var, value);\n+\t}\n+\n \t/* Add other config variables here and to Documentation/config.txt. */\n \treturn 0;\n }\ndiff --git a/environment.c b/environment.c\nindex e52aab3..7537565 100644\n--- a/environment.c\n+++ b/environment.c\n@@ -28,6 +28,7 @@ int warn_on_object_refname_ambiguity = 1;\n int ref_paranoia = -1;\n int repository_format_precious_objects;\n char *repository_format_partial_clone;\n+const char *core_partial_clone_filter_default;\n const char *git_commit_encoding;\n const char *git_log_output_encoding;\n const char *apply_default_whitespace;\ndiff --git a/list-objects-filter-options.c b/list-objects-filter-options.c\nindex 4c5b34e..5c47e2b 100644\n--- a/list-objects-filter-options.c\n+++ b/list-objects-filter-options.c\n@@ -21,29 +21,36 @@\n  * subordinate commands when necessary.  We also \"intern\" the arg for\n  * the convenience of the current command.\n  */\n-int parse_list_objects_filter(struct list_objects_filter_options *filter_options,\n-\t\t\t      const char *arg)\n+static int gently_parse_list_objects_filter(\n+\tstruct list_objects_filter_options *filter_options,\n+\tconst char *arg,\n+\tstruct strbuf *errbuf)\n {\n \tconst char *v0;\n \n-\tif (filter_options->choice)\n-\t\tdie(_(\"multiple object filter types cannot be combined\"));\n+\tif (filter_options->choice) {\n+\t\tif (errbuf) {\n+\t\t\tstrbuf_init(errbuf, 0);\n+\t\t\tstrbuf_addstr(\n+\t\t\t\terrbuf,\n+\t\t\t\t_(\"multiple filter-specs cannot be combined\"));\n+\t\t}\n+\t\treturn 1;\n+\t}\n \n \tfilter_options->filter_spec = strdup(arg);\n \n \tif (!strcmp(arg, \"blob:none\")) {\n \t\tfilter_options->choice = LOFC_BLOB_NONE;\n \t\treturn 0;\n-\t}\n \n-\tif (skip_prefix(arg, \"blob:limit=\", &v0)) {\n-\t\tif (!git_parse_ulong(v0, &filter_options->blob_limit_value))\n-\t\t\tdie(_(\"invalid filter-spec expression '%s'\"), arg);\n-\t\tfilter_options->choice = LOFC_BLOB_LIMIT;\n-\t\treturn 0;\n-\t}\n+\t} else if (skip_prefix(arg, \"blob:limit=\", &v0)) {\n+\t\tif (git_parse_ulong(v0, &filter_options->blob_limit_value)) {\n+\t\t\tfilter_options->choice = LOFC_BLOB_LIMIT;\n+\t\t\treturn 0;\n+\t\t}\n \n-\tif (skip_prefix(arg, \"sparse:oid=\", &v0)) {\n+\t} else if (skip_prefix(arg, \"sparse:oid=\", &v0)) {\n \t\tstruct object_context oc;\n \t\tstruct object_id sparse_oid;\n \n@@ -57,15 +64,27 @@ int parse_list_objects_filter(struct list_objects_filter_options *filter_options\n \t\t\tfilter_options->sparse_oid_value = oiddup(&sparse_oid);\n \t\tfilter_options->choice = LOFC_SPARSE_OID;\n \t\treturn 0;\n-\t}\n \n-\tif (skip_prefix(arg, \"sparse:path=\", &v0)) {\n+\t} else if (skip_prefix(arg, \"sparse:path=\", &v0)) {\n \t\tfilter_options->choice = LOFC_SPARSE_PATH;\n \t\tfilter_options->sparse_path_value = strdup(v0);\n \t\treturn 0;\n \t}\n \n-\tdie(_(\"invalid filter-spec expression '%s'\"), arg);\n+\tif (errbuf) {\n+\t\tstrbuf_init(errbuf, 0);\n+\t\tstrbuf_addf(errbuf, \"invalid filter-spec '%s'\", arg);\n+\t}\n+\tmemset(filter_options, 0, sizeof(*filter_options));\n+\treturn 1;\n+}\n+\n+int parse_list_objects_filter(struct list_objects_filter_options *filter_options,\n+\t\t\t      const char *arg)\n+{\n+\tstruct strbuf buf = STRBUF_INIT;\n+\tif (gently_parse_list_objects_filter(filter_options, arg, &buf))\n+\t\tdie(\"%s\", buf.buf);\n \treturn 0;\n }\n \n@@ -90,3 +109,44 @@ void list_objects_filter_release(\n \tfree(filter_options->sparse_path_value);\n \tmemset(filter_options, 0, sizeof(*filter_options));\n }\n+\n+void partial_clone_register(\n+\tconst char *remote,\n+\tconst struct list_objects_filter_options *filter_options)\n+{\n+\t/*\n+\t * Record the name of the partial clone remote in the\n+\t * config and in the global variable -- the latter is\n+\t * used throughout to indicate that partial clone is\n+\t * enabled and to expect missing objects.\n+\t */\n+\tif (repository_format_partial_clone &&\n+\t    *repository_format_partial_clone &&\n+\t    strcmp(remote, repository_format_partial_clone))\n+\t\tdie(_(\"cannot change partial clone promisor remote\"));\n+\n+\tgit_config_set(\"core.repositoryformatversion\", \"1\");\n+\tgit_config_set(\"extensions.partialclone\", remote);\n+\n+\trepository_format_partial_clone = xstrdup(remote);\n+\n+\t/*\n+\t * Record the initial filter-spec in the config as\n+\t * the default for subsequent fetches from this remote.\n+\t */\n+\tcore_partial_clone_filter_default =\n+\t\txstrdup(filter_options->filter_spec);\n+\tgit_config_set(\"core.partialclonefilter\",\n+\t\t       core_partial_clone_filter_default);\n+}\n+\n+void partial_clone_get_default_filter_spec(\n+\tstruct list_objects_filter_options *filter_options)\n+{\n+\t/*\n+\t * Parse default value, but silently ignore it if it is invalid.\n+\t */\n+\tgently_parse_list_objects_filter(filter_options,\n+\t\t\t\t\t core_partial_clone_filter_default,\n+\t\t\t\t\t NULL);\n+}\ndiff --git a/list-objects-filter-options.h b/list-objects-filter-options.h\nindex eea44a1..1143539 100644\n--- a/list-objects-filter-options.h\n+++ b/list-objects-filter-options.h\n@@ -58,4 +58,10 @@ int opt_parse_list_objects_filter(const struct option *opt,\n void list_objects_filter_release(\n \tstruct list_objects_filter_options *filter_options);\n \n+void partial_clone_register(\n+\tconst char *remote,\n+\tconst struct list_objects_filter_options *filter_options);\n+void partial_clone_get_default_filter_spec(\n+\tstruct list_objects_filter_options *filter_options);\n+\n #endif /* LIST_OBJECTS_FILTER_OPTIONS_H */\n-- \n2.9.3\n\n"},{"id":"334192","messageId":"20171205170254.65293-10-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 09/14] fixup: connected: conditionally pass --exclude-promisor-objects to rev-list","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:49Z","receivedAt":"2017-12-05T17:03:47Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jeff Hostetler <jeffhost@microsoft.com>\n\nTeach connected.c to only pass --exclude-promisor-objects to rev-list when\npartial clone is enabled.\n\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n connected.c | 3 ++-\n 1 file changed, 2 insertions(+), 1 deletion(-)\n\ndiff --git a/connected.c b/connected.c\nindex a51c01d..3a5bd67 100644\n--- a/connected.c\n+++ b/connected.c\n@@ -56,7 +56,8 @@ int check_connected(sha1_iterate_fn fn, void *cb_data,\n \targv_array_push(&rev_list.args,\"rev-list\");\n \targv_array_push(&rev_list.args, \"--objects\");\n \targv_array_push(&rev_list.args, \"--stdin\");\n-\targv_array_push(&rev_list.args, \"--exclude-promisor-objects\");\n+\tif (repository_format_partial_clone)\n+\t\targv_array_push(&rev_list.args, \"--exclude-promisor-objects\");\n \targv_array_push(&rev_list.args, \"--not\");\n \targv_array_push(&rev_list.args, \"--all\");\n \targv_array_push(&rev_list.args, \"--quiet\");\n-- \n2.9.3\n\n"},{"id":"334193","messageId":"20171205170254.65293-9-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 08/14] fixup: fetch: update --blob-max-bytes to --fitler","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:48Z","receivedAt":"2017-12-05T17:03:50Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jeff Hostetler <jeffhost@microsoft.com>\n\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n t/t5500-fetch-pack.sh | 10 +++++-----\n 1 file changed, 5 insertions(+), 5 deletions(-)\n\ndiff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh\nindex 65fc7bb..ec9ba9b 100755\n--- a/t/t5500-fetch-pack.sh\n+++ b/t/t5500-fetch-pack.sh\n@@ -782,7 +782,7 @@ test_expect_success 'filtering by size has no effect if support for it is not ad\n \ttest_i18ngrep \"filtering not recognized by server\" err\n '\n \n-fetch_blob_max_bytes () {\n+fetch_filter_blob_limit_zero () {\n \tSERVER=\"$1\"\n \tURL=\"$2\"\n \n@@ -804,15 +804,15 @@ fetch_blob_max_bytes () {\n \ttest_must_fail git -C client cat-file -e $(git hash-object \"$SERVER/two.t\")\n }\n \n-test_expect_success 'fetch with --blob-max-bytes' '\n-\tfetch_blob_max_bytes server server\n+test_expect_success 'fetch with --filter=blob:limit=0' '\n+\tfetch_filter_blob_limit_zero server server\n '\n \n . \"$TEST_DIRECTORY\"/lib-httpd.sh\n start_httpd\n \n-test_expect_success 'fetch with --blob-max-bytes and HTTP' '\n-\tfetch_blob_max_bytes \"$HTTPD_DOCUMENT_ROOT_PATH/server\" \"$HTTPD_URL/smart/server\"\n+test_expect_success 'fetch with --filter=blob:limit=0 and HTTP' '\n+\tfetch_filter_blob_limit_zero \"$HTTPD_DOCUMENT_ROOT_PATH/server\" \"$HTTPD_URL/smart/server\"\n '\n \n stop_httpd\n-- \n2.9.3\n\n"},{"id":"334194","messageId":"20171205170254.65293-8-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 07/14] fixup: fetch: update error messages from --blob-max-bytes to --filter","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:47Z","receivedAt":"2017-12-05T17:03:56Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jeff Hostetler <jeffhost@microsoft.com>\n\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n builtin/fetch.c | 6 +++---\n 1 file changed, 3 insertions(+), 3 deletions(-)\n\ndiff --git a/builtin/fetch.c b/builtin/fetch.c\nindex 05d0b1a..14aab71 100644\n--- a/builtin/fetch.c\n+++ b/builtin/fetch.c\n@@ -1372,7 +1372,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)\n \t\tdeepen = 1;\n \n \tif (filter_options.choice && !repository_format_partial_clone)\n-\t\tdie(\"--blob-max-bytes can only be used when extensions.partialClone is set\");\n+\t\tdie(\"--filter can only be used when extensions.partialClone is set\");\n \n \tif (all) {\n \t\tif (argc == 1)\n@@ -1406,11 +1406,11 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)\n \tif (remote) {\n \t\tif (filter_options.choice &&\n \t\t    strcmp(remote->name, repository_format_partial_clone))\n-\t\t\tdie(_(\"--blob-max-bytes can only be used with the remote configured in core.partialClone\"));\n+\t\t\tdie(_(\"--filter can only be used with the remote configured in core.partialClone\"));\n \t\tresult = fetch_one(remote, argc, argv);\n \t} else {\n \t\tif (filter_options.choice)\n-\t\t\tdie(_(\"--blob-max-bytes can only be used with the remote configured in core.partialClone\"));\n+\t\t\tdie(_(\"--filter can only be used with the remote configured in core.partialClone\"));\n \t\tresult = fetch_multiple(&list);\n \t}\n \n-- \n2.9.3\n\n"},{"id":"334195","messageId":"20171205170254.65293-5-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 04/14] fetch-pack: test support excluding large blobs","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:44Z","receivedAt":"2017-12-05T17:04:00Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jonathan Tan <jonathantanmy@google.com>\n\nCreated tests to verify fetch-pack and upload-pack support\nfor excluding large blobs using --filter=blobs:limit=<n>\nparameter.\n\nSigned-off-by: Jonathan Tan <jonathantanmy@google.com>\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n t/t5500-fetch-pack.sh | 27 +++++++++++++++++++++++++++\n upload-pack.c         | 13 +++++++++----\n 2 files changed, 36 insertions(+), 4 deletions(-)\n\ndiff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh\nindex 80a1a32..c57916b 100755\n--- a/t/t5500-fetch-pack.sh\n+++ b/t/t5500-fetch-pack.sh\n@@ -755,4 +755,31 @@ test_expect_success 'fetching deepen' '\n \t)\n '\n \n+test_expect_success 'filtering by size' '\n+\trm -rf server client &&\n+\ttest_create_repo server &&\n+\ttest_commit -C server one &&\n+\ttest_config -C server uploadpack.allowfilter 1 &&\n+\n+\ttest_create_repo client &&\n+\tgit -C client fetch-pack --filter=blob:limit=0 ../server HEAD &&\n+\n+\t# Ensure that object is not inadvertently fetched\n+\ttest_must_fail git -C client cat-file -e $(git hash-object server/one.t)\n+'\n+\n+test_expect_success 'filtering by size has no effect if support for it is not advertised' '\n+\trm -rf server client &&\n+\ttest_create_repo server &&\n+\ttest_commit -C server one &&\n+\n+\ttest_create_repo client &&\n+\tgit -C client fetch-pack --filter=blob:limit=0 ../server HEAD 2> err &&\n+\n+\t# Ensure that object is fetched\n+\tgit -C client cat-file -e $(git hash-object server/one.t) &&\n+\n+\ttest_i18ngrep \"filtering not recognized by server\" err\n+'\n+\n test_done\ndiff --git a/upload-pack.c b/upload-pack.c\nindex e6d38b9..15b6605 100644\n--- a/upload-pack.c\n+++ b/upload-pack.c\n@@ -139,10 +139,15 @@ static void create_pack_file(void)\n \tif (use_include_tag)\n \t\targv_array_push(&pack_objects.args, \"--include-tag\");\n \tif (filter_options.filter_spec) {\n-\t\tstruct strbuf buf = STRBUF_INIT;\n-\t\tsq_quote_buf(&buf, filter_options.filter_spec);\n-\t\targv_array_pushf(&pack_objects.args, \"--filter=%s\", buf.buf);\n-\t\tstrbuf_release(&buf);\n+\t\tif (pack_objects.use_shell) {\n+\t\t\tstruct strbuf buf = STRBUF_INIT;\n+\t\t\tsq_quote_buf(&buf, filter_options.filter_spec);\n+\t\t\targv_array_pushf(&pack_objects.args, \"--filter=%s\", buf.buf);\n+\t\t\tstrbuf_release(&buf);\n+\t\t} else {\n+\t\t\targv_array_pushf(&pack_objects.args, \"--filter=%s\",\n+\t\t\t\t\t filter_options.filter_spec);\n+\t\t}\n \t}\n \n \tpack_objects.in = -1;\n-- \n2.9.3\n\n"},{"id":"334196","messageId":"20171205170254.65293-2-git@jeffhostetler.com","threadId":"47381","inReplyTo":"20171205170254.65293-1-git@jeffhostetler.com","subject":"[PATCH v6 01/14] upload-pack: add object filtering for partial clone","fromName":"Jeff Hostetler","fromEmail":"git@jeffhostetler.com","sentAt":"2017-12-05T17:02:41Z","receivedAt":"2017-12-05T17:04:04Z","isPatch":true,"sender":{"key":"git@jeffhostetler.com","avatar":null},"body":"From: Jeff Hostetler <jeffhost@microsoft.com>\n\nTeach upload-pack to negotiate object filtering over the protocol and\nto send filter parameters to pack-objects.  This is intended for partial\nclone and fetch.\n\nThe idea to make upload-pack configurable using uploadpack.allowFilter\ncomes from Jonathan Tan's work in [1].\n\n[1] https://public-inbox.org/git/f211093280b422c32cc1b7034130072f35c5ed51.1506714999.git.jonathantanmy@google.com/\n\nSigned-off-by: Jeff Hostetler <jeffhost@microsoft.com>\n---\n Documentation/config.txt                          |  4 ++++\n Documentation/technical/pack-protocol.txt         |  8 +++++++\n Documentation/technical/protocol-capabilities.txt |  8 +++++++\n upload-pack.c                                     | 26 ++++++++++++++++++++++-\n 4 files changed, 45 insertions(+), 1 deletion(-)\n\ndiff --git a/Documentation/config.txt b/Documentation/config.txt\nindex 1ac0ae6..e528210 100644\n--- a/Documentation/config.txt\n+++ b/Documentation/config.txt\n@@ -3268,6 +3268,10 @@ uploadpack.packObjectsHook::\n \twas run. I.e., `upload-pack` will feed input intended for\n \t`pack-objects` to the hook, and expects a completed packfile on\n \tstdout.\n+\n+uploadpack.allowFilter::\n+\tIf this option is set, `upload-pack` will advertise partial\n+\tclone and partial fetch object filtering.\n +\n Note that this configuration variable is ignored if it is seen in the\n repository-level config (this is a safety measure against fetching from\ndiff --git a/Documentation/technical/pack-protocol.txt b/Documentation/technical/pack-protocol.txt\nindex ed1eae8..a43a113 100644\n--- a/Documentation/technical/pack-protocol.txt\n+++ b/Documentation/technical/pack-protocol.txt\n@@ -212,6 +212,7 @@ out of what the server said it could do with the first 'want' line.\n   upload-request    =  want-list\n \t\t       *shallow-line\n \t\t       *1depth-request\n+\t\t       [filter-request]\n \t\t       flush-pkt\n \n   want-list         =  first-want\n@@ -227,6 +228,8 @@ out of what the server said it could do with the first 'want' line.\n   additional-want   =  PKT-LINE(\"want\" SP obj-id)\n \n   depth             =  1*DIGIT\n+\n+  filter-request    =  PKT-LINE(\"filter\" SP filter-spec)\n ----\n \n Clients MUST send all the obj-ids it wants from the reference\n@@ -249,6 +252,11 @@ complete those commits. Commits whose parents are not received as a\n result are defined as shallow and marked as such in the server. This\n information is sent back to the client in the next step.\n \n+The client can optionally request that pack-objects omit various\n+objects from the packfile using one of several filtering techniques.\n+These are intended for use with partial clone and partial fetch\n+operations.  See `rev-list` for possible \"filter-spec\" values.\n+\n Once all the 'want's and 'shallow's (and optional 'deepen') are\n transferred, clients MUST send a flush-pkt, to tell the server side\n that it is done sending the list.\ndiff --git a/Documentation/technical/protocol-capabilities.txt b/Documentation/technical/protocol-capabilities.txt\nindex 26dcc6f..332d209 100644\n--- a/Documentation/technical/protocol-capabilities.txt\n+++ b/Documentation/technical/protocol-capabilities.txt\n@@ -309,3 +309,11 @@ to accept a signed push certificate, and asks the <nonce> to be\n included in the push certificate.  A send-pack client MUST NOT\n send a push-cert packet unless the receive-pack server advertises\n this capability.\n+\n+filter\n+------\n+\n+If the upload-pack server advertises the 'filter' capability,\n+fetch-pack may send \"filter\" commands to request a partial clone\n+or partial fetch and request that the server omit various objects\n+from the packfile.\ndiff --git a/upload-pack.c b/upload-pack.c\nindex e25f725..e6d38b9 100644\n--- a/upload-pack.c\n+++ b/upload-pack.c\n@@ -10,6 +10,8 @@\n #include \"diff.h\"\n #include \"revision.h\"\n #include \"list-objects.h\"\n+#include \"list-objects-filter.h\"\n+#include \"list-objects-filter-options.h\"\n #include \"run-command.h\"\n #include \"connect.h\"\n #include \"sigchain.h\"\n@@ -18,6 +20,7 @@\n #include \"parse-options.h\"\n #include \"argv-array.h\"\n #include \"prio-queue.h\"\n+#include \"quote.h\"\n \n static const char * const upload_pack_usage[] = {\n \tN_(\"git upload-pack [<options>] <dir>\"),\n@@ -64,6 +67,10 @@ static int advertise_refs;\n static int stateless_rpc;\n static const char *pack_objects_hook;\n \n+static int filter_capability_requested;\n+static int filter_advertise;\n+static struct list_objects_filter_options filter_options;\n+\n static void reset_timeout(void)\n {\n \talarm(timeout);\n@@ -131,6 +138,12 @@ static void create_pack_file(void)\n \t\targv_array_push(&pack_objects.args, \"--delta-base-offset\");\n \tif (use_include_tag)\n \t\targv_array_push(&pack_objects.args, \"--include-tag\");\n+\tif (filter_options.filter_spec) {\n+\t\tstruct strbuf buf = STRBUF_INIT;\n+\t\tsq_quote_buf(&buf, filter_options.filter_spec);\n+\t\targv_array_pushf(&pack_objects.args, \"--filter=%s\", buf.buf);\n+\t\tstrbuf_release(&buf);\n+\t}\n \n \tpack_objects.in = -1;\n \tpack_objects.out = -1;\n@@ -794,6 +807,12 @@ static void receive_needs(void)\n \t\t\tdeepen_rev_list = 1;\n \t\t\tcontinue;\n \t\t}\n+\t\tif (skip_prefix(line, \"filter \", &arg)) {\n+\t\t\tif (!filter_capability_requested)\n+\t\t\t\tdie(\"git upload-pack: filtering capability not negotiated\");\n+\t\t\tparse_list_objects_filter(&filter_options, arg);\n+\t\t\tcontinue;\n+\t\t}\n \t\tif (!skip_prefix(line, \"want \", &arg) ||\n \t\t    get_oid_hex(arg, &oid_buf))\n \t\t\tdie(\"git upload-pack: protocol error, \"\n@@ -821,6 +840,8 @@ static void receive_needs(void)\n \t\t\tno_progress = 1;\n \t\tif (parse_feature_request(features, \"include-tag\"))\n \t\t\tuse_include_tag = 1;\n+\t\tif (parse_feature_request(features, \"filter\"))\n+\t\t\tfilter_capability_requested = 1;\n \n \t\to = parse_object(&oid_buf);\n \t\tif (!o) {\n@@ -940,7 +961,7 @@ static int send_ref(const char *refname, const struct object_id *oid,\n \t\tstruct strbuf symref_info = STRBUF_INIT;\n \n \t\tformat_symref_info(&symref_info, cb_data);\n-\t\tpacket_write_fmt(1, \"%s %s%c%s%s%s%s%s agent=%s\\n\",\n+\t\tpacket_write_fmt(1, \"%s %s%c%s%s%s%s%s%s agent=%s\\n\",\n \t\t\t     oid_to_hex(oid), refname_nons,\n \t\t\t     0, capabilities,\n \t\t\t     (allow_unadvertised_object_request & ALLOW_TIP_SHA1) ?\n@@ -949,6 +970,7 @@ static int send_ref(const char *refname, const struct object_id *oid,\n \t\t\t\t     \" allow-reachable-sha1-in-want\" : \"\",\n \t\t\t     stateless_rpc ? \" no-done\" : \"\",\n \t\t\t     symref_info.buf,\n+\t\t\t     filter_advertise ? \" filter\" : \"\",\n \t\t\t     git_user_agent_sanitized());\n \t\tstrbuf_release(&symref_info);\n \t} else {\n@@ -1027,6 +1049,8 @@ static int upload_pack_config(const char *var, const char *value, void *unused)\n \t} else if (current_config_scope() != CONFIG_SCOPE_REPO) {\n \t\tif (!strcmp(\"uploadpack.packobjectshook\", var))\n \t\t\treturn git_config_string(&pack_objects_hook, var, value);\n+\t} else if (!strcmp(\"uploadpack.allowfilter\", var)) {\n+\t\tfilter_advertise = git_config_bool(var, value);\n \t}\n \treturn parse_hide_refs_config(var, value, \"uploadpack\");\n }\n-- \n2.9.3\n\n"}]}