{"thread":{"id":"47531","subject":"[PATCH 08/40] index-pack: refactor writing of .keep files","startedAt":"2018-01-03T16:34:43Z","lastAt":"2018-03-19T13:27:08Z","messageCount":51,"participants":["Christian Couder","Jeff Hostetler"],"isPatch":true,"patchVersion":1,"patchTotal":40},"messages":[{"id":"335721","messageId":"20180103163403.11303-9-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 08/40] index-pack: refactor writing of .keep files","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:31Z","receivedAt":"2018-01-03T16:34:43Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"From: Jonathan Tan \n\nIn a subsequent commit, index-pack will be taught to write \".promisor\"\nfiles which are similar to the \".keep\" files it knows how to write.\nRefactor the writing of \".keep\" files, so that the implementation of\nwriting \".promisor\" files becomes easier.\n\nSigned-off-by: Jonathan Tan \nSigned-off-by: Junio C Hamano \n---\n builtin/index-pack.c | 99 ++++++++++++++++++++++++++++------------------------\n 1 file changed, 53 insertions(+), 46 deletions(-)\n\ndiff --git a/builtin/index-pack.c b/builtin/index-pack.c\nindex 4c51aec81f..47515d8977 100644\n--- a/builtin/index-pack.c\n+++ b/builtin/index-pack.c\n@@ -1389,15 +1389,58 @@ static void fix_unresolved_deltas(struct sha1file *f)\n \tfree(sorted_by_pos);\n }\n \n+static const char *derive_filename(const char *pack_name, const char *suffix,\n+\t\t\t\t struct strbuf *buf)\n+{\n+\tsize_t len;\n+\tif (!strip_suffix(pack_name, \".pack\", &len))\n+\t\tdie(_(\"packfile name '%s' does not end with '.pack'\"),\n+\t\t pack_name);\n+\tstrbuf_add(buf, pack_name, len);\n+\tstrbuf_addch(buf, '.');\n+\tstrbuf_addstr(buf, suffix);\n+\treturn buf->buf;\n+}\n+\n+static void write_special_file(const char *suffix, const char *msg,\n+\t\t\t const char *pack_name, const unsigned char *sha1,\n+\t\t\t const char **report)\n+{\n+\tstruct strbuf name_buf = STRBUF_INIT;\n+\tconst char *filename;\n+\tint fd;\n+\tint msg_len = strlen(msg);\n+\n+\tif (pack_name)\n+\t\tfilename = derive_filename(pack_name, suffix, &name_buf);\n+\telse\n+\t\tfilename = odb_pack_name(&name_buf, sha1, suffix);\n+\n+\tfd = odb_pack_keep(filename);\n+\tif (fd < 0) {\n+\t\tif (errno != EEXIST)\n+\t\t\tdie_errno(_(\"cannot write %s file '%s'\"),\n+\t\t\t\t suffix, filename);\n+\t} else {\n+\t\tif (msg_len > 0) {\n+\t\t\twrite_or_die(fd, msg, msg_len);\n+\t\t\twrite_or_die(fd, \"\\n\", 1);\n+\t\t}\n+\t\tif (close(fd) != 0)\n+\t\t\tdie_errno(_(\"cannot close written %s file '%s'\"),\n+\t\t\t\t suffix, filename);\n+\t\t*report = suffix;\n+\t}\n+\tstrbuf_release(&name_buf);\n+}\n+\n static void final(const char *final_pack_name, const char *curr_pack_name,\n \t\t const char *final_index_name, const char *curr_index_name,\n-\t\t const char *keep_name, const char *keep_msg,\n-\t\t unsigned char *sha1)\n+\t\t const char *keep_msg, unsigned char *sha1)\n {\n \tconst char *report = \"pack\";\n \tstruct strbuf pack_name = STRBUF_INIT;\n \tstruct strbuf index_name = STRBUF_INIT;\n-\tstruct strbuf keep_name_buf = STRBUF_INIT;\n \tint err;\n \n \tif (!from_stdin) {\n@@ -1409,28 +1452,9 @@ static void final(const char *final_pack_name, const char *curr_pack_name,\n \t\t\tdie_errno(_(\"error while closing pack file\"));\n \t}\n \n-\tif (keep_msg) {\n-\t\tint keep_fd, keep_msg_len = strlen(keep_msg);\n-\n-\t\tif (!keep_name)\n-\t\t\tkeep_name = odb_pack_name(&keep_name_buf, sha1, \"keep\");\n-\n-\t\tkeep_fd = odb_pack_keep(keep_name);\n-\t\tif (keep_fd < 0) {\n-\t\t\tif (errno != EEXIST)\n-\t\t\t\tdie_errno(_(\"cannot write keep file '%s'\"),\n-\t\t\t\t\t keep_name);\n-\t\t} else {\n-\t\t\tif (keep_msg_len > 0) {\n-\t\t\t\twrite_or_die(keep_fd, keep_msg, keep_msg_len);\n-\t\t\t\twrite_or_die(keep_fd, \"\\n\", 1);\n-\t\t\t}\n-\t\t\tif (close(keep_fd) != 0)\n-\t\t\t\tdie_errno(_(\"cannot close written keep file '%s'\"),\n-\t\t\t\t\t keep_name);\n-\t\t\treport = \"keep\";\n-\t\t}\n-\t}\n+\tif (keep_msg)\n+\t\twrite_special_file(\"keep\", keep_msg, final_pack_name, sha1,\n+\t\t\t\t &report);\n \n \tif (final_pack_name != curr_pack_name) {\n \t\tif (!final_pack_name)\n@@ -1472,7 +1496,6 @@ static void final(const char *final_pack_name, const char *curr_pack_name,\n \n \tstrbuf_release(&index_name);\n \tstrbuf_release(&pack_name);\n-\tstrbuf_release(&keep_name_buf);\n }\n \n static int git_index_pack_config(const char *k, const char *v, void *cb)\n@@ -1615,26 +1638,13 @@ static void show_pack_info(int stat_only)\n \t}\n }\n \n-static const char *derive_filename(const char *pack_name, const char *suffix,\n-\t\t\t\t struct strbuf *buf)\n-{\n-\tsize_t len;\n-\tif (!strip_suffix(pack_name, \".pack\", &len))\n-\t\tdie(_(\"packfile name '%s' does not end with '.pack'\"),\n-\t\t pack_name);\n-\tstrbuf_add(buf, pack_name, len);\n-\tstrbuf_addstr(buf, suffix);\n-\treturn buf->buf;\n-}\n-\n int cmd_index_pack(int argc, const char **argv, const char *prefix)\n {\n \tint i, fix_thin_pack = 0, verify = 0, stat_only = 0;\n \tconst char *curr_index;\n \tconst char *index_name = NULL, *pack_name = NULL;\n-\tconst char *keep_name = NULL, *keep_msg = NULL;\n-\tstruct strbuf index_name_buf = STRBUF_INIT,\n-\t\t keep_name_buf = STRBUF_INIT;\n+\tconst char *keep_msg = NULL;\n+\tstruct strbuf index_name_buf = STRBUF_INIT;\n \tstruct pack_idx_entry **idx_objects;\n \tstruct pack_idx_option opts;\n \tunsigned char pack_sha1[20];\n@@ -1740,9 +1750,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)\n \tif (from_stdin && !startup_info->have_repository)\n \t\tdie(_(\"--stdin requires a git repository\"));\n \tif (!index_name && pack_name)\n-\t\tindex_name = derive_filename(pack_name, \".idx\", &index_name_buf);\n-\tif (keep_msg && !keep_name && pack_name)\n-\t\tkeep_name = derive_filename(pack_name, \".keep\", &keep_name_buf);\n+\t\tindex_name = derive_filename(pack_name, \"idx\", &index_name_buf);\n \n \tif (verify) {\n \t\tif (!index_name)\n@@ -1790,13 +1798,12 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)\n \tif (!verify)\n \t\tfinal(pack_name, curr_pack,\n \t\t index_name, curr_index,\n-\t\t keep_name, keep_msg,\n+\t\t keep_msg,\n \t\t pack_sha1);\n \telse\n \t\tclose(input_fd);\n \tfree(objects);\n \tstrbuf_release(&index_name_buf);\n-\tstrbuf_release(&keep_name_buf);\n \tif (pack_name == NULL)\n \t\tfree((void *) curr_pack);\n \tif (index_name == NULL)\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335722","messageId":"20180103163403.11303-13-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 12/40] rev-list: support termination at promisor objects","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:35Z","receivedAt":"2018-01-03T16:34:51Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"From: Jonathan Tan \n\nTeach rev-list to support termination of an object traversal at any\nobject from a promisor remote (whether one that the local repo also has,\nor one that the local repo knows about because it has another promisor\nobject that references it).\n\nThis will be used subsequently in gc and in the connectivity check used\nby fetch.\n\nFor efficiency, if an object is referenced by a promisor object, and is\nin the local repo only as a non-promisor object, object traversal will\nnot stop there. This is to avoid building the list of promisor object\nreferences.\n\n(In list-objects.c, the case where obj is NULL in process_blob() and\nprocess_tree() do not need to be changed because those happen only when\nthere is a conflict between the expected type and the existing object.\nIf the object doesn't exist, an object will be synthesized, which is\nfine.)\n\nSigned-off-by: Jonathan Tan \nSigned-off-by: Jeff Hostetler \nSigned-off-by: Junio C Hamano \n---\n Documentation/rev-list-options.txt | 11 ++++\n builtin/rev-list.c | 69 ++++++++++++++++++++++---\n list-objects.c | 29 ++++++++++-\n object.c | 2 +-\n revision.c | 33 +++++++++++-\n revision.h | 5 +-\n t/t0410-partial-clone.sh | 101 +++++++++++++++++++++++++++++++++++++\n 7 files changed, 239 insertions(+), 11 deletions(-)\n\ndiff --git a/Documentation/rev-list-options.txt b/Documentation/rev-list-options.txt\nindex 22f5c9b43d..7b273635de 100644\n--- a/Documentation/rev-list-options.txt\n+++ b/Documentation/rev-list-options.txt\n@@ -750,10 +750,21 @@ The form '--missing=allow-any' will allow object traversal to continue\n if a missing object is encountered. Missing objects will silently be\n omitted from the results.\n +\n+The form '--missing=allow-promisor' is like 'allow-any', but will only\n+allow object traversal to continue for EXPECTED promisor missing objects.\n+Unexpected missing objects will raise an error.\n++\n The form '--missing=print' is like 'allow-any', but will also print a\n list of the missing objects. Object IDs are prefixed with a ``?'' character.\n endif::git-rev-list[]\n \n+--exclude-promisor-objects::\n+\t(For internal use only.) Prefilter object traversal at\n+\tpromisor boundary. This is used with partial clone. This is\n+\tstronger than `--missing=allow-promisor` because it limits the\n+\ttraversal, rather than just silencing errors about missing\n+\tobjects.\n+\n --no-walk[=(sorted|unsorted)]::\n \tOnly show the given commits, but do not traverse their ancestors.\n \tThis has no effect if a range is specified. If the argument\ndiff --git a/builtin/rev-list.c b/builtin/rev-list.c\nindex d5345b6a2e..e27aa1fc07 100644\n--- a/builtin/rev-list.c\n+++ b/builtin/rev-list.c\n@@ -15,6 +15,7 @@\n #include \"progress.h\"\n #include \"reflog-walk.h\"\n #include \"oidset.h\"\n+#include \"packfile.h\"\n \n static const char rev_list_usage[] =\n \"git rev-list [OPTION] ... [ -- paths... ]\\n\"\n@@ -67,6 +68,7 @@ enum missing_action {\n \tMA_ERROR = 0, /* fail if any missing objects are encountered */\n \tMA_ALLOW_ANY, /* silently allow ALL missing objects */\n \tMA_PRINT, /* print ALL missing objects in special section */\n+\tMA_ALLOW_PROMISOR, /* silently allow all missing PROMISOR objects */\n };\n static enum missing_action arg_missing_action;\n \n@@ -197,6 +199,12 @@ static void finish_commit(struct commit *commit, void *data)\n \n static inline void finish_object__ma(struct object *obj)\n {\n+\t/*\n+\t * Whether or not we try to dynamically fetch missing objects\n+\t * from the server, we currently DO NOT have the object. We\n+\t * can either print, allow (ignore), or conditionally allow\n+\t * (ignore) them.\n+\t */\n \tswitch (arg_missing_action) {\n \tcase MA_ERROR:\n \t\tdie(\"missing blob object '%s'\", oid_to_hex(&obj->oid));\n@@ -209,25 +217,36 @@ static inline void finish_object__ma(struct object *obj)\n \t\toidset_insert(&missing_objects, &obj->oid);\n \t\treturn;\n \n+\tcase MA_ALLOW_PROMISOR:\n+\t\tif (is_promisor_object(&obj->oid))\n+\t\t\treturn;\n+\t\tdie(\"unexpected missing blob object '%s'\",\n+\t\t oid_to_hex(&obj->oid));\n+\t\treturn;\n+\n \tdefault:\n \t\tBUG(\"unhandled missing_action\");\n \t\treturn;\n \t}\n }\n \n-static void finish_object(struct object *obj, const char *name, void *cb_data)\n+static int finish_object(struct object *obj, const char *name, void *cb_data)\n {\n \tstruct rev_list_info *info = cb_data;\n-\tif (obj->type == OBJ_BLOB && !has_object_file(&obj->oid))\n+\tif (obj->type == OBJ_BLOB && !has_object_file(&obj->oid)) {\n \t\tfinish_object__ma(obj);\n+\t\treturn 1;\n+\t}\n \tif (info->revs->verify_objects && !obj->parsed && obj->type != OBJ_COMMIT)\n \t\tparse_object(&obj->oid);\n+\treturn 0;\n }\n \n static void show_object(struct object *obj, const char *name, void *cb_data)\n {\n \tstruct rev_list_info *info = cb_data;\n-\tfinish_object(obj, name, cb_data);\n+\tif (finish_object(obj, name, cb_data))\n+\t\treturn;\n \tdisplay_progress(progress, ++progress_counter);\n \tif (info->flags & REV_LIST_QUIET)\n \t\treturn;\n@@ -315,11 +334,19 @@ static inline int parse_missing_action_value(const char *value)\n \n \tif (!strcmp(value, \"allow-any\")) {\n \t\targ_missing_action = MA_ALLOW_ANY;\n+\t\tfetch_if_missing = 0;\n \t\treturn 1;\n \t}\n \n \tif (!strcmp(value, \"print\")) {\n \t\targ_missing_action = MA_PRINT;\n+\t\tfetch_if_missing = 0;\n+\t\treturn 1;\n+\t}\n+\n+\tif (!strcmp(value, \"allow-promisor\")) {\n+\t\targ_missing_action = MA_ALLOW_PROMISOR;\n+\t\tfetch_if_missing = 0;\n \t\treturn 1;\n \t}\n \n@@ -344,6 +371,35 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)\n \tinit_revisions(&revs, prefix);\n \trevs.abbrev = DEFAULT_ABBREV;\n \trevs.commit_format = CMIT_FMT_UNSPECIFIED;\n+\n+\t/*\n+\t * Scan the argument list before invoking setup_revisions(), so that we\n+\t * know if fetch_if_missing needs to be set to 0.\n+\t *\n+\t * \"--exclude-promisor-objects\" acts as a pre-filter on missing objects\n+\t * by not crossing the boundary from realized objects to promisor\n+\t * objects.\n+\t *\n+\t * Let \"--missing\" to conditionally set fetch_if_missing.\n+\t */\n+\tfor (i = 1; i < argc; i++) {\n+\t\tconst char *arg = argv[i];\n+\t\tif (!strcmp(arg, \"--exclude-promisor-objects\")) {\n+\t\t\tfetch_if_missing = 0;\n+\t\t\trevs.exclude_promisor_objects = 1;\n+\t\t\tbreak;\n+\t\t}\n+\t}\n+\tfor (i = 1; i < argc; i++) {\n+\t\tconst char *arg = argv[i];\n+\t\tif (skip_prefix(arg, \"--missing=\", &arg)) {\n+\t\t\tif (revs.exclude_promisor_objects)\n+\t\t\t\tdie(_(\"cannot combine --exclude-promisor-objects and --missing\"));\n+\t\t\tif (parse_missing_action_value(arg))\n+\t\t\t\tbreak;\n+\t\t}\n+\t}\n+\n \targc = setup_revisions(argc, argv, &revs, NULL);\n \n \tmemset(&info, 0, sizeof(info));\n@@ -412,9 +468,10 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)\n \t\t\tcontinue;\n \t\t}\n \n-\t\tif (skip_prefix(arg, \"--missing=\", &arg) &&\n-\t\t parse_missing_action_value(arg))\n-\t\t\tcontinue;\n+\t\tif (!strcmp(arg, \"--exclude-promisor-objects\"))\n+\t\t\tcontinue; /* already handled above */\n+\t\tif (skip_prefix(arg, \"--missing=\", &arg))\n+\t\t\tcontinue; /* already handled above */\n \n \t\tusage(rev_list_usage);\n \ndiff --git a/list-objects.c b/list-objects.c\nindex 0966cdc9fa..168bef688a 100644\n--- a/list-objects.c\n+++ b/list-objects.c\n@@ -9,6 +9,7 @@\n #include \"list-objects.h\"\n #include \"list-objects-filter.h\"\n #include \"list-objects-filter-options.h\"\n+#include \"packfile.h\"\n \n static void process_blob(struct rev_info *revs,\n \t\t\t struct blob *blob,\n@@ -30,6 +31,20 @@ static void process_blob(struct rev_info *revs,\n \tif (obj->flags & (UNINTERESTING | SEEN))\n \t\treturn;\n \n+\t/*\n+\t * Pre-filter known-missing objects when explicitly requested.\n+\t * Otherwise, a missing object error message may be reported\n+\t * later (depending on other filtering criteria).\n+\t *\n+\t * Note that this \"--exclude-promisor-objects\" pre-filtering\n+\t * may cause the actual filter to report an incomplete list\n+\t * of missing objects.\n+\t */\n+\tif (revs->exclude_promisor_objects &&\n+\t !has_object_file(&obj->oid) &&\n+\t is_promisor_object(&obj->oid))\n+\t\treturn;\n+\n \tpathlen = path->len;\n \tstrbuf_addstr(path, name);\n \tif (filter_fn)\n@@ -91,6 +106,8 @@ static void process_tree(struct rev_info *revs,\n \t\tall_entries_interesting: entry_not_interesting;\n \tint baselen = base->len;\n \tenum list_objects_filter_result r = LOFR_MARK_SEEN | LOFR_DO_SHOW;\n+\tint gently = revs->ignore_missing_links ||\n+\t\t revs->exclude_promisor_objects;\n \n \tif (!revs->tree_objects)\n \t\treturn;\n@@ -98,9 +115,19 @@ static void process_tree(struct rev_info *revs,\n \t\tdie(\"bad tree object\");\n \tif (obj->flags & (UNINTERESTING | SEEN))\n \t\treturn;\n-\tif (parse_tree_gently(tree, revs->ignore_missing_links) < 0) {\n+\tif (parse_tree_gently(tree, gently) < 0) {\n \t\tif (revs->ignore_missing_links)\n \t\t\treturn;\n+\n+\t\t/*\n+\t\t * Pre-filter known-missing tree objects when explicitly\n+\t\t * requested. This may cause the actual filter to report\n+\t\t * an incomplete list of missing objects.\n+\t\t */\n+\t\tif (revs->exclude_promisor_objects &&\n+\t\t is_promisor_object(&obj->oid))\n+\t\t\treturn;\n+\n \t\tdie(\"bad tree object %s\", oid_to_hex(&obj->oid));\n \t}\n \ndiff --git a/object.c b/object.c\nindex b9a4a0e501..4c222d6260 100644\n--- a/object.c\n+++ b/object.c\n@@ -252,7 +252,7 @@ struct object *parse_object(const struct object_id *oid)\n \tif (obj && obj->parsed)\n \t\treturn obj;\n \n-\tif ((obj && obj->type == OBJ_BLOB) ||\n+\tif ((obj && obj->type == OBJ_BLOB && has_object_file(oid)) ||\n \t (!obj && has_object_file(oid) &&\n \t sha1_object_info(oid->hash, NULL) == OBJ_BLOB)) {\n \t\tif (check_sha1_signature(repl, NULL, 0, NULL) < 0) {\ndiff --git a/revision.c b/revision.c\nindex 72f2b4572e..f246728f2d 100644\n--- a/revision.c\n+++ b/revision.c\n@@ -198,6 +198,8 @@ static struct object *get_reference(struct rev_info *revs, const char *name,\n \tif (!object) {\n \t\tif (revs->ignore_missing)\n \t\t\treturn object;\n+\t\tif (revs->exclude_promisor_objects && is_promisor_object(oid))\n+\t\t\treturn NULL;\n \t\tdie(\"bad object %s\", name);\n \t}\n \tobject->flags |= flags;\n@@ -799,9 +801,17 @@ static int add_parents_to_list(struct rev_info *revs, struct commit *commit,\n \n \tfor (parent = commit->parents; parent; parent = parent->next) {\n \t\tstruct commit *p = parent->item;\n-\n-\t\tif (parse_commit_gently(p, revs->ignore_missing_links) < 0)\n+\t\tint gently = revs->ignore_missing_links ||\n+\t\t\t revs->exclude_promisor_objects;\n+\t\tif (parse_commit_gently(p, gently) < 0) {\n+\t\t\tif (revs->exclude_promisor_objects &&\n+\t\t\t is_promisor_object(&p->object.oid)) {\n+\t\t\t\tif (revs->first_parent_only)\n+\t\t\t\t\tbreak;\n+\t\t\t\tcontinue;\n+\t\t\t}\n \t\t\treturn -1;\n+\t\t}\n \t\tif (revs->show_source && !p->util)\n \t\t\tp->util = commit->util;\n \t\tp->object.flags |= left_flag;\n@@ -2100,6 +2110,10 @@ static int handle_revision_opt(struct rev_info *revs, int argc, const char **arg\n \t\trevs->limited = 1;\n \t} else if (!strcmp(arg, \"--ignore-missing\")) {\n \t\trevs->ignore_missing = 1;\n+\t} else if (!strcmp(arg, \"--exclude-promisor-objects\")) {\n+\t\tif (fetch_if_missing)\n+\t\t\tdie(\"BUG: exclude_promisor_objects can only be used when fetch_if_missing is 0\");\n+\t\trevs->exclude_promisor_objects = 1;\n \t} else {\n \t\tint opts = diff_opt_parse(&revs->diffopt, argv, argc, revs->prefix);\n \t\tif (!opts)\n@@ -2842,6 +2856,16 @@ void reset_revision_walk(void)\n \tclear_object_flags(SEEN | ADDED | SHOWN);\n }\n \n+static int mark_uninteresting(const struct object_id *oid,\n+\t\t\t struct packed_git *pack,\n+\t\t\t uint32_t pos,\n+\t\t\t void *unused)\n+{\n+\tstruct object *o = parse_object(oid);\n+\to->flags |= UNINTERESTING | SEEN;\n+\treturn 0;\n+}\n+\n int prepare_revision_walk(struct rev_info *revs)\n {\n \tint i;\n@@ -2870,6 +2894,11 @@ int prepare_revision_walk(struct rev_info *revs)\n \t (revs->limited && limiting_can_increase_treesame(revs)))\n \t\trevs->treesame.name = \"treesame\";\n \n+\tif (revs->exclude_promisor_objects) {\n+\t\tfor_each_packed_object(mark_uninteresting, NULL,\n+\t\t\t\t FOR_EACH_OBJECT_PROMISOR_ONLY);\n+\t}\n+\n \tif (revs->no_walk != REVISION_WALK_NO_WALK_UNSORTED)\n \t\tcommit_list_sort_by_date(&revs->commits);\n \tif (revs->no_walk)\ndiff --git a/revision.h b/revision.h\nindex 19dc9bdddb..eb14994316 100644\n--- a/revision.h\n+++ b/revision.h\n@@ -122,7 +122,10 @@ struct rev_info {\n \t\t\tancestry_path:1,\n \t\t\tfirst_parent_only:1,\n \t\t\tline_level_traverse:1,\n-\t\t\ttree_blobs_in_commit_order:1;\n+\t\t\ttree_blobs_in_commit_order:1,\n+\n+\t\t\t/* for internal use only */\n+\t\t\texclude_promisor_objects:1;\n \n \t/* Diff flags */\n \tunsigned int\tdiff:1,\ndiff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh\nindex 8b20d18603..4dc02459a5 100755\n--- a/t/t0410-partial-clone.sh\n+++ b/t/t0410-partial-clone.sh\n@@ -160,6 +160,107 @@ test_expect_success 'fetching of missing objects' '\n \tgit verify-pack --verbose \"$IDX\" | grep \"$HASH\"\n '\n \n+test_expect_success 'rev-list stops traversal at missing and promised commit' '\n+\trm -rf repo &&\n+\ttest_create_repo repo &&\n+\ttest_commit -C repo foo &&\n+\ttest_commit -C repo bar &&\n+\n+\tFOO=$(git -C repo rev-parse foo) &&\n+\tpromise_and_delete \"$FOO\" &&\n+\n+\tgit -C repo config core.repositoryformatversion 1 &&\n+\tgit -C repo config odb.magic.promisorRemote \"arbitrary string\" &&\n+\tgit -C repo rev-list --exclude-promisor-objects --objects bar >out &&\n+\tgrep $(git -C repo rev-parse bar) out &&\n+\t! grep $FOO out\n+'\n+\n+test_expect_success 'rev-list stops traversal at missing and promised tree' '\n+\trm -rf repo &&\n+\ttest_create_repo repo &&\n+\ttest_commit -C repo foo &&\n+\tmkdir repo/a_dir &&\n+\techo something >repo/a_dir/something &&\n+\tgit -C repo add a_dir/something &&\n+\tgit -C repo commit -m bar &&\n+\n+\t# foo^{tree} (tree referenced from commit)\n+\tTREE=$(git -C repo rev-parse foo^{tree}) &&\n+\n+\t# a tree referenced by HEAD^{tree} (tree referenced from tree)\n+\tTREE2=$(git -C repo ls-tree HEAD^{tree} | grep \" tree \" | head -1 | cut -b13-52) &&\n+\n+\tpromise_and_delete \"$TREE\" &&\n+\tpromise_and_delete \"$TREE2\" &&\n+\n+\tgit -C repo config core.repositoryformatversion 1 &&\n+\tgit -C repo config odb.magic.promisorRemote \"arbitrary string\" &&\n+\tgit -C repo rev-list --exclude-promisor-objects --objects HEAD >out &&\n+\tgrep $(git -C repo rev-parse foo) out &&\n+\t! grep $TREE out &&\n+\tgrep $(git -C repo rev-parse HEAD) out &&\n+\t! grep $TREE2 out\n+'\n+\n+test_expect_success 'rev-list stops traversal at missing and promised blob' '\n+\trm -rf repo &&\n+\ttest_create_repo repo &&\n+\techo something >repo/something &&\n+\tgit -C repo add something &&\n+\tgit -C repo commit -m foo &&\n+\n+\tBLOB=$(git -C repo hash-object -w something) &&\n+\tpromise_and_delete \"$BLOB\" &&\n+\n+\tgit -C repo config core.repositoryformatversion 1 &&\n+\tgit -C repo config odb.magic.promisorRemote \"arbitrary string\" &&\n+\tgit -C repo rev-list --exclude-promisor-objects --objects HEAD >out &&\n+\tgrep $(git -C repo rev-parse HEAD) out &&\n+\t! grep $BLOB out\n+'\n+\n+test_expect_success 'rev-list stops traversal at promisor commit, tree, and blob' '\n+\trm -rf repo &&\n+\ttest_create_repo repo &&\n+\ttest_commit -C repo foo &&\n+\ttest_commit -C repo bar &&\n+\ttest_commit -C repo baz &&\n+\n+\tCOMMIT=$(git -C repo rev-parse foo) &&\n+\tTREE=$(git -C repo rev-parse bar^{tree}) &&\n+\tBLOB=$(git hash-object repo/baz.t) &&\n+\tprintf \"%s\\n%s\\n%s\\n\" $COMMIT $TREE $BLOB | pack_as_from_promisor &&\n+\n+\tgit -C repo config core.repositoryformatversion 1 &&\n+\tgit -C repo config odb.magic.promisorRemote \"arbitrary string\" &&\n+\tgit -C repo rev-list --exclude-promisor-objects --objects HEAD >out &&\n+\t! grep $COMMIT out &&\n+\t! grep $TREE out &&\n+\t! grep $BLOB out &&\n+\tgrep $(git -C repo rev-parse bar) out # sanity check that some walking was done\n+'\n+\n+test_expect_success 'rev-list accepts missing and promised objects on command line' '\n+\trm -rf repo &&\n+\ttest_create_repo repo &&\n+\ttest_commit -C repo foo &&\n+\ttest_commit -C repo bar &&\n+\ttest_commit -C repo baz &&\n+\n+\tCOMMIT=$(git -C repo rev-parse foo) &&\n+\tTREE=$(git -C repo rev-parse bar^{tree}) &&\n+\tBLOB=$(git hash-object repo/baz.t) &&\n+\n+\tpromise_and_delete $COMMIT &&\n+\tpromise_and_delete $TREE &&\n+\tpromise_and_delete $BLOB &&\n+\n+\tgit -C repo config core.repositoryformatversion 1 &&\n+\tgit -C repo config odb.magic.promisorRemote \"arbitrary string\" &&\n+\tgit -C repo rev-list --exclude-promisor-objects --objects \"$COMMIT\" \"$TREE\" \"$BLOB\"\n+'\n+\n LIB_HTTPD_PORT=12345 # default port, 410, cannot be used as non-root\n . \"$TEST_DIRECTORY\"/lib-httpd.sh\n start_httpd\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335723","messageId":"20180103163403.11303-12-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 11/40] sha1_file: support lazily fetching missing objects","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:34Z","receivedAt":"2018-01-03T16:34:55Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"From: Jonathan Tan \n\nTeach sha1_file to fetch objects from the remote configured in\nextensions.partialclone whenever an object is requested but missing.\n\nThe fetching of objects can be suppressed through a global variable.\nThis is used by fsck and index-pack.\n\nHowever, by default, such fetching is not suppressed. This is meant as a\ntemporary measure to ensure that all Git commands work in such a\nsituation. Future patches will update some commands to either tolerate\nmissing objects (without fetching them) or be more efficient in fetching\nthem.\n\nIn order to determine the code changes in sha1_file.c necessary, I\ninvestigated the following:\n (1) functions in sha1_file.c that take in a hash, without the user\n regarding how the object is stored (loose or packed)\n (2) functions in packfile.c (because I need to check callers that know\n about the loose/packed distinction and operate on both differently,\n and ensure that they can handle the concept of objects that are\n neither loose nor packed)\n\n(1) is handled by the modification to sha1_object_info_extended().\n\nFor (2), I looked at for_each_packed_object and others. For\nfor_each_packed_object, the callers either already work or are fixed in\nthis patch:\n - reachable - only to find recent objects\n - builtin/fsck - already knows about missing objects\n - builtin/cat-file - warning message added in this commit\n\nCallers of the other functions do not need to be changed:\n - parse_pack_index\n - http - indirectly from http_get_info_packs\n - find_pack_entry_one\n - this searches a single pack that is provided as an argument; the\n caller already knows (through other means) that the sought object\n is in a specific pack\n - find_sha1_pack\n - fast-import - appears to be an optimization to not store a file if\n it is already in a pack\n - http-walker - to search through a struct alt_base\n - http-push - to search through remote packs\n - has_sha1_pack\n - builtin/fsck - already knows about promisor objects\n - builtin/count-objects - informational purposes only (check if loose\n object is also packed)\n - builtin/prune-packed - check if object to be pruned is packed (if\n not, don't prune it)\n - revision - used to exclude packed objects if requested by user\n - diff - just for optimization\n\nSigned-off-by: Jonathan Tan \nSigned-off-by: Junio C Hamano \n---\n builtin/cat-file.c | 3 +++\n builtin/fetch-pack.c | 2 ++\n builtin/fsck.c | 3 +++\n builtin/index-pack.c | 6 ++++++\n cache.h | 8 ++++++++\n fetch-object.c | 3 +++\n sha1_file.c | 28 ++++++++++++++++++--------\n t/t0410-partial-clone.sh | 51 ++++++++++++++++++++++++++++++++++++++++++++++++\n 8 files changed, 96 insertions(+), 8 deletions(-)\n\ndiff --git a/builtin/cat-file.c b/builtin/cat-file.c\nindex f5fa4fd75a..1e4edd81a0 100644\n--- a/builtin/cat-file.c\n+++ b/builtin/cat-file.c\n@@ -13,6 +13,7 @@\n #include \"tree-walk.h\"\n #include \"sha1-array.h\"\n #include \"packfile.h\"\n+#include \"external-odb.h\"\n \n struct batch_options {\n \tint enabled;\n@@ -475,6 +476,8 @@ static int batch_objects(struct batch_options *opt)\n \n \t\tfor_each_loose_object(batch_loose_object, &sa, 0);\n \t\tfor_each_packed_object(batch_packed_object, &sa, 0);\n+\t\tif (has_external_odb())\n+\t\t\twarning(\"This repository uses an odb. Some objects may not be loaded.\");\n \n \t\tcb.opt = opt;\n \t\tcb.expand = &data;\ndiff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c\nindex 02abe7211e..15eeed7b17 100644\n--- a/builtin/fetch-pack.c\n+++ b/builtin/fetch-pack.c\n@@ -53,6 +53,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)\n \tstruct oid_array shallow = OID_ARRAY_INIT;\n \tstruct string_list deepen_not = STRING_LIST_INIT_DUP;\n \n+\tfetch_if_missing = 0;\n+\n \tpacket_trace_identity(\"fetch-pack\");\n \n \tmemset(&args, 0, sizeof(args));\ndiff --git a/builtin/fsck.c b/builtin/fsck.c\nindex a6fa6d6482..7a8a679d4f 100644\n--- a/builtin/fsck.c\n+++ b/builtin/fsck.c\n@@ -678,6 +678,9 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)\n \tint i;\n \tstruct alternate_object_database *alt;\n \n+\t/* fsck knows how to handle missing promisor objects */\n+\tfetch_if_missing = 0;\n+\n \terrors_found = 0;\n \tcheck_replace_refs = 0;\n \ndiff --git a/builtin/index-pack.c b/builtin/index-pack.c\nindex 9dffaf20ae..54c921fa71 100644\n--- a/builtin/index-pack.c\n+++ b/builtin/index-pack.c\n@@ -1657,6 +1657,12 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)\n \tunsigned foreign_nr = 1;\t/* zero is a \"good\" value, assume bad */\n \tint report_end_of_input = 0;\n \n+\t/*\n+\t * index-pack never needs to fetch missing objects, since it only\n+\t * accesses the repo to do hash collision checks\n+\t */\n+\tfetch_if_missing = 0;\n+\n \tif (argc == 2 && !strcmp(argv[1], \"-h\"))\n \t\tusage(index_pack_usage);\n \ndiff --git a/cache.h b/cache.h\nindex 078607ee91..3fabf998ce 100644\n--- a/cache.h\n+++ b/cache.h\n@@ -1789,6 +1789,14 @@ struct object_info {\n #define OBJECT_INFO_QUICK 8\n extern int sha1_object_info_extended(const unsigned char *, struct object_info *, unsigned flags);\n \n+/*\n+ * Set this to 0 to prevent sha1_object_info_extended() from fetching missing\n+ * blobs. This has a difference only if extensions.partialClone is set.\n+ *\n+ * Its default value is 1.\n+ */\n+extern int fetch_if_missing;\n+\n /* Dumb servers support */\n extern int update_server_info(int);\n \ndiff --git a/fetch-object.c b/fetch-object.c\nindex 23ec2bb0d0..8afadeda2b 100644\n--- a/fetch-object.c\n+++ b/fetch-object.c\n@@ -14,7 +14,9 @@ void fetch_object(const char *remote_name, const unsigned char *sha1)\n \tstruct remote *remote;\n \tstruct transport *transport;\n \tstruct ref *ref;\n+\tint original_fetch_if_missing = fetch_if_missing;\n \n+\tfetch_if_missing = 0;\n \tremote = remote_get(remote_name);\n \tif (!remote->url[0])\n \t\tdie(_(\"Remote with no URL\"));\n@@ -25,4 +27,5 @@ void fetch_object(const char *remote_name, const unsigned char *sha1)\n \ttransport_set_option(transport, TRANS_OPT_FROM_PROMISOR, \"1\");\n \ttransport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, \"1\");\n \ttransport_fetch_refs(transport, ref);\n+\tfetch_if_missing = original_fetch_if_missing;\n }\ndiff --git a/sha1_file.c b/sha1_file.c\nindex cba6b2a537..261baf800f 100644\n--- a/sha1_file.c\n+++ b/sha1_file.c\n@@ -1233,6 +1233,8 @@ static int sha1_loose_object_info(const unsigned char *sha1,\n \treturn (status < 0) ? status : 0;\n }\n \n+int fetch_if_missing = 1;\n+\n int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)\n {\n \tstatic struct object_info blank_oi = OBJECT_INFO_INIT;\n@@ -1241,6 +1243,7 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,\n \tconst unsigned char *real = (flags & OBJECT_INFO_LOOKUP_REPLACE) ?\n \t\t\t\t lookup_replace_object(sha1) :\n \t\t\t\t sha1;\n+\tint already_retried = 0;\n \n \tif (is_null_sha1(real))\n \t\treturn -1;\n@@ -1268,19 +1271,29 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,\n \t\t}\n \t}\n \n-\tif (!find_pack_entry(real, &e)) {\n+\twhile (1) {\n+\t\tif (find_pack_entry(real, &e))\n+\t\t\tbreak;\n+\n \t\t/* Most likely it's a loose object. */\n \t\tif (!sha1_loose_object_info(real, oi, flags))\n \t\t\treturn 0;\n \n \t\t/* Not a loose object; someone else may have just packed it. */\n-\t\tif (flags & OBJECT_INFO_QUICK) {\n-\t\t\treturn -1;\n-\t\t} else {\n-\t\t\treprepare_packed_git();\n-\t\t\tif (!find_pack_entry(real, &e))\n-\t\t\t\treturn -1;\n+\t\treprepare_packed_git();\n+\t\tif (find_pack_entry(real, &e))\n+\t\t\tbreak;\n+\n+\t\t/* Check if it is a missing object */\n+\t\tif (fetch_if_missing && has_external_odb() &&\n+\t\t !already_retried) {\n+\t\t\tif (!external_odb_get_direct(real))\n+\t\t\t\treturn 0;\n+\t\t\talready_retried = 1;\n+\t\t\tcontinue;\n \t\t}\n+\n+\t\treturn -1;\n \t}\n \n \tif (oi == &blank_oi)\n@@ -1289,7 +1302,6 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,\n \t\t * information below, so return early.\n \t\t */\n \t\treturn 0;\n-\n \trtype = packed_object_info(e.p, e.offset, oi);\n \tif (rtype < 0) {\n \t\tmark_bad_packed_object(e.p, real);\ndiff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh\nindex a0f901fa1d..8b20d18603 100755\n--- a/t/t0410-partial-clone.sh\n+++ b/t/t0410-partial-clone.sh\n@@ -138,4 +138,55 @@ test_expect_success 'missing CLI object, but promised, passes fsck' '\n \tgit -C repo fsck \"$A\"\n '\n \n+test_expect_success 'fetching of missing objects' '\n+\trm -rf repo &&\n+\ttest_create_repo server &&\n+\ttest_commit -C server foo &&\n+\tgit -C server repack -a -d --write-bitmap-index &&\n+\n+\tgit clone \"file://$(pwd)/server\" repo &&\n+\tHASH=$(git -C repo rev-parse foo) &&\n+\trm -rf repo/.git/objects/* &&\n+\n+\tgit -C repo config core.repositoryformatversion 1 &&\n+\tgit -C repo config odb.magic.promisorRemote \"origin\" &&\n+\tgit -C repo cat-file -p \"$HASH\" &&\n+\n+\t# Ensure that the .promisor file is written, and check that its\n+\t# associated packfile contains the object\n+\tls repo/.git/objects/pack/pack-*.promisor >promisorlist &&\n+\ttest_line_count = 1 promisorlist &&\n+\tIDX=$(cat promisorlist | sed \"s/promisor$/idx/\") &&\n+\tgit verify-pack --verbose \"$IDX\" | grep \"$HASH\"\n+'\n+\n+LIB_HTTPD_PORT=12345 # default port, 410, cannot be used as non-root\n+. \"$TEST_DIRECTORY\"/lib-httpd.sh\n+start_httpd\n+\n+test_expect_success 'fetching of missing objects from an HTTP server' '\n+\trm -rf repo &&\n+\tSERVER=\"$HTTPD_DOCUMENT_ROOT_PATH/server\" &&\n+\ttest_create_repo \"$SERVER\" &&\n+\ttest_commit -C \"$SERVER\" foo &&\n+\tgit -C \"$SERVER\" repack -a -d --write-bitmap-index &&\n+\n+\tgit clone $HTTPD_URL/smart/server repo &&\n+\tHASH=$(git -C repo rev-parse foo) &&\n+\trm -rf repo/.git/objects/* &&\n+\n+\tgit -C repo config core.repositoryformatversion 1 &&\n+\tgit -C repo config odb.magic.promisorRemote \"origin\" &&\n+\tgit -C repo cat-file -p \"$HASH\" &&\n+\n+\t# Ensure that the .promisor file is written, and check that its\n+\t# associated packfile contains the object\n+\tls repo/.git/objects/pack/pack-*.promisor >promisorlist &&\n+\ttest_line_count = 1 promisorlist &&\n+\tIDX=$(cat promisorlist | sed \"s/promisor$/idx/\") &&\n+\tgit verify-pack --verbose \"$IDX\" | grep \"$HASH\"\n+'\n+\n+stop_httpd\n+\n test_done\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335724","messageId":"20180103163403.11303-10-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 09/40] introduce fetch-object: fetch one promisor object","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:32Z","receivedAt":"2018-01-03T16:35:00Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"From: Jonathan Tan \n\nIntroduce fetch-object, providing the ability to fetch one object from a\npromisor remote.\n\nThis uses fetch-pack. To do this, the transport mechanism has been\nupdated with 2 flags, \"from-promisor\" to indicate that the resulting\npack comes from a promisor remote (and thus should be annotated as such\nby index-pack), and \"no-dependents\" to indicate that only the objects\nthemselves need to be fetched (but fetching additional objects is\nnevertheless safe).\n\nWhenever \"no-dependents\" is used, fetch-pack will refrain from using any\nobject flags, because it is most likely invoked as part of a dynamic\nobject fetch by another Git command (which may itself use object flags).\nAn alternative to this is to leave fetch-pack alone, and instead update\nthe allocation of flags so that fetch-pack's flags never overlap with\nany others, but this will end up shrinking the number of flags available\nto nearly every other Git command (that is, every Git command that\naccesses objects), so the approach in this commit was used instead.\n\nThis will be tested in a subsequent commit.\n\nSigned-off-by: Jonathan Tan \nSigned-off-by: Junio C Hamano \n---\n Documentation/gitremote-helpers.txt | 7 ++++++\n Makefile | 1 +\n builtin/fetch-pack.c | 8 +++++++\n builtin/index-pack.c | 14 ++++++++---\n fetch-object.c | 28 ++++++++++++++++++++++\n fetch-object.h | 6 +++++\n fetch-pack.c | 48 +++++++++++++++++++++----------------\n fetch-pack.h | 8 +++++++\n remote-curl.c | 14 ++++++++++-\n transport.c | 8 +++++++\n transport.h | 11 +++++++++\n 11 files changed, 128 insertions(+), 25 deletions(-)\n create mode 100644 fetch-object.c\n create mode 100644 fetch-object.h\n\ndiff --git a/Documentation/gitremote-helpers.txt b/Documentation/gitremote-helpers.txt\nindex 4a584f3c5d..4b8c93ec59 100644\n--- a/Documentation/gitremote-helpers.txt\n+++ b/Documentation/gitremote-helpers.txt\n@@ -466,6 +466,13 @@ set by Git if the remote helper has the 'option' capability.\n \tTransmit as a push option. As the push option\n \tmust not contain LF or NUL characters, the string is not encoded.\n \n+'option from-promisor' {'true'|'false'}::\n+\tIndicate that these objects are being fetched from a promisor.\n+\n+'option no-dependents' {'true'|'false'}::\n+\tIndicate that only the objects wanted need to be fetched, not\n+\ttheir dependents.\n+\n SEE ALSO\n --------\n linkgit:git-remote[1]\ndiff --git a/Makefile b/Makefile\nindex 07694185c9..25b878279e 100644\n--- a/Makefile\n+++ b/Makefile\n@@ -800,6 +800,7 @@ LIB_OBJS += ewah/ewah_io.o\n LIB_OBJS += ewah/ewah_rlw.o\n LIB_OBJS += exec_cmd.o\n LIB_OBJS += external-odb.o\n+LIB_OBJS += fetch-object.o\n LIB_OBJS += fetch-pack.o\n LIB_OBJS += fsck.o\n LIB_OBJS += fsmonitor.o\ndiff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c\nindex 366b9d13f9..02abe7211e 100644\n--- a/builtin/fetch-pack.c\n+++ b/builtin/fetch-pack.c\n@@ -143,6 +143,14 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)\n \t\t\targs.update_shallow = 1;\n \t\t\tcontinue;\n \t\t}\n+\t\tif (!strcmp(\"--from-promisor\", arg)) {\n+\t\t\targs.from_promisor = 1;\n+\t\t\tcontinue;\n+\t\t}\n+\t\tif (!strcmp(\"--no-dependents\", arg)) {\n+\t\t\targs.no_dependents = 1;\n+\t\t\tcontinue;\n+\t\t}\n \t\tusage(fetch_pack_usage);\n \t}\n \tif (deepen_not.nr)\ndiff --git a/builtin/index-pack.c b/builtin/index-pack.c\nindex 47515d8977..9dffaf20ae 100644\n--- a/builtin/index-pack.c\n+++ b/builtin/index-pack.c\n@@ -1429,14 +1429,16 @@ static void write_special_file(const char *suffix, const char *msg,\n \t\tif (close(fd) != 0)\n \t\t\tdie_errno(_(\"cannot close written %s file '%s'\"),\n \t\t\t\t suffix, filename);\n-\t\t*report = suffix;\n+\t\tif (report)\n+\t\t\t*report = suffix;\n \t}\n \tstrbuf_release(&name_buf);\n }\n \n static void final(const char *final_pack_name, const char *curr_pack_name,\n \t\t const char *final_index_name, const char *curr_index_name,\n-\t\t const char *keep_msg, unsigned char *sha1)\n+\t\t const char *keep_msg, const char *promisor_msg,\n+\t\t unsigned char *sha1)\n {\n \tconst char *report = \"pack\";\n \tstruct strbuf pack_name = STRBUF_INIT;\n@@ -1455,6 +1457,9 @@ static void final(const char *final_pack_name, const char *curr_pack_name,\n \tif (keep_msg)\n \t\twrite_special_file(\"keep\", keep_msg, final_pack_name, sha1,\n \t\t\t\t &report);\n+\tif (promisor_msg)\n+\t\twrite_special_file(\"promisor\", promisor_msg, final_pack_name,\n+\t\t\t\t sha1, NULL);\n \n \tif (final_pack_name != curr_pack_name) {\n \t\tif (!final_pack_name)\n@@ -1644,6 +1649,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)\n \tconst char *curr_index;\n \tconst char *index_name = NULL, *pack_name = NULL;\n \tconst char *keep_msg = NULL;\n+\tconst char *promisor_msg = NULL;\n \tstruct strbuf index_name_buf = STRBUF_INIT;\n \tstruct pack_idx_entry **idx_objects;\n \tstruct pack_idx_option opts;\n@@ -1688,6 +1694,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)\n \t\t\t\tstat_only = 1;\n \t\t\t} else if (skip_to_optional_arg(arg, \"--keep\", &keep_msg)) {\n \t\t\t\t; /* nothing to do */\n+\t\t\t} else if (skip_to_optional_arg(arg, \"--promisor\", &promisor_msg)) {\n+\t\t\t\t; /* nothing to do */\n \t\t\t} else if (starts_with(arg, \"--threads=\")) {\n \t\t\t\tchar *end;\n \t\t\t\tnr_threads = strtoul(arg+10, &end, 0);\n@@ -1798,7 +1806,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)\n \tif (!verify)\n \t\tfinal(pack_name, curr_pack,\n \t\t index_name, curr_index,\n-\t\t keep_msg,\n+\t\t keep_msg, promisor_msg,\n \t\t pack_sha1);\n \telse\n \t\tclose(input_fd);\ndiff --git a/fetch-object.c b/fetch-object.c\nnew file mode 100644\nindex 0000000000..23ec2bb0d0\n--- /dev/null\n+++ b/fetch-object.c\n@@ -0,0 +1,28 @@\n+#include \"cache.h\"\n+#include \"packfile.h\"\n+#include \"pkt-line.h\"\n+#include \"strbuf.h\"\n+#include \"transport.h\"\n+#include \"fetch-object.h\"\n+\n+/*\n+ * TODO Investigate having fetch_object() return error/success,\n+ * changing callers.\n+ */\n+void fetch_object(const char *remote_name, const unsigned char *sha1)\n+{\n+\tstruct remote *remote;\n+\tstruct transport *transport;\n+\tstruct ref *ref;\n+\n+\tremote = remote_get(remote_name);\n+\tif (!remote->url[0])\n+\t\tdie(_(\"Remote with no URL\"));\n+\ttransport = transport_get(remote, remote->url[0]);\n+\n+\tref = alloc_ref(sha1_to_hex(sha1));\n+\thashcpy(ref->old_oid.hash, sha1);\n+\ttransport_set_option(transport, TRANS_OPT_FROM_PROMISOR, \"1\");\n+\ttransport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, \"1\");\n+\ttransport_fetch_refs(transport, ref);\n+}\ndiff --git a/fetch-object.h b/fetch-object.h\nnew file mode 100644\nindex 0000000000..f371300c88\n--- /dev/null\n+++ b/fetch-object.h\n@@ -0,0 +1,6 @@\n+#ifndef FETCH_OBJECT_H\n+#define FETCH_OBJECT_H\n+\n+extern void fetch_object(const char *remote_name, const unsigned char *sha1);\n+\n+#endif\ndiff --git a/fetch-pack.c b/fetch-pack.c\nindex 9f6b07ad91..7aa1f58995 100644\n--- a/fetch-pack.c\n+++ b/fetch-pack.c\n@@ -450,6 +450,8 @@ static int find_common(struct fetch_pack_args *args,\n \n \tflushes = 0;\n \tretval = -1;\n+\tif (args->no_dependents)\n+\t\tgoto done;\n \twhile ((oid = get_rev())) {\n \t\tpacket_buf_write(&req_buf, \"have %s\\n\", oid_to_hex(oid));\n \t\tprint_verbose(args, \"have %s\", oid_to_hex(oid));\n@@ -735,29 +737,31 @@ static int everything_local(struct fetch_pack_args *args,\n \t\t}\n \t}\n \n-\tif (!args->deepen) {\n-\t\tfor_each_ref(mark_complete_oid, NULL);\n-\t\tfor_each_cached_alternate(mark_alternate_complete);\n-\t\tcommit_list_sort_by_date(&complete);\n-\t\tif (cutoff)\n-\t\t\tmark_recent_complete_commits(args, cutoff);\n-\t}\n+\tif (!args->no_dependents) {\n+\t\tif (!args->deepen) {\n+\t\t\tfor_each_ref(mark_complete_oid, NULL);\n+\t\t\tfor_each_cached_alternate(mark_alternate_complete);\n+\t\t\tcommit_list_sort_by_date(&complete);\n+\t\t\tif (cutoff)\n+\t\t\t\tmark_recent_complete_commits(args, cutoff);\n+\t\t}\n \n-\t/*\n-\t * Mark all complete remote refs as common refs.\n-\t * Don't mark them common yet; the server has to be told so first.\n-\t */\n-\tfor (ref = *refs; ref; ref = ref->next) {\n-\t\tstruct object *o = deref_tag(lookup_object(ref->old_oid.hash),\n-\t\t\t\t\t NULL, 0);\n+\t\t/*\n+\t\t * Mark all complete remote refs as common refs.\n+\t\t * Don't mark them common yet; the server has to be told so first.\n+\t\t */\n+\t\tfor (ref = *refs; ref; ref = ref->next) {\n+\t\t\tstruct object *o = deref_tag(lookup_object(ref->old_oid.hash),\n+\t\t\t\t\t\t NULL, 0);\n \n-\t\tif (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))\n-\t\t\tcontinue;\n+\t\t\tif (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))\n+\t\t\t\tcontinue;\n \n-\t\tif (!(o->flags & SEEN)) {\n-\t\t\trev_list_push((struct commit *)o, COMMON_REF | SEEN);\n+\t\t\tif (!(o->flags & SEEN)) {\n+\t\t\t\trev_list_push((struct commit *)o, COMMON_REF | SEEN);\n \n-\t\t\tmark_common((struct commit *)o, 1, 1);\n+\t\t\t\tmark_common((struct commit *)o, 1, 1);\n+\t\t\t}\n \t\t}\n \t}\n \n@@ -833,7 +837,7 @@ static int get_pack(struct fetch_pack_args *args,\n \t\targv_array_push(&cmd.args, alternate_shallow_file);\n \t}\n \n-\tif (do_keep) {\n+\tif (do_keep || args->from_promisor) {\n \t\tif (pack_lockfile)\n \t\t\tcmd.out = -1;\n \t\tcmd_name = \"index-pack\";\n@@ -843,7 +847,7 @@ static int get_pack(struct fetch_pack_args *args,\n \t\t\targv_array_push(&cmd.args, \"-v\");\n \t\tif (args->use_thin_pack)\n \t\t\targv_array_push(&cmd.args, \"--fix-thin\");\n-\t\tif (args->lock_pack || unpack_limit) {\n+\t\tif (do_keep && (args->lock_pack || unpack_limit)) {\n \t\t\tchar hostname[HOST_NAME_MAX + 1];\n \t\t\tif (xgethostname(hostname, sizeof(hostname)))\n \t\t\t\txsnprintf(hostname, sizeof(hostname), \"localhost\");\n@@ -853,6 +857,8 @@ static int get_pack(struct fetch_pack_args *args,\n \t\t}\n \t\tif (args->check_self_contained_and_connected)\n \t\t\targv_array_push(&cmd.args, \"--check-self-contained-and-connected\");\n+\t\tif (args->from_promisor)\n+\t\t\targv_array_push(&cmd.args, \"--promisor\");\n \t}\n \telse {\n \t\tcmd_name = \"unpack-objects\";\ndiff --git a/fetch-pack.h b/fetch-pack.h\nindex b6aeb43a8e..aeac152644 100644\n--- a/fetch-pack.h\n+++ b/fetch-pack.h\n@@ -29,6 +29,14 @@ struct fetch_pack_args {\n \tunsigned cloning:1;\n \tunsigned update_shallow:1;\n \tunsigned deepen:1;\n+\tunsigned from_promisor:1;\n+\n+\t/*\n+\t * If 1, fetch_pack() will also not modify any object flags.\n+\t * This allows fetch_pack() to safely be called by any function,\n+\t * regardless of which object flags it uses (if any).\n+\t */\n+\tunsigned no_dependents:1;\n };\n \n /*\ndiff --git a/remote-curl.c b/remote-curl.c\nindex 0053b09549..431839111a 100644\n--- a/remote-curl.c\n+++ b/remote-curl.c\n@@ -33,7 +33,9 @@ struct options {\n \t\tthin : 1,\n \t\t/* One of the SEND_PACK_PUSH_CERT_* constants. */\n \t\tpush_cert : 2,\n-\t\tdeepen_relative : 1;\n+\t\tdeepen_relative : 1,\n+\t\tfrom_promisor : 1,\n+\t\tno_dependents : 1;\n };\n static struct options options;\n static struct string_list cas_options = STRING_LIST_INIT_DUP;\n@@ -157,6 +159,12 @@ static int set_option(const char *name, const char *value)\n \t\t\treturn -1;\n \t\treturn 0;\n #endif /* LIBCURL_VERSION_NUM >= 0x070a08 */\n+\t} else if (!strcmp(name, \"from-promisor\")) {\n+\t\toptions.from_promisor = 1;\n+\t\treturn 0;\n+\t} else if (!strcmp(name, \"no-dependents\")) {\n+\t\toptions.no_dependents = 1;\n+\t\treturn 0;\n \t} else {\n \t\treturn 1 /* unsupported */;\n \t}\n@@ -822,6 +830,10 @@ static int fetch_git(struct discovery *heads,\n \t\t\t\t options.deepen_not.items[i].string);\n \tif (options.deepen_relative && options.depth)\n \t\targv_array_push(&args, \"--deepen-relative\");\n+\tif (options.from_promisor)\n+\t\targv_array_push(&args, \"--from-promisor\");\n+\tif (options.no_dependents)\n+\t\targv_array_push(&args, \"--no-dependents\");\n \targv_array_push(&args, url.buf);\n \n \tfor (i = 0; i < nr_heads; i++) {\ndiff --git a/transport.c b/transport.c\nindex fc802260f6..e82db773fd 100644\n--- a/transport.c\n+++ b/transport.c\n@@ -161,6 +161,12 @@ static int set_git_option(struct git_transport_options *opts,\n \t} else if (!strcmp(name, TRANS_OPT_DEEPEN_RELATIVE)) {\n \t\topts->deepen_relative = !!value;\n \t\treturn 0;\n+\t} else if (!strcmp(name, TRANS_OPT_FROM_PROMISOR)) {\n+\t\topts->from_promisor = !!value;\n+\t\treturn 0;\n+\t} else if (!strcmp(name, TRANS_OPT_NO_DEPENDENTS)) {\n+\t\topts->no_dependents = !!value;\n+\t\treturn 0;\n \t}\n \treturn 1;\n }\n@@ -229,6 +235,8 @@ static int fetch_refs_via_pack(struct transport *transport,\n \t\tdata->options.check_self_contained_and_connected;\n \targs.cloning = transport->cloning;\n \targs.update_shallow = data->options.update_shallow;\n+\targs.from_promisor = data->options.from_promisor;\n+\targs.no_dependents = data->options.no_dependents;\n \n \tif (!data->got_remote_heads) {\n \t\tconnect_setup(transport, 0);\ndiff --git a/transport.h b/transport.h\nindex 731c78b679..8c3430a5b9 100644\n--- a/transport.h\n+++ b/transport.h\n@@ -15,6 +15,8 @@ struct git_transport_options {\n \tunsigned self_contained_and_connected : 1;\n \tunsigned update_shallow : 1;\n \tunsigned deepen_relative : 1;\n+\tunsigned from_promisor : 1;\n+\tunsigned no_dependents : 1;\n \tint depth;\n \tconst char *deepen_since;\n \tconst struct string_list *deepen_not;\n@@ -159,6 +161,15 @@ void transport_check_allowed(const char *type);\n /* Send push certificates */\n #define TRANS_OPT_PUSH_CERT \"pushcert\"\n \n+/* Indicate that these objects are being fetched by a promisor */\n+#define TRANS_OPT_FROM_PROMISOR \"from-promisor\"\n+\n+/*\n+ * Indicate that only the objects wanted need to be fetched, not their\n+ * dependents\n+ */\n+#define TRANS_OPT_NO_DEPENDENTS \"no-dependents\"\n+\n /**\n * Returns 0 if the option was used, non-zero otherwise. Prints a\n * message to stderr if the option is not used.\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335725","messageId":"20180103163403.11303-19-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 18/40] t0400: add 'put_raw_obj' instruction to odb-helper script","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:41Z","receivedAt":"2018-01-03T16:35:05Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"To properly test passing objects from Git to an external odb\nwe need an odb-helper script that supports a 'put'\ncapability/instruction.\n\nFor now we will support only sending raw blobs, so the\nsupported capability/instruction will be 'put_raw_obj'.\n\nWhile at it let's add a test to check that our odb-helper\nscript works well.\n\nSigned-off-by: Christian Couder \n---\n t/t0400-external-odb.sh | 24 ++++++++++++++++++++++++\n 1 file changed, 24 insertions(+)\n\ndiff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh\nindex 977fea852d..4ccca1e965 100755\n--- a/t/t0400-external-odb.sh\n+++ b/t/t0400-external-odb.sh\n@@ -7,10 +7,15 @@ test_description='basic tests for external object databases'\n ALT_SOURCE=\"$PWD/alt-repo/.git\"\n export ALT_SOURCE\n write_script odb-helper <<\\EOF\n+die() {\n+\tprintf >&2 \"%s\\n\" \"$@\"\n+\texit 1\n+}\n GIT_DIR=$ALT_SOURCE; export GIT_DIR\n case \"$1\" in\n init)\n \techo \"capability=get_git_obj\"\n+\techo \"capability=put_raw_obj\"\n \techo \"capability=have\"\n \t;;\n have)\n@@ -20,6 +25,16 @@ have)\n get_git_obj)\n \tcat \"$GIT_DIR\"/objects/$(echo $2 | sed 's#..#&/#')\n \t;;\n+put_raw_obj)\n+\tsha1=\"$2\"\n+\tsize=\"$3\"\n+\tkind=\"$4\"\n+\twritten=$(git hash-object -w -t \"$kind\" --stdin)\n+\ttest \"$written\" = \"$sha1\" || die \"bad sha1 passed '$sha1' vs written '$written'\"\n+\t;;\n+*)\n+\tdie \"unknown command '$1'\"\n+\t;;\n esac\n EOF\n HELPER=\"\\\"$PWD\\\"/odb-helper\"\n@@ -45,4 +60,13 @@ test_expect_success 'helper can retrieve alt objects' '\n \ttest_cmp expect actual\n '\n \n+test_expect_success 'helper can add objects to alt repo' '\n+\thash=$(echo \"Hello odb!\" | git hash-object -w -t blob --stdin) &&\n+\ttest -f .git/objects/$(echo $hash | sed \"s#..#&/#\") &&\n+\tsize=$(git cat-file -s \"$hash\") &&\n+\tgit cat-file blob \"$hash\" | ./odb-helper put_raw_obj \"$hash\" \"$size\" blob &&\n+\talt_size=$(git -C alt-repo cat-file -s \"$hash\") &&\n+\ttest \"$size\" -eq \"$alt_size\"\n+'\n+\n test_done\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335726","messageId":"20180103163403.11303-22-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 21/40] t0400: add test for external odb write support","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:44Z","receivedAt":"2018-01-03T16:35:07Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"Signed-off-by: Christian Couder \n---\n t/t0400-external-odb.sh | 8 ++++++++\n 1 file changed, 8 insertions(+)\n\ndiff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh\nindex 4ccca1e965..f924de870f 100755\n--- a/t/t0400-external-odb.sh\n+++ b/t/t0400-external-odb.sh\n@@ -69,4 +69,12 @@ test_expect_success 'helper can add objects to alt repo' '\n \ttest \"$size\" -eq \"$alt_size\"\n '\n \n+test_expect_success 'commit adds objects to alt repo' '\n+\ttest_config odb.magic.scriptCommand \"$HELPER\" &&\n+\ttest_commit three &&\n+\thash3=$(git ls-tree HEAD | grep three.t | cut -f1 | cut -d\\ -f3) &&\n+\tcontent=$(git -C alt-repo show \"$hash3\") &&\n+\ttest \"$content\" = \"three\"\n+'\n+\n test_done\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335727","messageId":"20180103163403.11303-25-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 24/40] lib-httpd: add upload.sh","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:47Z","receivedAt":"2018-01-03T16:35:10Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This cgi will be used to upload objects to, or to delete\nobjects from, an apache web server.\n\nThis way the apache server can work as an external object\ndatabase.\n\nSigned-off-by: Christian Couder \n---\n t/lib-httpd.sh | 1 +\n t/lib-httpd/upload.sh | 45 +++++++++++++++++++++++++++++++++++++++++++++\n 2 files changed, 46 insertions(+)\n create mode 100644 t/lib-httpd/upload.sh\n\ndiff --git a/t/lib-httpd.sh b/t/lib-httpd.sh\nindex 2e659a8ee2..d80b004549 100644\n--- a/t/lib-httpd.sh\n+++ b/t/lib-httpd.sh\n@@ -132,6 +132,7 @@ prepare_httpd() {\n \tcp \"$TEST_PATH\"/passwd \"$HTTPD_ROOT_PATH\"\n \tinstall_script broken-smart-http.sh\n \tinstall_script error.sh\n+\tinstall_script upload.sh\n \n \tln -s \"$LIB_HTTPD_MODULE_PATH\" \"$HTTPD_ROOT_PATH/modules\"\n \ndiff --git a/t/lib-httpd/upload.sh b/t/lib-httpd/upload.sh\nnew file mode 100644\nindex 0000000000..64d3f31c31\n--- /dev/null\n+++ b/t/lib-httpd/upload.sh\n@@ -0,0 +1,45 @@\n+#!/bin/sh\n+\n+# In part from http://codereview.stackexchange.com/questions/79549/bash-cgi-upload-file\n+\n+FILES_DIR=\"www/files\"\n+\n+OLDIFS=\"$IFS\"\n+IFS='&'\n+set -- $QUERY_STRING\n+IFS=\"$OLDIFS\"\n+\n+while test $# -gt 0\n+do\n+\tkey=${1%%=*}\n+\tval=${1#*=}\n+\n+\tcase \"$key\" in\n+\t\"sha1\") sha1=\"$val\" ;;\n+\t\"type\") type=\"$val\" ;;\n+\t\"size\") size=\"$val\" ;;\n+\t\"delete\") delete=1 ;;\n+\t*) echo >&2 \"unknown key '$key'\" ;;\n+\tesac\n+\n+\tshift\n+done\n+\n+case \"$REQUEST_METHOD\" in\n+POST)\n+\tif test \"$delete\" = \"1\"\n+\tthen\n+\t\trm -f \"$FILES_DIR/$sha1-$size-$type\"\n+\telse\n+\t\tmkdir -p \"$FILES_DIR\"\n+\t\tcat >\"$FILES_DIR/$sha1-$size-$type\"\n+\tfi\n+\n+\techo 'Status: 204 No Content'\n+\techo\n+\t;;\n+\n+*)\n+\techo 'Status: 405 Method Not Allowed'\n+\techo\n+esac\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335728","messageId":"20180103163403.11303-26-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 25/40] lib-httpd: add list.sh","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:48Z","receivedAt":"2018-01-03T16:35:13Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This cgi script can list Git objects that have been uploaded as\nfiles to an apache web server. This script can also retrieve\nthe content of each of these files.\n\nThis will help make apache work as an external object database.\n\nSigned-off-by: Christian Couder \n---\n t/lib-httpd.sh | 1 +\n t/lib-httpd/list.sh | 41 +++++++++++++++++++++++++++++++++++++++++\n 2 files changed, 42 insertions(+)\n create mode 100644 t/lib-httpd/list.sh\n\ndiff --git a/t/lib-httpd.sh b/t/lib-httpd.sh\nindex d80b004549..f31ea261f5 100644\n--- a/t/lib-httpd.sh\n+++ b/t/lib-httpd.sh\n@@ -133,6 +133,7 @@ prepare_httpd() {\n \tinstall_script broken-smart-http.sh\n \tinstall_script error.sh\n \tinstall_script upload.sh\n+\tinstall_script list.sh\n \n \tln -s \"$LIB_HTTPD_MODULE_PATH\" \"$HTTPD_ROOT_PATH/modules\"\n \ndiff --git a/t/lib-httpd/list.sh b/t/lib-httpd/list.sh\nnew file mode 100644\nindex 0000000000..b6d6c29a2f\n--- /dev/null\n+++ b/t/lib-httpd/list.sh\n@@ -0,0 +1,41 @@\n+#!/bin/sh\n+\n+FILES_DIR=\"www/files\"\n+\n+OLDIFS=\"$IFS\"\n+IFS='&'\n+set -- $QUERY_STRING\n+IFS=\"$OLDIFS\"\n+\n+while test $# -gt 0\n+do\n+\tkey=${1%%=*}\n+\tval=${1#*=}\n+\n+\tcase \"$key\" in\n+\t\"sha1\") sha1=\"$val\" ;;\n+\t*) echo >&2 \"unknown key '$key'\" ;;\n+\tesac\n+\n+\tshift\n+done\n+\n+if test -d \"$FILES_DIR\"\n+then\n+\tif test -z \"$sha1\"\n+\tthen\n+\t\techo 'Status: 200 OK'\n+\t\techo\n+\t\tls \"$FILES_DIR\" | tr '-' ' '\n+\telse\n+\t\tif test -f \"$FILES_DIR/$sha1\"-*\n+\t\tthen\n+\t\t\techo 'Status: 200 OK'\n+\t\t\techo\n+\t\t\tcat \"$FILES_DIR/$sha1\"-*\n+\t\telse\n+\t\t\techo 'Status: 404 Not Found'\n+\t\t\techo\n+\t\tfi\n+\tfi\n+fi\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335729","messageId":"20180103163403.11303-24-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 23/40] lib-httpd: pass config file to start_httpd()","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:46Z","receivedAt":"2018-01-03T16:35:15Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This makes it possible to start an apache web server with different\nconfig files.\n\nThis will be used in a later patch to pass a config file that makes\napache store external objects.\n\nSigned-off-by: Christian Couder \n---\n t/lib-httpd.sh | 6 ++++--\n 1 file changed, 4 insertions(+), 2 deletions(-)\n\ndiff --git a/t/lib-httpd.sh b/t/lib-httpd.sh\nindex 435a37465a..2e659a8ee2 100644\n--- a/t/lib-httpd.sh\n+++ b/t/lib-httpd.sh\n@@ -171,12 +171,14 @@ prepare_httpd() {\n }\n \n start_httpd() {\n+\tAPACHE_CONF_FILE=${1-apache.conf}\n+\n \tprepare_httpd >&3 2>&4\n \n \ttrap 'code=$?; stop_httpd; (exit $code); die' EXIT\n \n \t\"$LIB_HTTPD_PATH\" -d \"$HTTPD_ROOT_PATH\" \\\n-\t\t-f \"$TEST_PATH/apache.conf\" $HTTPD_PARA \\\n+\t\t-f \"$TEST_PATH/$APACHE_CONF_FILE\" $HTTPD_PARA \\\n \t\t-c \"Listen 127.0.0.1:$LIB_HTTPD_PORT\" -k start \\\n \t\t>&3 2>&4\n \tif test $? -ne 0\n@@ -191,7 +193,7 @@ stop_httpd() {\n \ttrap 'die' EXIT\n \n \t\"$LIB_HTTPD_PATH\" -d \"$HTTPD_ROOT_PATH\" \\\n-\t\t-f \"$TEST_PATH/apache.conf\" $HTTPD_PARA -k stop\n+\t\t-f \"$TEST_PATH/$APACHE_CONF_FILE\" $HTTPD_PARA -k stop\n }\n \n test_http_push_nonff () {\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335730","messageId":"20180103163403.11303-28-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 27/40] odb-helper: add odb_helper_get_raw_object()","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:50Z","receivedAt":"2018-01-03T16:35:18Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"The existing odb_helper_get_object() is renamed\nodb_helper_get_git_object() and a new odb_helper_get_raw_object()\nis introduced to deal with external objects that are not in Git format.\n\nSigned-off-by: Christian Couder \n---\n odb-helper.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--\n 1 file changed, 111 insertions(+), 2 deletions(-)\n\ndiff --git a/odb-helper.c b/odb-helper.c\nindex 6f56f07b38..fc30c2fa57 100644\n--- a/odb-helper.c\n+++ b/odb-helper.c\n@@ -222,8 +222,107 @@ int odb_helper_has_object(struct odb_helper *o, const unsigned char *sha1)\n \treturn !!odb_helper_lookup(o, sha1);\n }\n \n-int odb_helper_get_object(struct odb_helper *o, const unsigned char *sha1,\n-\t\t\t int fd)\n+static int odb_helper_get_raw_object(struct odb_helper *o,\n+\t\t\t\t const unsigned char *sha1,\n+\t\t\t\t int fd)\n+{\n+\tstruct odb_helper_object *obj;\n+\tstruct odb_helper_cmd cmd;\n+\tunsigned long total_got = 0;\n+\n+\tchar hdr[32];\n+\tint hdrlen;\n+\n+\tint ret = Z_STREAM_END;\n+\tunsigned char compressed[4096];\n+\tgit_zstream stream;\n+\tgit_SHA_CTX hash;\n+\tunsigned char real_sha1[20];\n+\n+\tobj = odb_helper_lookup(o, sha1);\n+\tif (!obj)\n+\t\treturn -1;\n+\n+\tif (odb_helper_start(o, &cmd, 0, \"get_raw_obj %s\", sha1_to_hex(sha1)) < 0)\n+\t\treturn -1;\n+\n+\t/* Set it up */\n+\tgit_deflate_init(&stream, zlib_compression_level);\n+\tstream.next_out = compressed;\n+\tstream.avail_out = sizeof(compressed);\n+\tgit_SHA1_Init(&hash);\n+\n+\t/* First header.. */\n+\thdrlen = xsnprintf(hdr, sizeof(hdr), \"%s %lu\", typename(obj->type), obj->size) + 1;\n+\tstream.next_in = (unsigned char *)hdr;\n+\tstream.avail_in = hdrlen;\n+\twhile (git_deflate(&stream, 0) == Z_OK)\n+\t\t; /* nothing */\n+\tgit_SHA1_Update(&hash, hdr, hdrlen);\n+\n+\tfor (;;) {\n+\t\tunsigned char buf[4096];\n+\t\tint r;\n+\n+\t\tr = xread(cmd.child.out, buf, sizeof(buf));\n+\t\tif (r < 0) {\n+\t\t\terror(\"unable to read from odb helper '%s': %s\",\n+\t\t\t o->name, strerror(errno));\n+\t\t\tclose(cmd.child.out);\n+\t\t\todb_helper_finish(o, &cmd);\n+\t\t\tgit_deflate_end(&stream);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tif (r == 0)\n+\t\t\tbreak;\n+\n+\t\ttotal_got += r;\n+\n+\t\t/* Then the data itself.. */\n+\t\tstream.next_in = (void *)buf;\n+\t\tstream.avail_in = r;\n+\t\tdo {\n+\t\t\tunsigned char *in0 = stream.next_in;\n+\t\t\tret = git_deflate(&stream, Z_FINISH);\n+\t\t\tgit_SHA1_Update(&hash, in0, stream.next_in - in0);\n+\t\t\twrite_or_die(fd, compressed, stream.next_out - compressed);\n+\t\t\tstream.next_out = compressed;\n+\t\t\tstream.avail_out = sizeof(compressed);\n+\t\t} while (ret == Z_OK);\n+\t}\n+\n+\tclose(cmd.child.out);\n+\tif (ret != Z_STREAM_END) {\n+\t\twarning(\"bad zlib data from odb helper '%s' for %s\",\n+\t\t\to->name, sha1_to_hex(sha1));\n+\t\treturn -1;\n+\t}\n+\tret = git_deflate_end_gently(&stream);\n+\tif (ret != Z_OK) {\n+\t\twarning(\"deflateEnd on object %s from odb helper '%s' failed (%d)\",\n+\t\t\tsha1_to_hex(sha1), o->name, ret);\n+\t\treturn -1;\n+\t}\n+\tgit_SHA1_Final(real_sha1, &hash);\n+\tif (hashcmp(sha1, real_sha1)) {\n+\t\twarning(\"sha1 mismatch from odb helper '%s' for %s (got %s)\",\n+\t\t\to->name, sha1_to_hex(sha1), sha1_to_hex(real_sha1));\n+\t\treturn -1;\n+\t}\n+\tif (odb_helper_finish(o, &cmd))\n+\t\treturn -1;\n+\tif (total_got != obj->size) {\n+\t\twarning(\"size mismatch from odb helper '%s' for %s (%lu != %lu)\",\n+\t\t\to->name, sha1_to_hex(sha1), total_got, obj->size);\n+\t\treturn -1;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int odb_helper_get_git_object(struct odb_helper *o,\n+\t\t\t\t const unsigned char *sha1,\n+\t\t\t\t int fd)\n {\n \tstruct odb_helper_object *obj;\n \tstruct odb_helper_cmd cmd;\n@@ -340,6 +439,16 @@ int odb_helper_get_direct(struct odb_helper *o,\n \treturn res;\n }\n \n+int odb_helper_get_object(struct odb_helper *o,\n+\t\t\t const unsigned char *sha1,\n+\t\t\t int fd)\n+{\n+\tif (o->supported_capabilities & ODB_HELPER_CAP_GET_RAW_OBJ)\n+\t\treturn odb_helper_get_raw_object(o, sha1, fd);\n+\telse\n+\t\treturn odb_helper_get_git_object(o, sha1, fd);\n+}\n+\n int odb_helper_put_object(struct odb_helper *o,\n \t\t\t const void *buf, size_t len,\n \t\t\t const char *type, unsigned char *sha1)\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335731","messageId":"20180103163403.11303-30-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 29/40] Add t0420 to test transfer to HTTP external odb","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:52Z","receivedAt":"2018-01-03T16:35:19Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This tests that an apache web server can be used as an\nexternal object database and store files in their native\nformat instead of converting them to a Git object.\n\nSigned-off-by: Christian Couder \n---\n t/t0420-transfer-http-e-odb.sh | 142 +++++++++++++++++++++++++++++++++++++++++\n 1 file changed, 142 insertions(+)\n create mode 100755 t/t0420-transfer-http-e-odb.sh\n\ndiff --git a/t/t0420-transfer-http-e-odb.sh b/t/t0420-transfer-http-e-odb.sh\nnew file mode 100755\nindex 0000000000..f84fe950ec\n--- /dev/null\n+++ b/t/t0420-transfer-http-e-odb.sh\n@@ -0,0 +1,142 @@\n+#!/bin/sh\n+\n+test_description='tests for transfering external objects to an HTTPD server'\n+\n+. ./test-lib.sh\n+\n+# If we don't specify a port, the current test number will be used\n+# which will not work as it is less than 1024, so it can only be used by root.\n+LIB_HTTPD_PORT=$(expr ${this_test#t} + 12000)\n+\n+. \"$TEST_DIRECTORY\"/lib-httpd.sh\n+\n+start_httpd apache-e-odb.conf\n+\n+# odb helper script must see this\n+export HTTPD_URL\n+\n+write_script odb-http-helper <<\\EOF\n+die() {\n+\tprintf >&2 \"%s\\n\" \"$@\"\n+\texit 1\n+}\n+echo >&2 \"odb-http-helper args:\" \"$@\"\n+case \"$1\" in\n+init)\n+\techo \"capability=get_raw_obj\"\n+\techo \"capability=put_raw_obj\"\n+\techo \"capability=have\"\n+\t;;\n+have)\n+\tlist_url=\"$HTTPD_URL/list/\"\n+\tcurl \"$list_url\" ||\n+\tdie \"curl '$list_url' failed\"\n+\t;;\n+get_raw_obj)\n+\tget_url=\"$HTTPD_URL/list/?sha1=$2\"\n+\tcurl \"$get_url\" ||\n+\tdie \"curl '$get_url' failed\"\n+\t;;\n+put_raw_obj)\n+\tsha1=\"$2\"\n+\tsize=\"$3\"\n+\tkind=\"$4\"\n+\tupload_url=\"$HTTPD_URL/upload/?sha1=$sha1&size=$size&type=$kind\"\n+\tcurl --data-binary @- --include \"$upload_url\" >out ||\n+\tdie \"curl '$upload_url' failed\"\n+\tref_hash=$(echo \"$sha1 $size $kind\" | GIT_NO_EXTERNAL_ODB=1 git hash-object -w -t blob --stdin) || exit\n+\tgit update-ref refs/odbs/magic/\"$sha1\" \"$ref_hash\"\n+\t;;\n+*)\n+\tdie \"unknown command '$1'\"\n+\t;;\n+esac\n+EOF\n+HELPER=\"\\\"$PWD\\\"/odb-http-helper\"\n+\n+test_expect_success 'setup repo with a root commit and the helper' '\n+\ttest_commit zero &&\n+\tgit config odb.magic.scriptCommand \"$HELPER\"\n+'\n+\n+test_expect_success 'setup another repo from the first one' '\n+\tgit init other-repo &&\n+\t(cd other-repo &&\n+\t git remote add origin .. &&\n+\t git pull origin master &&\n+\t git checkout master &&\n+\t git log)\n+'\n+\n+UPLOADFILENAME=\"hello_apache_upload.txt\"\n+\n+UPLOAD_URL=\"$HTTPD_URL/upload/?sha1=$UPLOADFILENAME&size=123&type=blob\"\n+\n+test_expect_success 'can upload a file' '\n+\techo \"Hello Apache World!\" >hello_to_send.txt &&\n+\techo \"How are you?\" >>hello_to_send.txt &&\n+\tcurl --data-binary @hello_to_send.txt --include \"$UPLOAD_URL\" >out_upload\n+'\n+\n+LIST_URL=\"$HTTPD_URL/list/\"\n+\n+test_expect_success 'can list uploaded files' '\n+\tcurl --include \"$LIST_URL\" >out_list &&\n+\tgrep \"$UPLOADFILENAME\" out_list\n+'\n+\n+test_expect_success 'can delete uploaded files' '\n+\tcurl --data \"delete\" --include \"$UPLOAD_URL&delete=1\" >out_delete &&\n+\tcurl --include \"$LIST_URL\" >out_list2 &&\n+\t! grep \"$UPLOADFILENAME\" out_list2\n+'\n+\n+FILES_DIR=\"httpd/www/files\"\n+\n+test_expect_success 'new blobs are transfered to the http server' '\n+\ttest_commit one &&\n+\thash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\\ -f3) &&\n+\techo \"$hash1-4-blob\" >expected &&\n+\tls \"$FILES_DIR\" >actual &&\n+\ttest_cmp expected actual\n+'\n+\n+test_expect_success 'blobs can be retrieved from the http server' '\n+\tgit cat-file blob \"$hash1\" &&\n+\tgit log -p >expected\n+'\n+\n+test_expect_success 'update other repo from the first one' '\n+\t(cd other-repo &&\n+\t git fetch origin \"refs/odbs/magic/*:refs/odbs/magic/*\" &&\n+\t test_must_fail git cat-file blob \"$hash1\" &&\n+\t git config odb.magic.scriptCommand \"$HELPER\" &&\n+\t git cat-file blob \"$hash1\" &&\n+\t git pull origin master)\n+'\n+\n+test_expect_success 'local clone from the first repo' '\n+\tmkdir my-clone &&\n+\t(cd my-clone &&\n+\t git clone .. . &&\n+\t git cat-file blob \"$hash1\")\n+'\n+\n+test_expect_success 'no-local clone from the first repo fails' '\n+\tmkdir my-other-clone &&\n+\t(cd my-other-clone &&\n+\t test_must_fail git clone --no-local .. .) &&\n+\trm -rf my-other-clone\n+'\n+\n+test_expect_success 'no-local clone from the first repo with helper succeeds' '\n+\tmkdir my-other-clone &&\n+\t(cd my-other-clone &&\n+\t git clone -c odb.magic.scriptCommand=\"$HELPER\" \\\n+\t\t--no-local .. .) &&\n+\trm -rf my-other-clone\n+'\n+\n+stop_httpd\n+\n+test_done\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335732","messageId":"20180103163403.11303-27-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 26/40] lib-httpd: add apache-e-odb.conf","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:49Z","receivedAt":"2018-01-03T16:35:21Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This is an apache config file to test external object databases.\nIt uses the upload.sh and list.sh cgi that have been added\npreviously to make apache store external objects.\n\nSigned-off-by: Christian Couder \n---\n t/lib-httpd/apache-e-odb.conf | 214 ++++++++++++++++++++++++++++++++++++++++++\n 1 file changed, 214 insertions(+)\n create mode 100644 t/lib-httpd/apache-e-odb.conf\n\ndiff --git a/t/lib-httpd/apache-e-odb.conf b/t/lib-httpd/apache-e-odb.conf\nnew file mode 100644\nindex 0000000000..19a1540c82\n--- /dev/null\n+++ b/t/lib-httpd/apache-e-odb.conf\n@@ -0,0 +1,214 @@\n+ServerName dummy\n+PidFile httpd.pid\n+DocumentRoot www\n+LogFormat \"%h %l %u %t \\\"%r\\\" %>s %b\" common\n+CustomLog access.log common\n+ErrorLog error.log\n+\n+\tLoadModule log_config_module modules/mod_log_config.so\n+\n+\n+\tLoadModule alias_module modules/mod_alias.so\n+\n+\n+\tLoadModule cgi_module modules/mod_cgi.so\n+\n+\n+\tLoadModule env_module modules/mod_env.so\n+\n+\n+\tLoadModule rewrite_module modules/mod_rewrite.so\n+\n+\n+\tLoadModule version_module modules/mod_version.so\n+\n+\n+\tLoadModule headers_module modules/mod_headers.so\n+\n+\n+\n+LockFile accept.lock\n+\n+\n+\n+\n+\tLoadModule auth_module modules/mod_auth.so\n+\n+\n+\n+= 2.1>\n+\n+\tLoadModule auth_basic_module modules/mod_auth_basic.so\n+\n+\n+\tLoadModule authn_file_module modules/mod_authn_file.so\n+\n+\n+\tLoadModule authz_user_module modules/mod_authz_user.so\n+\n+\n+\tLoadModule authz_host_module modules/mod_authz_host.so\n+\n+\n+\n+= 2.4>\n+\n+\tLoadModule authn_core_module modules/mod_authn_core.so\n+\n+\n+\tLoadModule authz_core_module modules/mod_authz_core.so\n+\n+\n+\tLoadModule access_compat_module modules/mod_access_compat.so\n+\n+\n+\tLoadModule mpm_prefork_module modules/mod_mpm_prefork.so\n+\n+\n+\tLoadModule unixd_module modules/mod_unixd.so\n+\n+\n+\n+PassEnv GIT_VALGRIND\n+PassEnv GIT_VALGRIND_OPTIONS\n+PassEnv GNUPGHOME\n+PassEnv ASAN_OPTIONS\n+PassEnv GIT_TRACE\n+PassEnv GIT_CONFIG_NOSYSTEM\n+\n+Alias /dumb/ www/\n+Alias /auth/dumb/ www/auth/dumb/\n+\n+\n+\tSetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}\n+\tSetEnv GIT_HTTP_EXPORT_ALL\n+\n+\n+\tSetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}\n+\n+\n+\tSetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}\n+\tSetEnv GIT_HTTP_EXPORT_ALL\n+\tSetEnv GIT_COMMITTER_NAME \"Custom User\"\n+\tSetEnv GIT_COMMITTER_EMAIL custom@example.com\n+\n+\n+\tSetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}\n+\tSetEnv GIT_HTTP_EXPORT_ALL\n+\tSetEnv GIT_NAMESPACE ns\n+\n+\n+\tSetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}\n+\tSetEnv GIT_HTTP_EXPORT_ALL\n+\tHeader set Set-Cookie name=value\n+\n+\n+\tSetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}\n+\tSetEnv GIT_HTTP_EXPORT_ALL\n+\n+ScriptAlias /upload/ upload.sh/\n+ScriptAlias /list/ list.sh/\n+\n+\tOptions FollowSymlinks\n+\n+\n+ Options ExecCGI\n+\n+\n+ Options ExecCGI\n+\n+\n+\tOptions ExecCGI\n+\n+\n+RewriteEngine on\n+RewriteRule ^/smart-redir-perm/(.*)$ /smart/$1 [R=301]\n+RewriteRule ^/smart-redir-temp/(.*)$ /smart/$1 [R=302]\n+RewriteRule ^/smart-redir-auth/(.*)$ /auth/smart/$1 [R=301]\n+RewriteRule ^/smart-redir-limited/(.*)/info/refs$ /smart/$1/info/refs [R=301]\n+RewriteRule ^/ftp-redir/(.*)$ ftp://localhost:1000/$1 [R=302]\n+\n+RewriteRule ^/loop-redir/x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-(.*) /$1 [R=302]\n+RewriteRule ^/loop-redir/(.*)$ /loop-redir/x-$1 [R=302]\n+\n+# Apache 2.2 does not understand , so we use RewriteCond.\n+# And as RewriteCond does not allow testing for non-matches, we match\n+# the desired case first (one has abra, two has cadabra), and let it\n+# pass by marking the RewriteRule as [L], \"last rule, do not process\n+# any other matching RewriteRules after this\"), and then have another\n+# RewriteRule that matches all other cases and lets them fail via '[F]',\n+# \"fail the request\".\n+RewriteCond %{HTTP:x-magic-one} =abra\n+RewriteCond %{HTTP:x-magic-two} =cadabra\n+RewriteRule ^/smart_headers/.* - [L]\n+RewriteRule ^/smart_headers/.* - [F]\n+\n+\n+LoadModule ssl_module modules/mod_ssl.so\n+\n+SSLCertificateFile httpd.pem\n+SSLCertificateKeyFile httpd.pem\n+SSLRandomSeed startup file:/dev/urandom 512\n+SSLRandomSeed connect file:/dev/urandom 512\n+SSLSessionCache none\n+SSLMutex file:ssl_mutex\n+SSLEngine On\n+\n+\n+\n+\tAuthType Basic\n+\tAuthName \"git-auth\"\n+\tAuthUserFile passwd\n+\tRequire valid-user\n+\n+\n+\n+\tAuthType Basic\n+\tAuthName \"git-auth\"\n+\tAuthUserFile passwd\n+\tRequire valid-user\n+\n+\n+\n+\tAuthType Basic\n+\tAuthName \"git-auth\"\n+\tAuthUserFile passwd\n+\tRequire valid-user\n+\n+\n+RewriteCond %{QUERY_STRING} service=git-receive-pack [OR]\n+RewriteCond %{REQUEST_URI} /git-receive-pack$\n+RewriteRule ^/half-auth-complete/ - [E=AUTHREQUIRED:yes]\n+\n+\n+ Order Deny,Allow\n+ Deny from env=AUTHREQUIRED\n+\n+ AuthType Basic\n+ AuthName \"Git Access\"\n+ AuthUserFile passwd\n+ Require valid-user\n+ Satisfy Any\n+\n+\n+\n+\tLoadModule dav_module modules/mod_dav.so\n+\tLoadModule dav_fs_module modules/mod_dav_fs.so\n+\n+\tDAVLockDB DAVLock\n+\t\n+\t\tDav on\n+\t\n+\t\n+\t\tDav on\n+\t\n+\n+\n+\n+\tLoadModule dav_svn_module modules/mod_dav_svn.so\n+\n+\t\n+\t\tDAV svn\n+\t\tSVNPath \"${LIB_HTTPD_SVNPATH}\"\n+\t\n+\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335733","messageId":"20180103163403.11303-29-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 28/40] pack-objects: don't pack objects in external odbs","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:51Z","receivedAt":"2018-01-03T16:35:25Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"Objects managed by an external ODB should not be put into\npack files. They should be transfered using other mechanism\nthat can be specific to the external odb.\n\nSigned-off-by: Christian Couder \n---\n builtin/pack-objects.c | 4 ++++\n 1 file changed, 4 insertions(+)\n\ndiff --git a/builtin/pack-objects.c b/builtin/pack-objects.c\nindex 6c71552cdf..4ed66c7677 100644\n--- a/builtin/pack-objects.c\n+++ b/builtin/pack-objects.c\n@@ -28,6 +28,7 @@\n #include \"argv-array.h\"\n #include \"mru.h\"\n #include \"packfile.h\"\n+#include \"external-odb.h\"\n \n static const char *pack_usage[] = {\n \tN_(\"git pack-objects --stdout [...] [< | < ]\"),\n@@ -1026,6 +1027,9 @@ static int want_object_in_pack(const struct object_id *oid,\n \t\t\treturn want;\n \t}\n \n+\tif (external_odb_has_object(oid->hash))\n+\t\treturn 0;\n+\n \tfor (entry = packed_git_mru.head; entry; entry = entry->next) {\n \t\tstruct packed_git *p = entry->item;\n \t\toff_t offset;\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335734","messageId":"20180103163403.11303-31-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 30/40] external-odb: add 'get_direct' support","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:53Z","receivedAt":"2018-01-03T16:35:29Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This implements the 'get_direct' capability/instruction that makes\nit possible for external odb helper scripts to pass blobs to Git\nby directly writing them as loose objects files.\n\nIt is better to call this a \"direct\" mode rather than a \"fault-in\"\nmode as we could have the same kind of mechanism to \"put\" objects\ninto an external odb, where the odb helper would access blobs it\nwants to send to an external odb directly from files, but it\nwould be strange to call that a fault-in mode too.\n\nSigned-off-by: Christian Couder \n---\n external-odb.c | 3 ++-\n odb-helper.c | 28 +++++++++++++++++++++++-----\n 2 files changed, 25 insertions(+), 6 deletions(-)\n\ndiff --git a/external-odb.c b/external-odb.c\nindex 93971e9ce4..3ce3d111f3 100644\n--- a/external-odb.c\n+++ b/external-odb.c\n@@ -109,7 +109,8 @@ int external_odb_get_object(const unsigned char *sha1)\n \t\tint ret;\n \t\tint fd;\n \n-\t\tif (!odb_helper_has_object(o, sha1))\n+\t\tif (!(o->supported_capabilities & ODB_HELPER_CAP_GET_RAW_OBJ) &&\n+\t\t !(o->supported_capabilities & ODB_HELPER_CAP_GET_GIT_OBJ))\n \t\t\tcontinue;\n \n \t\tfd = create_object_tmpfile(&tmpfile, path);\ndiff --git a/odb-helper.c b/odb-helper.c\nindex fc30c2fa57..0fa7af0348 100644\n--- a/odb-helper.c\n+++ b/odb-helper.c\n@@ -429,24 +429,42 @@ static int odb_helper_get_git_object(struct odb_helper *o,\n int odb_helper_get_direct(struct odb_helper *o,\n \t\t\t const unsigned char *sha1)\n {\n-\tint res = 0;\n \tuint64_t start = getnanotime();\n \n-\tfetch_object(o->dealer, sha1);\n+\tif (o->type == ODB_HELPER_GIT_REMOTE) {\n+\t\tfetch_object(o->dealer, sha1);\n+\t} else {\n+\t\tstruct odb_helper_object *obj;\n+\t\tstruct odb_helper_cmd cmd;\n+\n+\t\tobj = odb_helper_lookup(o, sha1);\n+\t\tif (!obj)\n+\t\t\treturn -1;\n+\n+\t\tif (odb_helper_start(o, &cmd, 0, \"get_direct %s\", sha1_to_hex(sha1)) < 0)\n+\t\t\treturn -1;\n+\n+\t\tif (odb_helper_finish(o, &cmd))\n+\t\t\treturn -1;\n+\t}\n \n \ttrace_performance_since(start, \"odb_helper_get_direct\");\n \n-\treturn res;\n+\treturn 0;\n }\n \n int odb_helper_get_object(struct odb_helper *o,\n \t\t\t const unsigned char *sha1,\n \t\t\t int fd)\n {\n+\tif (o->supported_capabilities & ODB_HELPER_CAP_GET_GIT_OBJ)\n+\t\treturn odb_helper_get_git_object(o, sha1, fd);\n \tif (o->supported_capabilities & ODB_HELPER_CAP_GET_RAW_OBJ)\n \t\treturn odb_helper_get_raw_object(o, sha1, fd);\n-\telse\n-\t\treturn odb_helper_get_git_object(o, sha1, fd);\n+\tif (o->supported_capabilities & ODB_HELPER_CAP_GET_DIRECT)\n+\t\treturn 0;\n+\n+\tBUG(\"invalid get capability (capabilities: '%d')\", o->supported_capabilities);\n }\n \n int odb_helper_put_object(struct odb_helper *o,\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335735","messageId":"20180103163403.11303-32-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 31/40] odb-helper: add 'script_mode' to 'struct odb_helper'","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:54Z","receivedAt":"2018-01-03T16:35:31Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"to prepare for having a long running odb helper sub-process\nhandling the communication between Git and an external odb.\n\nWe introduce \"odb..subprocesscommand\" to make it\npossible to define such a sub-process, and we mark such odb\nhelpers with the new 'script_mode' field set to 0.\n\nHelpers defined using the existing \"odb..scriptcommand\"\nare marked with the 'script_mode' field set to 1.\n\nImplementation of the different capabilities/instructions in\nthe new (sub-)process mode is left for following commits.\n\nSigned-off-by: Christian Couder \n---\n external-odb.c | 4 ++++\n odb-helper.c | 19 ++++++++++++++-----\n odb-helper.h | 2 +-\n 3 files changed, 19 insertions(+), 6 deletions(-)\n\ndiff --git a/external-odb.c b/external-odb.c\nindex 3ce3d111f3..f38c2c2fe3 100644\n--- a/external-odb.c\n+++ b/external-odb.c\n@@ -43,6 +43,10 @@ static int external_odb_config(const char *var, const char *value, void *data)\n \t\to->type = ODB_HELPER_SCRIPT_CMD;\n \t\treturn git_config_string(&o->dealer, var, value);\n \t}\n+\tif (!strcmp(subkey, \"subprocesscommand\")) {\n+\t\to->type = ODB_HELPER_SUBPROCESS_CMD;\n+\t\treturn git_config_string(&o->dealer, var, value);\n+\t}\n \n \treturn 0;\n }\ndiff --git a/odb-helper.c b/odb-helper.c\nindex 0fa7af0348..91b4de1a05 100644\n--- a/odb-helper.c\n+++ b/odb-helper.c\n@@ -124,6 +124,9 @@ int odb_helper_init(struct odb_helper *o)\n \tFILE *fh;\n \tstruct strbuf line = STRBUF_INIT;\n \n+\tif (o->type != ODB_HELPER_SCRIPT_CMD)\n+\t\treturn 0;\n+\n \tif (odb_helper_start(o, &cmd, 0, \"init\") < 0)\n \t\treturn -1;\n \n@@ -174,16 +177,12 @@ static int odb_helper_object_cmp(const void *va, const void *vb)\n \treturn hashcmp(a->sha1, b->sha1);\n }\n \n-static void odb_helper_load_have(struct odb_helper *o)\n+static void have_object_script(struct odb_helper *o)\n {\n \tstruct odb_helper_cmd cmd;\n \tFILE *fh;\n \tstruct strbuf line = STRBUF_INIT;\n \n-\tif (o->have_valid)\n-\t\treturn;\n-\to->have_valid = 1;\n-\n \tif (odb_helper_start(o, &cmd, 0, \"have\") < 0)\n \t\treturn;\n \n@@ -195,6 +194,16 @@ static void odb_helper_load_have(struct odb_helper *o)\n \tstrbuf_release(&line);\n \tfclose(fh);\n \todb_helper_finish(o, &cmd);\n+}\n+\n+static void odb_helper_load_have(struct odb_helper *o)\n+{\n+\tif (o->have_valid)\n+\t\treturn;\n+\to->have_valid = 1;\n+\n+\tif (o->type == ODB_HELPER_SCRIPT_CMD)\n+\t\thave_object_script(o);\n \n \tqsort(o->have, o->have_nr, sizeof(*o->have), odb_helper_object_cmp);\n }\ndiff --git a/odb-helper.h b/odb-helper.h\nindex 4a9cc7f07b..9a33adbf0b 100644\n--- a/odb-helper.h\n+++ b/odb-helper.h\n@@ -7,7 +7,7 @@ enum odb_helper_type {\n \tODB_HELPER_NONE = 0,\n \tODB_HELPER_GIT_REMOTE,\n \tODB_HELPER_SCRIPT_CMD,\n-\tODB_HELPER_PROCESS_CMD,\n+\tODB_HELPER_SUBPROCESS_CMD,\n \tOBJ_HELPER_MAX\n };\n \n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335736","messageId":"20180103163403.11303-33-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 32/40] odb-helper: add init_object_process()","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:55Z","receivedAt":"2018-01-03T16:35:34Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This adds the infrastructure to launch and use long running\nsub-processes as external odb helpers.\n\nFor now only the 'init' and 'get_direct' capabilities are\nsupported with sub-processes.\n\nSigned-off-by: Christian Couder \n---\n external-odb.c | 46 +++---\n odb-helper.c | 474 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---\n sha1_file.c | 12 +-\n 3 files changed, 491 insertions(+), 41 deletions(-)\n\ndiff --git a/external-odb.c b/external-odb.c\nindex f38c2c2fe3..50c1cec50b 100644\n--- a/external-odb.c\n+++ b/external-odb.c\n@@ -81,29 +81,11 @@ const char *external_odb_root(void)\n \treturn root;\n }\n \n-int external_odb_has_object(const unsigned char *sha1)\n-{\n-\tstruct odb_helper *o;\n-\n-\texternal_odb_init();\n-\n-\tfor (o = helpers; o; o = o->next) {\n-\t\tif (!(o->supported_capabilities & ODB_HELPER_CAP_HAVE))\n-\t\t\treturn 1;\n-\t\tif (odb_helper_has_object(o, sha1))\n-\t\t\treturn 1;\n-\t}\n-\treturn 0;\n-}\n-\n-int external_odb_get_object(const unsigned char *sha1)\n+static int external_odb_do_get_object(const unsigned char *sha1)\n {\n \tstruct odb_helper *o;\n \tconst char *path;\n \n-\tif (!external_odb_has_object(sha1))\n-\t\treturn -1;\n-\n \tpath = sha1_file_name_alt(external_odb_root(), sha1);\n \tsafe_create_leading_directories_const(path);\n \tprepare_external_alt_odb();\n@@ -157,6 +139,32 @@ int external_odb_get_direct(const unsigned char *sha1)\n \treturn -1;\n }\n \n+int external_odb_has_object(const unsigned char *sha1)\n+{\n+\tstruct odb_helper *o;\n+\n+\texternal_odb_init();\n+\n+\tfor (o = helpers; o; o = o->next) {\n+\t\tif (!(o->supported_capabilities & ODB_HELPER_CAP_HAVE)) {\n+\t\t\tif (o->supported_capabilities & ODB_HELPER_CAP_GET_DIRECT)\n+\t\t\t\treturn 1;\n+\t\t\treturn !external_odb_do_get_object(sha1);\n+\t\t}\n+\t\tif (odb_helper_has_object(o, sha1))\n+\t\t\treturn 1;\n+\t}\n+\treturn 0;\n+}\n+\n+int external_odb_get_object(const unsigned char *sha1)\n+{\n+\tif (!external_odb_has_object(sha1))\n+\t\treturn -1;\n+\n+\treturn external_odb_do_get_object(sha1);\n+}\n+\n int external_odb_put_object(const void *buf, size_t len,\n \t\t\t const char *type, unsigned char *sha1)\n {\ndiff --git a/odb-helper.c b/odb-helper.c\nindex 91b4de1a05..a67dfddca0 100644\n--- a/odb-helper.c\n+++ b/odb-helper.c\n@@ -5,6 +5,22 @@\n #include \"run-command.h\"\n #include \"sha1-lookup.h\"\n #include \"fetch-object.h\"\n+#include \"sub-process.h\"\n+#include \"pkt-line.h\"\n+#include \"sigchain.h\"\n+\n+struct object_process {\n+\tstruct subprocess_entry subprocess;\n+\tunsigned int supported_capabilities;\n+};\n+\n+static struct hashmap subprocess_map;\n+\n+static int check_object_process_status(int fd, struct strbuf *status)\n+{\n+\tsubprocess_read_status(fd, status);\n+\treturn strcmp(status->buf, \"success\");\n+}\n \n static void parse_capabilities(char *cap_buf,\n \t\t\t unsigned int *supported_capabilities,\n@@ -40,6 +56,384 @@ static void parse_capabilities(char *cap_buf,\n \tstring_list_clear(&cap_list, 0);\n }\n \n+static int start_object_process_fn(struct subprocess_entry *subprocess)\n+{\n+\tstatic int versions[] = {1, 0};\n+\tstatic struct subprocess_capability capabilities[] = {\n+\t\t{ \"get_git_obj\", ODB_HELPER_CAP_GET_GIT_OBJ },\n+\t\t{ \"get_raw_obj\", ODB_HELPER_CAP_GET_RAW_OBJ },\n+\t\t{ \"get_direct\", ODB_HELPER_CAP_GET_DIRECT },\n+\t\t{ \"put_git_obj\", ODB_HELPER_CAP_PUT_GIT_OBJ },\n+\t\t{ \"put_raw_obj\", ODB_HELPER_CAP_PUT_RAW_OBJ },\n+\t\t{ \"put_direct\", ODB_HELPER_CAP_PUT_DIRECT },\n+\t\t{ \"have\", ODB_HELPER_CAP_HAVE },\n+\t\t{ NULL, 0 }\n+\t};\n+\tstruct object_process *entry = (struct object_process *)subprocess;\n+\treturn subprocess_handshake(subprocess, \"git-read-object\", versions, NULL,\n+\t\t\t\t capabilities,\n+\t\t\t\t &entry->supported_capabilities);\n+}\n+\n+static struct object_process *launch_object_process(struct odb_helper *o,\n+\t\t\t\t\t\t unsigned int capability)\n+{\n+\tstruct object_process *entry = NULL;\n+\n+\tif (!subprocess_map.tablesize)\n+\t\thashmap_init(&subprocess_map, (hashmap_cmp_fn) cmd2process_cmp, NULL, 0);\n+\telse\n+\t\tentry = (struct object_process *)subprocess_find_entry(&subprocess_map, o->dealer);\n+\n+\tfflush(NULL);\n+\n+\tif (!entry) {\n+\t\tentry = xmalloc(sizeof(*entry));\n+\t\tentry->supported_capabilities = 0;\n+\n+\t\tif (subprocess_start(&subprocess_map, &entry->subprocess, o->dealer, start_object_process_fn)) {\n+\t\t\terror(\"Could not launch process for cmd '%s'\", o->dealer);\n+\t\t\tfree(entry);\n+\t\t\treturn NULL;\n+\t\t}\n+\t}\n+\n+\to->supported_capabilities = entry->supported_capabilities;\n+\n+\tif (capability && !(capability & entry->supported_capabilities)) {\n+\t\terror(\"The cmd '%s' does not support capability '%d'\", o->dealer, capability);\n+\t\treturn NULL;\n+\t}\n+\n+\tsigchain_push(SIGPIPE, SIG_IGN);\n+\n+\treturn entry;\n+}\n+\n+static int check_object_process_error(int err,\n+\t\t\t\t const char *status,\n+\t\t\t\t struct object_process *entry,\n+\t\t\t\t const char *cmd,\n+\t\t\t\t unsigned int capability)\n+{\n+\tsigchain_pop(SIGPIPE);\n+\n+\tif (!err)\n+\t\treturn 0;\n+\n+\tif (!strcmp(status, \"error\")) {\n+\t\t/* The process signaled a problem with the file. */\n+\t} else if (!strcmp(status, \"notfound\")) {\n+\t\t/* Object was not found */\n+\t\terr = -1;\n+\t} else if (!strcmp(status, \"abort\")) {\n+\t\t/*\n+\t\t * The process signaled a permanent problem. Don't try to read\n+\t\t * objects with the same command for the lifetime of the current\n+\t\t * Git process.\n+\t\t */\n+\t\tif (capability)\n+\t\t\tentry->supported_capabilities &= ~capability;\n+\t} else {\n+\t\t/*\n+\t\t * Something went wrong with the read-object process.\n+\t\t * Force shutdown and restart if needed.\n+\t\t */\n+\t\terror(\"external object process '%s' failed\", cmd);\n+\t\tsubprocess_stop(&subprocess_map, &entry->subprocess);\n+\t\tfree(entry);\n+\t}\n+\n+\treturn err;\n+}\n+\n+static int send_init_packets(struct object_process *entry,\n+\t\t\t struct strbuf *status)\n+{\n+\tstruct child_process *process = &entry->subprocess.process;\n+\n+\treturn packet_write_fmt_gently(process->in, \"command=init\\n\") ||\n+\t\tpacket_flush_gently(process->in) ||\n+\t\tcheck_object_process_status(process->out, status);\n+}\n+\n+static int init_object_process(struct odb_helper *o)\n+{\n+\tint err;\n+\tstruct strbuf status = STRBUF_INIT;\n+\tstruct object_process *entry = launch_object_process(o, 0);\n+\tif (!entry)\n+\t\treturn -1;\n+\n+\terr = send_init_packets(entry, &status);\n+\n+\treturn check_object_process_error(err, status.buf, entry,\n+\t\t\t\t\t o->dealer, 0);\n+}\n+\n+static ssize_t read_packetized_raw_object_to_fd(struct odb_helper *o,\n+\t\t\t\t\t\tconst unsigned char *sha1,\n+\t\t\t\t\t\tint fd_in, int fd_out)\n+{\n+\tssize_t total_read = 0;\n+\tunsigned long total_got = 0;\n+\tint packet_len;\n+\n+\tchar hdr[32];\n+\tint hdrlen;\n+\n+\tint ret = Z_STREAM_END;\n+\tunsigned char compressed[4096];\n+\tgit_zstream stream;\n+\tgit_SHA_CTX hash;\n+\tunsigned char real_sha1[20];\n+\n+\toff_t size;\n+\tenum object_type type;\n+\tconst char *s;\n+\tint pkt_size;\n+\tchar *size_buf;\n+\n+\tsize_buf = packet_read_line(fd_in, &pkt_size);\n+\tif (!skip_prefix(size_buf, \"size=\", &s))\n+\t\treturn error(\"odb helper '%s' did not send size of plain object\", o->name);\n+\tsize = strtoumax(s, NULL, 10);\n+\tif (!skip_prefix(packet_read_line(fd_in, NULL), \"kind=\", &s))\n+\t\treturn error(\"odb helper '%s' did not send kind of plain object\", o->name);\n+\t/* Check if the object is not available */\n+\tif (!strcmp(s, \"none\"))\n+\t\treturn -1;\n+\ttype = type_from_string_gently(s, strlen(s), 1);\n+\tif (type < 0)\n+\t\treturn error(\"odb helper '%s' sent bad type '%s'\", o->name, s);\n+\n+\t/* Set it up */\n+\tgit_deflate_init(&stream, zlib_compression_level);\n+\tstream.next_out = compressed;\n+\tstream.avail_out = sizeof(compressed);\n+\tgit_SHA1_Init(&hash);\n+\n+\t/* First header.. */\n+\thdrlen = xsnprintf(hdr, sizeof(hdr), \"%s %lu\", typename(type), size) + 1;\n+\tstream.next_in = (unsigned char *)hdr;\n+\tstream.avail_in = hdrlen;\n+\twhile (git_deflate(&stream, 0) == Z_OK)\n+\t\t; /* nothing */\n+\tgit_SHA1_Update(&hash, hdr, hdrlen);\n+\n+\tfor (;;) {\n+\t\t/* packet_read() writes a '\\0' extra byte at the end */\n+\t\tchar buf[LARGE_PACKET_DATA_MAX + 1];\n+\n+\t\tpacket_len = packet_read(fd_in, NULL, NULL,\n+\t\t\tbuf, LARGE_PACKET_DATA_MAX + 1,\n+\t\t\tPACKET_READ_GENTLE_ON_EOF);\n+\n+\t\tif (packet_len <= 0)\n+\t\t\tbreak;\n+\n+\t\ttotal_got += packet_len;\n+\n+\t\t/* Then the data itself.. */\n+\t\tstream.next_in = (void *)buf;\n+\t\tstream.avail_in = packet_len;\n+\t\tdo {\n+\t\t\tunsigned char *in0 = stream.next_in;\n+\t\t\tret = git_deflate(&stream, Z_FINISH);\n+\t\t\tgit_SHA1_Update(&hash, in0, stream.next_in - in0);\n+\t\t\twrite_or_die(fd_out, compressed, stream.next_out - compressed);\n+\t\t\tstream.next_out = compressed;\n+\t\t\tstream.avail_out = sizeof(compressed);\n+\t\t} while (ret == Z_OK);\n+\n+\t\ttotal_read += packet_len;\n+\t}\n+\n+\tif (packet_len < 0) {\n+\t\terror(\"unable to read from odb helper '%s': %s\",\n+\t\t o->name, strerror(errno));\n+\t\tgit_deflate_end(&stream);\n+\t\treturn packet_len;\n+\t}\n+\n+\tif (ret != Z_STREAM_END) {\n+\t\twarning(\"bad zlib data from odb helper '%s' for %s\",\n+\t\t\to->name, sha1_to_hex(sha1));\n+\t\treturn -1;\n+\t}\n+\n+\tret = git_deflate_end_gently(&stream);\n+\tif (ret != Z_OK) {\n+\t\twarning(\"deflateEnd on object %s from odb helper '%s' failed (%d)\",\n+\t\t\tsha1_to_hex(sha1), o->name, ret);\n+\t\treturn -1;\n+\t}\n+\tgit_SHA1_Final(real_sha1, &hash);\n+\tif (hashcmp(sha1, real_sha1)) {\n+\t\twarning(\"sha1 mismatch from odb helper '%s' for %s (got %s)\",\n+\t\t\to->name, sha1_to_hex(sha1), sha1_to_hex(real_sha1));\n+\t\treturn -1;\n+\t}\n+\tif (total_got != size) {\n+\t\twarning(\"size mismatch from odb helper '%s' for %s (%lu != %lu)\",\n+\t\t\to->name, sha1_to_hex(sha1), total_got, size);\n+\t\treturn -1;\n+\t}\n+\n+\treturn total_read;\n+}\n+\n+static ssize_t read_packetized_git_object_to_fd(struct odb_helper *o,\n+\t\t\t\t\t\tconst unsigned char *sha1,\n+\t\t\t\t\t\tint fd_in, int fd_out)\n+{\n+\tssize_t total_read = 0;\n+\tunsigned long total_got = 0;\n+\tint packet_len;\n+\tgit_zstream stream;\n+\tint zret = Z_STREAM_END;\n+\tgit_SHA_CTX hash;\n+\tunsigned char real_sha1[20];\n+\tstruct strbuf header = STRBUF_INIT;\n+\tunsigned long hdr_size;\n+\n+\tmemset(&stream, 0, sizeof(stream));\n+\tgit_inflate_init(&stream);\n+\tgit_SHA1_Init(&hash);\n+\n+\tfor (;;) {\n+\t\t/* packet_read() writes a '\\0' extra byte at the end */\n+\t\tchar buf[LARGE_PACKET_DATA_MAX + 1];\n+\n+\t\tpacket_len = packet_read(fd_in, NULL, NULL,\n+\t\t\tbuf, LARGE_PACKET_DATA_MAX + 1,\n+\t\t\tPACKET_READ_GENTLE_ON_EOF);\n+\n+\t\tif (packet_len <= 0)\n+\t\t\tbreak;\n+\n+\t\twrite_or_die(fd_out, buf, packet_len);\n+\n+\t\tstream.next_in = (unsigned char *)buf;\n+\t\tstream.avail_in = packet_len;\n+\t\tdo {\n+\t\t\tunsigned char inflated[4096];\n+\t\t\tunsigned long got;\n+\n+\t\t\tstream.next_out = inflated;\n+\t\t\tstream.avail_out = sizeof(inflated);\n+\t\t\tzret = git_inflate(&stream, Z_SYNC_FLUSH);\n+\t\t\tgot = sizeof(inflated) - stream.avail_out;\n+\n+\t\t\tgit_SHA1_Update(&hash, inflated, got);\n+\t\t\t/* skip header when counting size */\n+\t\t\tif (!total_got) {\n+\t\t\t\tconst unsigned char *p = memchr(inflated, '\\0', got);\n+\t\t\t\tif (p) {\n+\t\t\t\t\tunsigned long hdr_last = p - inflated + 1;\n+\t\t\t\t\tstrbuf_add(&header, inflated, hdr_last);\n+\t\t\t\t\tgot -= hdr_last;\n+\t\t\t\t} else {\n+\t\t\t\t\tstrbuf_add(&header, inflated, got);\n+\t\t\t\t\tgot = 0;\n+\t\t\t\t}\n+\t\t\t}\n+\t\t\ttotal_got += got;\n+\t\t} while (stream.avail_in && zret == Z_OK);\n+\n+\t\ttotal_read += packet_len;\n+\t}\n+\n+\tgit_inflate_end(&stream);\n+\n+\tif (packet_len < 0)\n+\t\treturn packet_len;\n+\n+\tgit_SHA1_Final(real_sha1, &hash);\n+\n+\tif (zret != Z_STREAM_END) {\n+\t\twarning(\"bad zlib data from odb helper '%s' for %s\",\n+\t\t\to->name, sha1_to_hex(sha1));\n+\t\treturn -1;\n+\t}\n+\tif (hashcmp(real_sha1, sha1)) {\n+\t\twarning(\"sha1 mismatch from odb helper '%s' for %s (got %s)\",\n+\t\t\to->name, sha1_to_hex(sha1), sha1_to_hex(real_sha1));\n+\t\treturn -1;\n+\t}\n+\tif (parse_sha1_header(header.buf, &hdr_size) < 0) {\n+\t\twarning(\"could not parse header from odb helper '%s' for %s\",\n+\t\t\to->name, sha1_to_hex(sha1));\n+\t\treturn -1;\n+\t}\n+\tif (total_got != hdr_size) {\n+\t\twarning(\"size mismatch from odb helper '%s' for %s (%lu != %lu)\",\n+\t\t\to->name, sha1_to_hex(sha1), total_got, hdr_size);\n+\t\treturn -1;\n+\t}\n+\n+\treturn total_read;\n+}\n+\n+static int send_get_packets(struct odb_helper *o,\n+\t\t\t struct object_process *entry,\n+\t\t\t const unsigned char *sha1,\n+\t\t\t int fd,\n+\t\t\t unsigned int *cur_cap,\n+\t\t\t struct strbuf *status)\n+{\n+\tconst char *instruction;\n+\tint err;\n+\tstruct child_process *process = &entry->subprocess.process;\n+\n+\tif (entry->supported_capabilities & ODB_HELPER_CAP_GET_GIT_OBJ) {\n+\t\t*cur_cap = ODB_HELPER_CAP_GET_GIT_OBJ;\n+\t\tinstruction = \"get_git_obj\";\n+\t} else if (entry->supported_capabilities & ODB_HELPER_CAP_GET_RAW_OBJ) {\n+\t\t*cur_cap = ODB_HELPER_CAP_GET_RAW_OBJ;\n+\t\tinstruction = \"get_raw_obj\";\n+\t} else if (entry->supported_capabilities & ODB_HELPER_CAP_GET_DIRECT) {\n+\t\t*cur_cap = ODB_HELPER_CAP_GET_DIRECT;\n+\t\tinstruction = \"get_direct\";\n+\t} else {\n+\t\tBUG(\"No known ODB_HELPER_CAP_GET_XXX capability!\");\n+\t}\n+\n+\terr = packet_write_fmt_gently(process->in, \"command=%s\\n\", instruction);\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = packet_write_fmt_gently(process->in, \"sha1=%s\\n\", sha1_to_hex(sha1));\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = packet_flush_gently(process->in);\n+\tif (err)\n+\t\treturn err;\n+\n+\tif (entry->supported_capabilities & ODB_HELPER_CAP_GET_RAW_OBJ)\n+\t\terr = read_packetized_raw_object_to_fd(o, sha1, process->out, fd) < 0;\n+\telse if (entry->supported_capabilities & ODB_HELPER_CAP_GET_GIT_OBJ)\n+\t\terr = read_packetized_git_object_to_fd(o, sha1, process->out, fd) < 0;\n+\n+\treturn check_object_process_status(process->out, status);\n+}\n+\n+static int get_object_process(struct odb_helper *o, const unsigned char *sha1, int fd)\n+{\n+\tint err;\n+\tstruct strbuf status = STRBUF_INIT;\n+\tunsigned int cur_cap = 0;\n+\tstruct object_process *entry = launch_object_process(o, 0);\n+\tif (!entry)\n+\t\treturn -1;\n+\n+\terr = send_get_packets(o, entry, sha1, fd, &cur_cap, &status);\n+\n+\treturn check_object_process_error(err, status.buf, entry,\n+\t\t\t\t\t o->dealer, cur_cap);\n+}\n+\n struct odb_helper *odb_helper_new(const char *name, int namelen)\n {\n \tstruct odb_helper *o;\n@@ -118,15 +512,12 @@ static int odb_helper_finish(struct odb_helper *o,\n \treturn 0;\n }\n \n-int odb_helper_init(struct odb_helper *o)\n+static int init_object_script(struct odb_helper *o)\n {\n \tstruct odb_helper_cmd cmd;\n \tFILE *fh;\n \tstruct strbuf line = STRBUF_INIT;\n \n-\tif (o->type != ODB_HELPER_SCRIPT_CMD)\n-\t\treturn 0;\n-\n \tif (odb_helper_start(o, &cmd, 0, \"init\") < 0)\n \t\treturn -1;\n \n@@ -141,6 +532,21 @@ int odb_helper_init(struct odb_helper *o)\n \treturn 0;\n }\n \n+int odb_helper_init(struct odb_helper *o)\n+{\n+\tint res = 0;\n+\tuint64_t start = getnanotime();\n+\n+\tif (o->type == ODB_HELPER_SCRIPT_CMD)\n+\t\tres = init_object_script(o);\n+\telse if (o->type == ODB_HELPER_SUBPROCESS_CMD)\n+\t\tres = init_object_process(o);\n+\n+\ttrace_performance_since(start, \"odb_helper_init\");\n+\n+\treturn res;\n+}\n+\n static int parse_object_line(struct odb_helper_object *o, const char *line)\n {\n \tchar *end;\n@@ -435,36 +841,45 @@ static int odb_helper_get_git_object(struct odb_helper *o,\n \treturn 0;\n }\n \n+static int get_direct_script(struct odb_helper *o, const unsigned char *sha1)\n+{\n+\tstruct odb_helper_cmd cmd;\n+\n+\tif (odb_helper_start(o, &cmd, 0, \"get_direct %s\", sha1_to_hex(sha1)) < 0)\n+\t\treturn -1;\n+\tif (odb_helper_finish(o, &cmd))\n+\t\treturn -1;\n+\treturn 0;\n+}\n+\n int odb_helper_get_direct(struct odb_helper *o,\n \t\t\t const unsigned char *sha1)\n {\n-\tuint64_t start = getnanotime();\n-\n-\tif (o->type == ODB_HELPER_GIT_REMOTE) {\n-\t\tfetch_object(o->dealer, sha1);\n-\t} else {\n-\t\tstruct odb_helper_object *obj;\n-\t\tstruct odb_helper_cmd cmd;\n+\tint res = 0;\n+\tuint64_t start;\n \n-\t\tobj = odb_helper_lookup(o, sha1);\n+\tif (o->supported_capabilities & ODB_HELPER_CAP_HAVE &&\n+\t o->type != ODB_HELPER_GIT_REMOTE) {\n+\t\tstruct odb_helper_object *obj = odb_helper_lookup(o, sha1);\n \t\tif (!obj)\n \t\t\treturn -1;\n+\t}\n \n-\t\tif (odb_helper_start(o, &cmd, 0, \"get_direct %s\", sha1_to_hex(sha1)) < 0)\n-\t\t\treturn -1;\n+\tstart = getnanotime();\n \n-\t\tif (odb_helper_finish(o, &cmd))\n-\t\t\treturn -1;\n-\t}\n+\tif (o->type == ODB_HELPER_GIT_REMOTE)\n+\t\tfetch_object(o->dealer, sha1);\n+\telse if (o->type == ODB_HELPER_SCRIPT_CMD)\n+\t\tres = get_direct_script(o, sha1);\n+\telse\n+\t\tres = get_object_process(o, sha1, -1);\n \n \ttrace_performance_since(start, \"odb_helper_get_direct\");\n \n-\treturn 0;\n+\treturn res;\n }\n \n-int odb_helper_get_object(struct odb_helper *o,\n-\t\t\t const unsigned char *sha1,\n-\t\t\t int fd)\n+static int get_object_script(struct odb_helper *o, const unsigned char *sha1, int fd)\n {\n \tif (o->supported_capabilities & ODB_HELPER_CAP_GET_GIT_OBJ)\n \t\treturn odb_helper_get_git_object(o, sha1, fd);\n@@ -476,6 +891,23 @@ int odb_helper_get_object(struct odb_helper *o,\n \tBUG(\"invalid get capability (capabilities: '%d')\", o->supported_capabilities);\n }\n \n+int odb_helper_get_object(struct odb_helper *o,\n+\t\t\t const unsigned char *sha1,\n+\t\t\t int fd)\n+{\n+\tint res = 0;\n+\tuint64_t start = getnanotime();\n+\n+\tif (o->type == ODB_HELPER_SCRIPT_CMD)\n+\t\tres = get_object_script(o, sha1, fd);\n+\telse if (o->type == ODB_HELPER_SUBPROCESS_CMD)\n+\t\tres = get_object_process(o, sha1, fd);\n+\n+\ttrace_performance_since(start, \"odb_helper_get_object\");\n+\n+\treturn res;\n+}\n+\n int odb_helper_put_object(struct odb_helper *o,\n \t\t\t const void *buf, size_t len,\n \t\t\t const char *type, unsigned char *sha1)\ndiff --git a/sha1_file.c b/sha1_file.c\nindex 8c873554b2..300029459f 100644\n--- a/sha1_file.c\n+++ b/sha1_file.c\n@@ -735,7 +735,17 @@ int check_and_freshen_file(const char *fn, int freshen)\n \n static int check_and_freshen_local(const unsigned char *sha1, int freshen)\n {\n-\treturn check_and_freshen_file(sha1_file_name(sha1), freshen);\n+\tint ret;\n+\tint tried_hook = 0;\n+\n+retry:\n+\tret = check_and_freshen_file(sha1_file_name(sha1), freshen);\n+\tif (!ret && !tried_hook) {\n+\t\ttried_hook = 1;\n+\t\tif (!external_odb_get_direct(sha1))\n+\t\t\tgoto retry;\n+\t}\n+\treturn ret;\n }\n \n static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335737","messageId":"20180103163403.11303-35-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 34/40] Add t0460 to test passing git objects","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:57Z","receivedAt":"2018-01-03T16:35:37Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"Signed-off-by: Christian Couder \n---\n t/t0460-read-object-git.sh | 28 +++++++++++++++++\n t/t0460/read-object-git | 78 ++++++++++++++++++++++++++++++++++++++++++++++\n 2 files changed, 106 insertions(+)\n create mode 100755 t/t0460-read-object-git.sh\n create mode 100755 t/t0460/read-object-git\n\ndiff --git a/t/t0460-read-object-git.sh b/t/t0460-read-object-git.sh\nnew file mode 100755\nindex 0000000000..2873b445f3\n--- /dev/null\n+++ b/t/t0460-read-object-git.sh\n@@ -0,0 +1,28 @@\n+#!/bin/sh\n+\n+test_description='tests for long running read-object process passing git objects'\n+\n+. ./test-lib.sh\n+\n+PATH=\"$PATH:$TEST_DIRECTORY/t0460\"\n+\n+test_expect_success 'setup host repo with a root commit' '\n+\ttest_commit zero &&\n+\thash1=$(git ls-tree HEAD | grep zero.t | cut -f1 | cut -d\\ -f3)\n+'\n+\n+HELPER=\"read-object-git\"\n+\n+test_expect_success 'blobs can be retrieved from the host repo' '\n+\tgit init guest-repo &&\n+\t(cd guest-repo &&\n+\t git config odb.magic.subprocessCommand \"$HELPER\" &&\n+\t git cat-file blob \"$hash1\" >/dev/null)\n+'\n+\n+test_expect_success 'invalid blobs generate errors' '\n+\tcd guest-repo &&\n+\ttest_must_fail git cat-file blob \"invalid\"\n+'\n+\n+test_done\ndiff --git a/t/t0460/read-object-git b/t/t0460/read-object-git\nnew file mode 100755\nindex 0000000000..4b3ca0948b\n--- /dev/null\n+++ b/t/t0460/read-object-git\n@@ -0,0 +1,78 @@\n+#!/usr/bin/perl\n+#\n+# Example implementation for the Git read-object protocol version 1\n+# See Documentation/technical/read-object-protocol.txt\n+#\n+# Allows you to test the ability for blobs to be pulled from a host git repo\n+# \"on demand.\" Called when git needs a blob it couldn't find locally due to\n+# a lazy clone that only cloned the commits and trees.\n+#\n+# A lazy clone can be simulated via the following commands from the host repo\n+# you wish to create a lazy clone of:\n+#\n+# cd /host_repo\n+# git rev-parse HEAD\n+# git init /guest_repo\n+# git cat-file --batch-check --batch-all-objects | grep -v 'blob' |\n+#\tcut -d' ' -f1 | git pack-objects /e/guest_repo/.git/objects/pack/noblobs\n+# cd /guest_repo\n+# git config core.virtualizeobjects true\n+# git reset --hard \n+#\n+# Please note, this sample is a minimal skeleton. No proper error handling \n+# was implemented.\n+#\n+\n+use 5.008;\n+use lib (split(/:/, $ENV{GITPERLLIB}));\n+use strict;\n+use warnings;\n+use Git::Packet;\n+\n+#\n+# Point $DIR to the folder where your host git repo is located so we can pull\n+# missing objects from it\n+#\n+my $DIR = \"../.git/\";\n+\n+packet_initialize(\"git-read-object\", 1);\n+\n+my %remote_caps = packet_read_and_check_capabilities(\"get_git_obj\");\n+packet_check_and_write_capabilities(\\%remote_caps, \"get_git_obj\");\n+\n+while (1) {\n+\tmy ($res, $command) = packet_txt_read();\n+\n+\tif ( $res == -1 ) {\n+\t\texit 0;\n+\t}\n+\n+\t$command =~ s/^command=//;\n+\n+\tif ( $command eq \"init\" ) {\n+\t\tpacket_bin_read();\n+\n+\t\tpacket_txt_write(\"status=success\");\n+\t\tpacket_flush();\n+\t} elsif ( $command eq \"get_git_obj\" ) {\n+\t\tmy ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/;\n+\t\tpacket_bin_read();\n+\n+\t\tmy $path = $sha1;\n+\t\t$path =~ s{..}{$&/};\n+\t\t$path = $DIR . \"/objects/\" . $path;\n+\n+\t\tmy $contents = do {\n+\t\t local $/;\n+\t\t open my $fh, $path or die \"Can't open '$path': $!\";\n+\t\t <$fh>\n+\t\t};\n+\n+\t\tpacket_bin_write($contents);\n+\t\tpacket_flush();\n+\t\tpacket_txt_write(\"status=success\");\n+\t\tpacket_flush();\n+\t} else {\n+\t\tdie \"bad command '$command'\";\n+\t}\n+}\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335738","messageId":"20180103163403.11303-34-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 33/40] Add t0450 to test 'get_direct' mechanism","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:56Z","receivedAt":"2018-01-03T16:35:39Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"From: Ben Peart \n\nSigned-off-by: Ben Peart \nSigned-off-by: Christian Couder \n---\n t/t0450-read-object.sh | 28 +++++++++++++++++++++\n t/t0450/read-object | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++\n 2 files changed, 96 insertions(+)\n create mode 100755 t/t0450-read-object.sh\n create mode 100755 t/t0450/read-object\n\ndiff --git a/t/t0450-read-object.sh b/t/t0450-read-object.sh\nnew file mode 100755\nindex 0000000000..6b97305452\n--- /dev/null\n+++ b/t/t0450-read-object.sh\n@@ -0,0 +1,28 @@\n+#!/bin/sh\n+\n+test_description='tests for long running read-object process'\n+\n+. ./test-lib.sh\n+\n+PATH=\"$PATH:$TEST_DIRECTORY/t0450\"\n+\n+test_expect_success 'setup host repo with a root commit' '\n+\ttest_commit zero &&\n+\thash1=$(git ls-tree HEAD | grep zero.t | cut -f1 | cut -d\\ -f3)\n+'\n+\n+HELPER=\"read-object\"\n+\n+test_expect_success 'blobs can be retrieved from the host repo' '\n+\tgit init guest-repo &&\n+\t(cd guest-repo &&\n+\t git config odb.magic.subprocessCommand \"$HELPER\" &&\n+\t git cat-file blob \"$hash1\" >/dev/null)\n+'\n+\n+test_expect_success 'invalid blobs generate errors' '\n+\tcd guest-repo &&\n+\ttest_must_fail git cat-file blob \"invalid\"\n+'\n+\n+test_done\ndiff --git a/t/t0450/read-object b/t/t0450/read-object\nnew file mode 100755\nindex 0000000000..004e9368c9\n--- /dev/null\n+++ b/t/t0450/read-object\n@@ -0,0 +1,68 @@\n+#!/usr/bin/perl\n+#\n+# Example implementation for the Git read-object protocol version 1\n+# See Documentation/technical/read-object-protocol.txt\n+#\n+# Allows you to test the ability for blobs to be pulled from a host git repo\n+# \"on demand.\" Called when git needs a blob it couldn't find locally due to\n+# a lazy clone that only cloned the commits and trees.\n+#\n+# A lazy clone can be simulated via the following commands from the host repo\n+# you wish to create a lazy clone of:\n+#\n+# cd /host_repo\n+# git rev-parse HEAD\n+# git init /guest_repo\n+# git cat-file --batch-check --batch-all-objects | grep -v 'blob' |\n+#\tcut -d' ' -f1 | git pack-objects /e/guest_repo/.git/objects/pack/noblobs\n+# cd /guest_repo\n+# git config core.virtualizeobjects true\n+# git reset --hard \n+#\n+# Please note, this sample is a minimal skeleton. No proper error handling \n+# was implemented.\n+#\n+\n+use 5.008;\n+use lib (split(/:/, $ENV{GITPERLLIB}));\n+use strict;\n+use warnings;\n+use Git::Packet;\n+\n+#\n+# Point $DIR to the folder where your host git repo is located so we can pull\n+# missing objects from it\n+#\n+my $DIR = \"../.git/\";\n+\n+packet_initialize(\"git-read-object\", 1);\n+\n+my %remote_caps = packet_read_and_check_capabilities(\"get_direct\");\n+packet_check_and_write_capabilities(\\%remote_caps, \"get_direct\");\n+\n+while (1) {\n+\tmy ($res, $command) = packet_txt_read();\n+\n+\tif ( $res == -1 ) {\n+\t\texit 0;\n+\t}\n+\n+\t$command =~ s/^command=//;\n+\n+\tif ( $command eq \"init\" ) {\n+\t\tpacket_bin_read();\n+\n+\t\tpacket_txt_write(\"status=success\");\n+\t\tpacket_flush();\n+\t} elsif ( $command eq \"get_direct\" ) {\n+\t\tmy ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/;\n+\t\tpacket_bin_read();\n+\n+\t\tsystem ('git --git-dir=\"' . $DIR . '\" cat-file blob ' . $sha1 . ' | GIT_NO_EXTERNAL_ODB=1 git hash-object -w --stdin >/dev/null 2>&1');\n+\n+\t\tpacket_txt_write(($?) ? \"status=error\" : \"status=success\");\n+\t\tpacket_flush();\n+\t} else {\n+\t\tdie \"bad command '$command'\";\n+\t}\n+}\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335739","messageId":"20180103163403.11303-41-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 40/40] Add Documentation/technical/external-odb.txt","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:34:03Z","receivedAt":"2018-01-03T16:35:41Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This describes the external odb mechanism's purpose and\nhow it works.\n\nHelped-by: Ben Peart \nSigned-off-by: Christian Couder \n---\n Documentation/technical/external-odb.txt | 342 +++++++++++++++++++++++++++++++\n 1 file changed, 342 insertions(+)\n create mode 100644 Documentation/technical/external-odb.txt\n\ndiff --git a/Documentation/technical/external-odb.txt b/Documentation/technical/external-odb.txt\nnew file mode 100644\nindex 0000000000..58ec8a8145\n--- /dev/null\n+++ b/Documentation/technical/external-odb.txt\n@@ -0,0 +1,342 @@\n+External ODBs\n+^^^^^^^^^^^^^\n+\n+The External ODB mechanism makes it possible for Git objects, only\n+blobs for now though, to be stored in an \"external object database\"\n+(External ODB).\n+\n+An External ODB can be any object store as long as there is an helper\n+program called an \"odb helper\" that can communicate with Git to\n+transfer objects to/from the external odb and to retrieve information\n+about available objects in the external odb.\n+\n+Purpose\n+=======\n+\n+The purpose of this mechanism is to make possible to handle Git\n+objects, especially blobs, in much more flexible ways.\n+\n+Currently Git can store its objects only in the form of loose objects\n+in separate files or packed objects in a pack file. These existing\n+object stores cannot be easily optimized for many different kind of\n+contents.\n+\n+So the current stores are not flexible enough for some important use\n+cases like handling really big binary files or handling a really big\n+number of files that are fetched only as needed. And it is not\n+realistic to expect that Git could fully natively handle many of such\n+use cases. Git would need to natively implement different internal\n+stores which would be a huge burden and which could lead to\n+re-implement things like HTTP servers, Docker registries or artifact\n+stores that already exist outside Git.\n+\n+Furthermore many improvements that are dependent on specific setups\n+could be implemented in the way Git objects are managed if it was\n+possible to customize how the Git objects are handled. For example a\n+restartable clone using the bundle mechanism has often been requested,\n+but implementing that would go against the current strict rules under\n+which the Git objects are currently handled.\n+\n+What Git needs is a mechanism to make it possible to customize in a\n+lot of different ways how the Git objects are handled. Though this\n+mechanism should try as much as possible to avoid interfering with the\n+usual way in which Git handle its objects.\n+\n+Helpers\n+=======\n+\n+ODB helpers are commands that have to be registered using either the\n+\"odb..subprocessCommand\" or the \"odb..scriptCommand\"\n+config variables.\n+\n+Registering such a command tells Git that an external odb called\n+ exists and that the registered command should be used to\n+communicate with it.\n+\n+The communication happens through instructions that are sent by Git\n+and that the commands should answer. If it makes sense, Git can send\n+the same instruction to many commands in the order in which they are\n+configured.\n+\n+There are 2 kinds of commands. Commands registered using the\n+\"odb..subprocessCommand\" config variable are called \"process\n+commands\" and the associated mode is called \"process mode\". Commands\n+registered using the \"odb..scriptCommand\" config variables\n+are called \"script commands\" and the associated mode is called \"script\n+mode\".\n+\n+Early on git commands send an 'init' instruction to the registered\n+commands. A capability negociation will take place during this\n+request/response exchange which will let Git and the helpers know how\n+they can further collaborate. The attribute system can also be used to\n+tell Git which objects should be handled by which helper.\n+\n+Process Mode\n+============\n+\n+In process mode the command is started as a single process invocation\n+that should last for the entire life of the single Git command that\n+started it.\n+\n+A packet format (pkt-line, see technical/protocol-common.txt) based\n+protocol over standard input and standard output is used for\n+communication between Git and the helper command.\n+\n+After the process command is started, Git sends a welcome message\n+(\"git-read-object-client\"), a list of supported protocol version\n+numbers, and a flush packet. Git expects to read a welcome response\n+message (\"git-read-object-server\"), exactly one protocol version\n+number from the previously sent list, and a flush packet. All further\n+communication will be based on the selected version.\n+\n+The remaining protocol description below documents \"version=1\". Please\n+note that \"version=42\" in the example below does not exist and is only\n+there to illustrate how the protocol would look with more than one\n+version.\n+\n+After the version negotiation Git sends a list of all capabilities\n+that it supports and a flush packet. Git expects to read a list of\n+desired capabilities, which must be a subset of the supported\n+capabilities list, and a flush packet as response:\n+\n+------------------------\n+packet: git> git-read-object-client\n+packet: git> version=1\n+packet: git> version=42\n+packet: git> 0000\n+packet: git< git-read-object-server\n+packet: git< version=1\n+packet: git< 0000\n+packet: git> capability=get_raw_obj\n+packet: git> capability=have\n+packet: git> capability=put_raw_obj\n+packet: git> capability=not-yet-invented\n+packet: git> 0000\n+packet: git< capability=get_raw_obj\n+packet: git< 0000\n+------------------------\n+\n+Afterwards Git sends a list of \"key=value\" pairs terminated with a\n+flush packet. The list will contain at least the instruction (based on\n+the supported capabilities) and the arguments for the\n+instruction. Please note, that the process must not send any response\n+before it received the final flush packet.\n+\n+In general any response from the helper should end with a status\n+packet. See the documentation of the 'get_*' instructions below for\n+examples of status packets.\n+\n+After the helper has processed an instruction, it is expected to wait\n+for the next \"key=value\" list containing another instruction.\n+\n+On exit Git will close the pipe to the helper. The helper is then\n+expected to detect EOF and exit gracefully on its own. Git will wait\n+until the process has stopped.\n+\n+Script Mode\n+===========\n+\n+In this mode Git launches the script command each time it wants to\n+communicates with the helper. There is no welcome message and no\n+protocol version in this mode.\n+\n+The instruction and associated arguments are passed as arguments when\n+launching the script command and if needed further information is\n+passed between Git and the command through stdin and stdout.\n+\n+Capabilities/Instructions\n+=========================\n+\n+The following instructions are currently supported by Git:\n+\n+- init\n+- get_git_obj\n+- get_raw_obj\n+- get_direct\n+- put_raw_obj\n+- have\n+\n+The plan is to also support 'put_git_obj' and 'put_direct' soon, for\n+consistency with the 'get_*' instructions.\n+\n+ - 'init'\n+\n+All the process and script commands must accept the 'init'\n+instruction. It should be the first instruction sent to a command. It\n+should not be advertised in the capability exchange. Any argument\n+should be ignored.\n+\n+In process mode, after receiving the 'init' instruction and a flush\n+packet, the helper should just send a status packet and then a flush\n+packet. See the 'get_*' instructions below for examples of status\n+packets.\n+\n+In script mode the command should print on stdout the capabilities\n+that it supports if any. This is the only time in script mode when a\n+capability exchange happens.\n+\n+For example a script command could use the following shell code\n+snippet to handle the 'init' instruction:\n+\n+------------------------\n+case \"$1\" in\n+init)\n+\techo \"capability=get_git_obj\"\n+\techo \"capability=put_raw_obj\"\n+\techo \"capability=have\"\n+\t;;\n+------------------------\n+\n+ - 'get_git_obj ' and 'get_raw_obj '\n+\n+These instructions should have a hexadecimal argument to tell\n+which object the helper should send to git.\n+\n+In process mode the sha1 argument should be followed by a flush packet\n+like this:\n+\n+------------------------\n+packet: git> command=get_git_obj\n+packet: git> sha1=0a214a649e1b3d5011e14a3dc227753f2bd2be05\n+packet: git> 0000\n+------------------------\n+\n+After reading that the helper should send the requested object to Git in a\n+packet series followed by a flush packet. If the helper does not experience\n+problems then the helper must send a \"success\" status like the following:\n+\n+------------------------\n+packet: git< status=success\n+packet: git< 0000\n+------------------------\n+\n+In case the helper cannot or does not want to send the requested\n+object as well as any other object for the lifetime of the Git\n+process, then it is expected to respond with an \"abort\" status at any\n+point in the protocol:\n+\n+------------------------\n+packet: git< status=abort\n+packet: git< 0000\n+------------------------\n+\n+Git neither stops nor restarts the helper in case a\n+\"notfound\"/\"error\"/\"abort\" status is set. An \"error\" status means a\n+possibly more transient error than an abort. The helper should also\n+send a \"notfound\" error in case of a \"get_*\" instruction, which means\n+that the requested object cannot be found.\n+\n+If the helper dies during the communication or does not adhere to the\n+protocol then Git will stop and restart it with the next instruction.\n+\n+In script mode the helper should just send the requested object to Git\n+by writing it to stdout and should then exit. The exit code should\n+signal to Git if a problem occured or not.\n+\n+The only difference between 'get_git_obj' and 'get_raw_obj' is that in\n+case of 'get_git_obj' the requested object should be sent as a Git\n+object (that is in the same format as loose object files). In case of\n+'get_raw_obj' the object should be sent in its raw format (that is the\n+same output as `git cat-file `).\n+\n+ - 'get_direct '\n+\n+This instruction is similar as the other 'get_*' instructions except\n+that no object should be sent from the helper to Git. Instead the\n+helper should directly write the requested object into a loose object\n+file in the \".git/objects\" directory.\n+\n+After the helper has sent the \"status=success\" packet and the\n+following flush packet in process mode, or after it has exited in the\n+script mode, Git will lookup again for the requested sha1 in its loose\n+object files and pack files.\n+\n+ - 'put_raw_obj '\n+\n+This instruction should be following by three arguments to tell which\n+object the helper will receive from git: , and\n+. The hexadecimal argument describes the object that will\n+be sent from Git to the helper. The is the object type (\"blob\",\n+\"tree\", \"commit\" or \"tag\") of this object. The is the size in\n+bytes of the (decompressed) object content.\n+\n+In process mode the last argument (the type) should be followed by a\n+flush packet.\n+\n+After reading that the helper should read the announced object from\n+Git in a packet series followed by a flush packet.\n+\n+If the helper does not experience problems when receiving and storing\n+or processing the object, then the helper must send a \"success\" status\n+as described for the 'get_*' instructions.\n+\n+In script mode the helper should just receive the announced object\n+from its standard input. After receiving and processing the object,\n+the helper should exit and its exit code should signal to Git if a\n+problem occured or not.\n+\n+- 'have'\n+\n+In process mode this instruction should be followed by a flush\n+packet. After receiving this packet the helper should send the sha1,\n+size and type, in this order, of all the objects it can provide to Git\n+(through a 'get_*' instruction). There should be a space character\n+between the sha1 and the size and between the size and the type, and\n+then a new line character after the type.\n+\n+If many packets are needed to send back all this information, the\n+split between packets should be made after the new line characters.\n+\n+If the helper does not experience problems, then it must then send a\n+\"success\" status as described for the 'get_*' instructions.\n+\n+In script mode the helper should send to its standard output the sha1,\n+size and type, in this order of all the objects it can provide to\n+Git. There should also be a space character between the sha1 and the\n+size and between the size and the type, and then a new line character\n+after the type.\n+\n+After sending this, the script helper should exit and its exit code\n+should signal to Git if a problem occured or not.\n+\n+Order of instructions\n+=====================\n+\n+For get_*_object instructions the regular code to find objects is\n+called before the odb helpers.\n+\n+For put_*_object instructions the regular code to store the objects is\n+called after the odb helpers.\n+\n+For now this order is not configurable.\n+\n+Object caching\n+==============\n+\n+If a helper returns the object data as requested by get_git_obj or\n+get_raw_obj, then Git will itself store the object locally in its\n+regular object store, so it is redundant for the helper to also store\n+or try to store the object in the regular object store.\n+\n+Yeah, this seems to defeat the goal of enabling specialized object\n+handlers to handle large or other \"unusual\" objects that git normally\n+doesn't deal well with. So in the long run there should be a way to\n+make this configurable.\n+\n+Selecting objects\n+=================\n+\n+To select objects that should be handled by an external odb, one can\n+use the git attributes system. For now this will only work with blobs\n+and this will only work along with the 'put_raw_obj' instruction.\n+\n+For example if one has an external odb called \"magic\" and has\n+registered an associated a process command helper that supports the\n+'put_raw_obj' instruction, then one can tell Git that all the .jpg\n+files should be handled by the \"magic\" odb using a .gitattributes file\n+can that contains:\n+\n+------------------------\n+*.jpg odb=magic\n+------------------------\n+\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335740","messageId":"20180103163403.11303-36-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 35/40] odb-helper: add put_object_process()","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:58Z","receivedAt":"2018-01-03T16:35:43Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This adds the infrastructure to send objects to a sub-process\nhandling the communication with an external odb.\n\nFor now we only handle sending raw blobs using the 'put_raw_obj'\ninstruction.\n\nSigned-off-by: Christian Couder \n---\n odb-helper.c | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++---\n 1 file changed, 72 insertions(+), 3 deletions(-)\n\ndiff --git a/odb-helper.c b/odb-helper.c\nindex a67dfddca0..d901f6d0bc 100644\n--- a/odb-helper.c\n+++ b/odb-helper.c\n@@ -434,6 +434,58 @@ static int get_object_process(struct odb_helper *o, const unsigned char *sha1, i\n \t\t\t\t\t o->dealer, cur_cap);\n }\n \n+static int send_put_packets(struct object_process *entry,\n+\t\t\t const unsigned char *sha1,\n+\t\t\t const void *buf,\n+\t\t\t size_t len,\n+\t\t\t struct strbuf *status)\n+{\n+\tstruct child_process *process = &entry->subprocess.process;\n+\tint err = packet_write_fmt_gently(process->in, \"command=put_raw_obj\\n\");\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = packet_write_fmt_gently(process->in, \"sha1=%s\\n\", sha1_to_hex(sha1));\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = packet_write_fmt_gently(process->in, \"size=%\"PRIuMAX\"\\n\", len);\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = packet_write_fmt_gently(process->in, \"kind=blob\\n\");\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = packet_flush_gently(process->in);\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = write_packetized_from_buf(buf, len, process->in);\n+\tif (err)\n+\t\treturn err;\n+\n+\treturn check_object_process_status(process->out, status);\n+}\n+\n+static int put_object_process(struct odb_helper *o,\n+\t\t\t const void *buf, size_t len,\n+\t\t\t const char *type, unsigned char *sha1)\n+{\n+\tint err;\n+\tstruct object_process *entry;\n+\tstruct strbuf status = STRBUF_INIT;\n+\n+\tentry = launch_object_process(o, ODB_HELPER_CAP_PUT_RAW_OBJ);\n+\tif (!entry)\n+\t\treturn -1;\n+\n+\terr = send_put_packets(entry, sha1, buf, len, &status);\n+\n+\treturn check_object_process_error(err, status.buf, entry, o->dealer,\n+\t\t\t\t\t ODB_HELPER_CAP_PUT_RAW_OBJ);\n+}\n+\n struct odb_helper *odb_helper_new(const char *name, int namelen)\n {\n \tstruct odb_helper *o;\n@@ -908,9 +960,9 @@ int odb_helper_get_object(struct odb_helper *o,\n \treturn res;\n }\n \n-int odb_helper_put_object(struct odb_helper *o,\n-\t\t\t const void *buf, size_t len,\n-\t\t\t const char *type, unsigned char *sha1)\n+static int put_raw_object_script(struct odb_helper *o,\n+\t\t\t\t const void *buf, size_t len,\n+\t\t\t\t const char *type, unsigned char *sha1)\n {\n \tstruct odb_helper_cmd cmd;\n \n@@ -936,3 +988,20 @@ int odb_helper_put_object(struct odb_helper *o,\n \todb_helper_finish(o, &cmd);\n \treturn 0;\n }\n+\n+int odb_helper_put_object(struct odb_helper *o,\n+\t\t\t const void *buf, size_t len,\n+\t\t\t const char *type, unsigned char *sha1)\n+{\n+\tint res;\n+\tuint64_t start = getnanotime();\n+\n+\tif (o->type == ODB_HELPER_SCRIPT_CMD)\n+\t\tres = put_raw_object_script(o, buf, len, type, sha1);\n+\telse if (o->type == ODB_HELPER_SUBPROCESS_CMD)\n+\t\tres = put_object_process(o, buf, len, type, sha1);\n+\n+\ttrace_performance_since(start, \"odb_helper_put_object\");\n+\n+\treturn res;\n+}\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335741","messageId":"20180103163403.11303-37-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 36/40] Add t0470 to test passing raw objects","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:59Z","receivedAt":"2018-01-03T16:35:52Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"Signed-off-by: Christian Couder \n---\n t/t0470-read-object-http-e-odb.sh | 109 ++++++++++++++++++++++++++++++++++++++\n t/t0470/read-object-plain | 83 +++++++++++++++++++++++++++++\n 2 files changed, 192 insertions(+)\n create mode 100755 t/t0470-read-object-http-e-odb.sh\n create mode 100755 t/t0470/read-object-plain\n\ndiff --git a/t/t0470-read-object-http-e-odb.sh b/t/t0470-read-object-http-e-odb.sh\nnew file mode 100755\nindex 0000000000..774528c04f\n--- /dev/null\n+++ b/t/t0470-read-object-http-e-odb.sh\n@@ -0,0 +1,109 @@\n+#!/bin/sh\n+\n+test_description='tests for read-object process passing plain objects to an HTTPD server'\n+\n+. ./test-lib.sh\n+\n+# If we don't specify a port, the current test number will be used\n+# which will not work as it is less than 1024, so it can only be used by root.\n+LIB_HTTPD_PORT=$(expr ${this_test#t} + 12000)\n+\n+. \"$TEST_DIRECTORY\"/lib-httpd.sh\n+\n+start_httpd apache-e-odb.conf\n+\n+PATH=\"$PATH:$TEST_DIRECTORY/t0470\"\n+\n+# odb helper script must see this\n+export HTTPD_URL\n+\n+HELPER=\"read-object-plain\"\n+\n+test_expect_success 'setup repo with a root commit' '\n+\ttest_commit zero\n+'\n+\n+test_expect_success 'setup another repo from the first one' '\n+\tgit init other-repo &&\n+\t(cd other-repo &&\n+\t git remote add origin .. &&\n+\t git pull origin master &&\n+\t git checkout master &&\n+\t git log)\n+'\n+\n+test_expect_success 'setup the helper in the root repo' '\n+\tgit config odb.magic.subprocessCommand \"$HELPER\"\n+'\n+\n+UPLOADFILENAME=\"hello_apache_upload.txt\"\n+\n+UPLOAD_URL=\"$HTTPD_URL/upload/?sha1=$UPLOADFILENAME&size=123&type=blob\"\n+\n+test_expect_success 'can upload a file' '\n+\techo \"Hello Apache World!\" >hello_to_send.txt &&\n+\techo \"How are you?\" >>hello_to_send.txt &&\n+\tcurl --data-binary @hello_to_send.txt --include \"$UPLOAD_URL\" >out_upload\n+'\n+\n+LIST_URL=\"$HTTPD_URL/list/\"\n+\n+test_expect_success 'can list uploaded files' '\n+\tcurl --include \"$LIST_URL\" >out_list &&\n+\tgrep \"$UPLOADFILENAME\" out_list\n+'\n+\n+test_expect_success 'can delete uploaded files' '\n+\tcurl --data \"delete\" --include \"$UPLOAD_URL&delete=1\" >out_delete &&\n+\tcurl --include \"$LIST_URL\" >out_list2 &&\n+\t! grep \"$UPLOADFILENAME\" out_list2\n+'\n+\n+FILES_DIR=\"httpd/www/files\"\n+\n+test_expect_success 'new blobs are transfered to the http server' '\n+\ttest_commit one &&\n+\thash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\\ -f3) &&\n+\techo \"$hash1-4-blob\" >expected &&\n+\tls \"$FILES_DIR\" >actual &&\n+\ttest_cmp expected actual\n+'\n+\n+test_expect_success 'blobs can be retrieved from the http server' '\n+\tgit cat-file blob \"$hash1\" &&\n+\tgit log -p >expected\n+'\n+\n+test_expect_success 'update other repo from the first one' '\n+\t(cd other-repo &&\n+\t git fetch origin \"refs/odbs/magic/*:refs/odbs/magic/*\" &&\n+\t test_must_fail git cat-file blob \"$hash1\" &&\n+\t git config odb.magic.subprocesscommand \"$HELPER\" &&\n+\t git cat-file blob \"$hash1\" &&\n+\t git pull origin master)\n+'\n+\n+test_expect_success 'local clone from the first repo' '\n+\tmkdir my-clone &&\n+\t(cd my-clone &&\n+\t git clone .. . &&\n+\t git cat-file blob \"$hash1\")\n+'\n+\n+test_expect_success 'no-local clone from the first repo fails' '\n+\tmkdir my-other-clone &&\n+\t(cd my-other-clone &&\n+\t test_must_fail git clone --no-local .. .) &&\n+\trm -rf my-other-clone\n+'\n+\n+test_expect_success 'no-local clone from the first repo with helper succeeds' '\n+\tmkdir my-other-clone &&\n+\t(cd my-other-clone &&\n+\t git clone -c odb.magic.subprocessCommand=\"$HELPER\" --no-local .. .) &&\n+\trm -rf my-other-clone\n+'\n+\n+stop_httpd\n+\n+test_done\ndiff --git a/t/t0470/read-object-plain b/t/t0470/read-object-plain\nnew file mode 100755\nindex 0000000000..0766e16032\n--- /dev/null\n+++ b/t/t0470/read-object-plain\n@@ -0,0 +1,83 @@\n+#!/usr/bin/perl\n+#\n+\n+use 5.008;\n+use lib (split(/:/, $ENV{GITPERLLIB}));\n+use strict;\n+use warnings;\n+use Git::Packet;\n+use LWP::UserAgent;\n+use HTTP::Request::Common;\n+\n+packet_initialize(\"git-read-object\", 1);\n+\n+my %remote_caps = packet_read_and_check_capabilities(\"get_raw_obj\", \"put_raw_obj\");\n+packet_check_and_write_capabilities(\\%remote_caps, \"get_raw_obj\", \"put_raw_obj\");\n+\n+my $http_url = $ENV{HTTPD_URL};\n+\n+while (1) {\n+\tmy ($res, $command) = packet_txt_read();\n+\n+\tif ( $res == -1 ) {\n+\t\texit 0;\n+\t}\n+\n+\t$command =~ s/^command=//;\n+\n+\tif ( $command eq \"init\" ) {\n+\t\tpacket_bin_read();\n+\n+\t\tpacket_txt_write(\"status=success\");\n+\t\tpacket_flush();\n+\t} elsif ( $command eq \"get_raw_obj\" ) {\n+\t\tmy ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/;\n+\t\tpacket_bin_read();\n+\n+\t\tmy $get_url = $http_url . \"/list/?sha1=\" . $sha1;\n+\n+\t\tmy $userAgent = LWP::UserAgent->new();\n+\n+\t\tmy $response = $userAgent->get( $get_url );\n+\n+\t\tif ($response->is_error) {\n+\t\t packet_txt_write(\"size=0\");\n+\t\t packet_txt_write(\"kind=none\");\t \n+\t\t packet_txt_write(\"status=notfound\");\n+\t\t} else {\n+\t\t packet_txt_write(\"size=\" . length($response->content));\n+\t\t packet_txt_write(\"kind=blob\");\n+\t\t packet_bin_write($response->content);\n+\t\t packet_flush();\n+\t\t packet_txt_write(\"status=success\");\n+\t\t}\n+\n+\t\tpacket_flush();\n+\t} elsif ( $command eq \"put_raw_obj\" ) {\n+\t\tmy ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/;\n+\t\tmy ($size) = packet_txt_read() =~ /^size=([0-9]+)$/;\n+\t\tmy ($kind) = packet_txt_read() =~ /^kind=(\\w+)$/;\n+\t\tpacket_bin_read();\n+\n+\t\t# We must read the content we are sent and send it to the right url\n+\t\tmy ($res, $buf) = packet_bin_read();\n+\t\tdie \"bad packet_bin_read res ($res)\" unless ($res eq 0);\n+\t\t( packet_bin_read() eq ( 1, \"\" ) ) || die \"bad send end\";\t\t\n+\n+\t\tmy $upload_url = $http_url . \"/upload/?sha1=\" . $sha1 . \"&size=\" . $size . \"&type=blob\";\n+\n+\t\tmy $userAgent = LWP::UserAgent->new();\n+\t\tmy $request = POST $upload_url, Content_Type => 'multipart/form-data', Content => $buf;\n+\n+\t\tmy $response = $userAgent->request($request);\n+\n+\t\tif ($response->is_error) {\n+\t\t\tpacket_txt_write(\"status=failure\");\n+\t\t} else {\n+\t\t\tpacket_txt_write(\"status=success\");\n+\t\t}\n+\t\tpacket_flush();\n+\t} else {\n+\t\tdie \"bad command '$command'\";\n+\t}\n+}\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335742","messageId":"20180103163403.11303-38-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 37/40] odb-helper: add have_object_process()","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:34:00Z","receivedAt":"2018-01-03T16:35:55Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This adds the infrastructure to handle 'have' instructions in\nprocess mode.\n\nThe answer from the helper sub-process should be like the\noutput in script mode, that is lines like this:\n\nsha1 SPACE size SPACE type NEWLINE\n\nSigned-off-by: Christian Couder \n---\n odb-helper.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--\n 1 file changed, 74 insertions(+), 2 deletions(-)\n\ndiff --git a/odb-helper.c b/odb-helper.c\nindex d901f6d0bc..d8902a9541 100644\n--- a/odb-helper.c\n+++ b/odb-helper.c\n@@ -635,6 +635,70 @@ static int odb_helper_object_cmp(const void *va, const void *vb)\n \treturn hashcmp(a->sha1, b->sha1);\n }\n \n+static int send_have_packets(struct odb_helper *o,\n+\t\t\t struct object_process *entry,\n+\t\t\t struct strbuf *status)\n+{\n+\tint packet_len;\n+\tint total_got = 0;\n+\tstruct child_process *process = &entry->subprocess.process;\n+\tint err = packet_write_fmt_gently(process->in, \"command=have\\n\");\n+\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = packet_flush_gently(process->in);\n+\tif (err)\n+\t\treturn err;\n+\n+\tfor (;;) {\n+\t\t/* packet_read() writes a '\\0' extra byte at the end */\n+\t\tchar buf[LARGE_PACKET_DATA_MAX + 1];\n+\t\tchar *p = buf;\n+\t\tint more;\n+\n+\t\tpacket_len = packet_read(process->out, NULL, NULL,\n+\t\t\tbuf, LARGE_PACKET_DATA_MAX + 1,\n+\t\t\tPACKET_READ_GENTLE_ON_EOF);\n+\n+\t\tif (packet_len <= 0)\n+\t\t\tbreak;\n+\n+\t\ttotal_got += packet_len;\n+\n+\t\t/* 'have' packets should end with '\\n' or '\\0' */\n+\t\tdo {\n+\t\t\tchar *eol = strchrnul(p, '\\n');\n+\t\t\tmore = (*eol == '\\n');\n+\t\t\t*eol = '\\0';\n+\t\t\tif (add_have_entry(o, p))\n+\t\t\t\tbreak;\n+\t\t\tp = eol + 1;\n+\t\t} while (more && *p);\n+\t}\n+\n+\tif (packet_len < 0)\n+\t\treturn packet_len;\n+\n+\treturn check_object_process_status(process->out, status);\n+}\n+\n+static int have_object_process(struct odb_helper *o)\n+{\n+\tint err;\n+\tstruct object_process *entry;\n+\tstruct strbuf status = STRBUF_INIT;\n+\n+\tentry = launch_object_process(o, ODB_HELPER_CAP_HAVE);\n+\tif (!entry)\n+\t\treturn -1;\n+\n+\terr = send_have_packets(o, entry, &status);\n+\n+\treturn check_object_process_error(err, status.buf, entry, o->dealer,\n+\t\t\t\t\t ODB_HELPER_CAP_HAVE);\n+}\n+\n static void have_object_script(struct odb_helper *o)\n {\n \tstruct odb_helper_cmd cmd;\n@@ -656,12 +720,20 @@ static void have_object_script(struct odb_helper *o)\n \n static void odb_helper_load_have(struct odb_helper *o)\n {\n+\tuint64_t start;\n+\n \tif (o->have_valid)\n \t\treturn;\n \to->have_valid = 1;\n \n+\tstart = getnanotime();\n+\n \tif (o->type == ODB_HELPER_SCRIPT_CMD)\n \t\thave_object_script(o);\n+\telse if (o->type == ODB_HELPER_SUBPROCESS_CMD)\n+\t\thave_object_process(o);\n+\n+\ttrace_performance_since(start, \"odb_helper_load_have\");\n \n \tqsort(o->have, o->have_nr, sizeof(*o->have), odb_helper_object_cmp);\n }\n@@ -923,7 +995,7 @@ int odb_helper_get_direct(struct odb_helper *o,\n \t\tfetch_object(o->dealer, sha1);\n \telse if (o->type == ODB_HELPER_SCRIPT_CMD)\n \t\tres = get_direct_script(o, sha1);\n-\telse\n+\telse if (o->type == ODB_HELPER_SUBPROCESS_CMD)\n \t\tres = get_object_process(o, sha1, -1);\n \n \ttrace_performance_since(start, \"odb_helper_get_direct\");\n@@ -993,7 +1065,7 @@ int odb_helper_put_object(struct odb_helper *o,\n \t\t\t const void *buf, size_t len,\n \t\t\t const char *type, unsigned char *sha1)\n {\n-\tint res;\n+\tint res = 0;\n \tuint64_t start = getnanotime();\n \n \tif (o->type == ODB_HELPER_SCRIPT_CMD)\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335743","messageId":"20180103163403.11303-39-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 38/40] Add t0480 to test \"have\" capability and raw objects","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:34:01Z","receivedAt":"2018-01-03T16:35:58Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"Signed-off-by: Christian Couder \n---\n t/t0480-read-object-have-http-e-odb.sh | 109 +++++++++++++++++++++++++++++++++\n t/t0480/read-object-plain-have | 103 +++++++++++++++++++++++++++++++\n 2 files changed, 212 insertions(+)\n create mode 100755 t/t0480-read-object-have-http-e-odb.sh\n create mode 100755 t/t0480/read-object-plain-have\n\ndiff --git a/t/t0480-read-object-have-http-e-odb.sh b/t/t0480-read-object-have-http-e-odb.sh\nnew file mode 100755\nindex 0000000000..056a40f2bb\n--- /dev/null\n+++ b/t/t0480-read-object-have-http-e-odb.sh\n@@ -0,0 +1,109 @@\n+#!/bin/sh\n+\n+test_description='tests for read-object process with \"have\" cap and plain objects'\n+\n+. ./test-lib.sh\n+\n+# If we don't specify a port, the current test number will be used\n+# which will not work as it is less than 1024, so it can only be used by root.\n+LIB_HTTPD_PORT=$(expr ${this_test#t} + 12000)\n+\n+. \"$TEST_DIRECTORY\"/lib-httpd.sh\n+\n+start_httpd apache-e-odb.conf\n+\n+PATH=\"$PATH:$TEST_DIRECTORY/t0480\"\n+\n+# odb helper script must see this\n+export HTTPD_URL\n+\n+HELPER=\"read-object-plain-have\"\n+\n+test_expect_success 'setup repo with a root commit' '\n+\ttest_commit zero\n+'\n+\n+test_expect_success 'setup another repo from the first one' '\n+\tgit init other-repo &&\n+\t(cd other-repo &&\n+\t git remote add origin .. &&\n+\t git pull origin master &&\n+\t git checkout master &&\n+\t git log)\n+'\n+\n+test_expect_success 'setup the helper in the root repo' '\n+\tgit config odb.magic.subprocessCommand \"$HELPER\"\n+'\n+\n+UPLOADFILENAME=\"hello_apache_upload.txt\"\n+\n+UPLOAD_URL=\"$HTTPD_URL/upload/?sha1=$UPLOADFILENAME&size=123&type=blob\"\n+\n+test_expect_success 'can upload a file' '\n+\techo \"Hello Apache World!\" >hello_to_send.txt &&\n+\techo \"How are you?\" >>hello_to_send.txt &&\n+\tcurl --data-binary @hello_to_send.txt --include \"$UPLOAD_URL\" >out_upload\n+'\n+\n+LIST_URL=\"$HTTPD_URL/list/\"\n+\n+test_expect_success 'can list uploaded files' '\n+\tcurl --include \"$LIST_URL\" >out_list &&\n+\tgrep \"$UPLOADFILENAME\" out_list\n+'\n+\n+test_expect_success 'can delete uploaded files' '\n+\tcurl --data \"delete\" --include \"$UPLOAD_URL&delete=1\" >out_delete &&\n+\tcurl --include \"$LIST_URL\" >out_list2 &&\n+\t! grep \"$UPLOADFILENAME\" out_list2\n+'\n+\n+FILES_DIR=\"httpd/www/files\"\n+\n+test_expect_success 'new blobs are transfered to the http server' '\n+\ttest_commit one &&\n+\thash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\\ -f3) &&\n+\techo \"$hash1-4-blob\" >expected &&\n+\tls \"$FILES_DIR\" >actual &&\n+\ttest_cmp expected actual\n+'\n+\n+test_expect_success 'blobs can be retrieved from the http server' '\n+\tgit cat-file blob \"$hash1\" &&\n+\tgit log -p >expected\n+'\n+\n+test_expect_success 'update other repo from the first one' '\n+\t(cd other-repo &&\n+\t git fetch origin \"refs/odbs/magic/*:refs/odbs/magic/*\" &&\n+\t test_must_fail git cat-file blob \"$hash1\" &&\n+\t git config odb.magic.subprocessCommand \"$HELPER\" &&\n+\t git cat-file blob \"$hash1\" &&\n+\t git pull origin master)\n+'\n+\n+test_expect_success 'local clone from the first repo' '\n+\tmkdir my-clone &&\n+\t(cd my-clone &&\n+\t git clone .. . &&\n+\t git cat-file blob \"$hash1\")\n+'\n+\n+test_expect_success 'no-local clone from the first repo fails' '\n+\tmkdir my-other-clone &&\n+\t(cd my-other-clone &&\n+\t test_must_fail git clone --no-local .. .) &&\n+\trm -rf my-other-clone\n+'\n+\n+test_expect_success 'no-local clone from the first repo with helper succeeds' '\n+\tmkdir my-other-clone &&\n+\t(cd my-other-clone &&\n+\t git clone -c odb.magic.subprocessCommand=\"$HELPER\" --no-local .. .) &&\n+\trm -rf my-other-clone\n+'\n+\n+stop_httpd\n+\n+test_done\ndiff --git a/t/t0480/read-object-plain-have b/t/t0480/read-object-plain-have\nnew file mode 100755\nindex 0000000000..f230cbd5eb\n--- /dev/null\n+++ b/t/t0480/read-object-plain-have\n@@ -0,0 +1,103 @@\n+#!/usr/bin/perl\n+#\n+\n+use 5.008;\n+use lib (split(/:/, $ENV{GITPERLLIB}));\n+use strict;\n+use warnings;\n+use Git::Packet;\n+use LWP::UserAgent;\n+use HTTP::Request::Common;\n+\n+packet_initialize(\"git-read-object\", 1);\n+\n+my %remote_caps = packet_read_and_check_capabilities(\"get_raw_obj\", \"put_raw_obj\", \"have\");\n+packet_check_and_write_capabilities(\\%remote_caps, \"get_raw_obj\", \"put_raw_obj\", \"have\");\n+\n+my $http_url = $ENV{HTTPD_URL};\n+\n+while (1) {\n+\tmy ($res, $command) = packet_txt_read();\n+\n+\tif ( $res == -1 ) {\n+\t\texit 0;\n+\t}\n+\n+\t$command =~ s/^command=//;\n+\n+\tif ( $command eq \"init\" ) {\n+\t\tpacket_bin_read();\n+\n+\t\tpacket_txt_write(\"status=success\");\n+\t\tpacket_flush();\n+\t} elsif ( $command eq \"have\" ) {\n+\t\t# read the flush after the command\n+\t\tpacket_bin_read();\n+\n+\t\tmy $have_url = $http_url . \"/list/\";\n+\n+\t\tmy $userAgent = LWP::UserAgent->new();\n+\t\tmy $response = $userAgent->get( $have_url );\n+\n+\t\tif ($response->is_error) {\n+\t\t\tpacket_bin_write(\"\");\n+\t\t\tpacket_flush();\n+\t\t\tpacket_txt_write(\"status=failure\");\n+\t\t} else {\n+\t\t\tpacket_bin_write($response->content);\n+\t\t\tpacket_flush();\n+\t\t\tpacket_txt_write(\"status=success\");\n+\t\t}\n+\t\tpacket_flush();\n+\t} elsif ( $command eq \"get_raw_obj\" ) {\n+\t\tmy ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/;\n+\t\tpacket_bin_read();\n+\n+\t\tmy $get_url = $http_url . \"/list/?sha1=\" . $sha1;\n+\n+\t\tmy $userAgent = LWP::UserAgent->new();\n+\n+\t\tmy $response = $userAgent->get( $get_url );\n+\n+\t\tif ($response->is_error) {\n+\t\t\tpacket_txt_write(\"size=0\");\n+\t\t\tpacket_txt_write(\"kind=none\");\t \n+\t\t\tpacket_txt_write(\"status=notfound\");\n+\t\t} else {\n+\t\t\tpacket_txt_write(\"size=\" . length($response->content));\n+\t\t\tpacket_txt_write(\"kind=blob\");\n+\t\t\tpacket_bin_write($response->content);\n+\t\t\tpacket_flush();\n+\t\t\tpacket_txt_write(\"status=success\");\n+\t\t}\n+\n+\t\tpacket_flush();\n+\t} elsif ( $command eq \"put_raw_obj\" ) {\n+\t\tmy ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/;\n+\t\tmy ($size) = packet_txt_read() =~ /^size=([0-9]+)$/;\n+\t\tmy ($kind) = packet_txt_read() =~ /^kind=(\\w+)$/;\n+\n+\t\tpacket_bin_read();\n+\n+\t\t# We must read the content we are sent and send it to the right url\n+\t\tmy ($res, $buf) = packet_bin_read();\n+\t\tdie \"bad packet_bin_read res ($res)\" unless ($res eq 0);\n+\t\t( packet_bin_read() eq ( 1, \"\" ) ) || die \"bad send end\";\t\t\n+\n+\t\tmy $upload_url = $http_url . \"/upload/?sha1=\" . $sha1 . \"&size=\" . $size . \"&type=blob\";\n+\n+\t\tmy $userAgent = LWP::UserAgent->new();\n+\t\tmy $request = POST $upload_url, Content_Type => 'multipart/form-data', Content => $buf;\n+\n+\t\tmy $response = $userAgent->request($request);\n+\n+\t\tif ($response->is_error) {\n+\t\t\tpacket_txt_write(\"status=failure\");\n+\t\t} else {\n+\t\t\tpacket_txt_write(\"status=success\");\n+\t\t}\n+\t\tpacket_flush();\n+\t} else {\n+\t\tdie \"bad command '$command'\";\n+\t}\n+}\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335744","messageId":"20180103163403.11303-40-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 39/40] external-odb: use 'odb=magic' attribute to mark odb blobs","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:34:02Z","receivedAt":"2018-01-03T16:36:07Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"To tell which blobs should be sent to the \"magic\" external odb,\nlet's require that the blobs be marked using the 'odb=magic'\nattribute.\n\nSigned-off-by: Christian Couder \n---\n external-odb.c | 25 ++++++++++++++++++++++---\n external-odb.h | 3 ++-\n sha1_file.c | 20 +++++++++++++++-----\n t/t0400-external-odb.sh | 3 +++\n t/t0410-transfer-e-odb.sh | 3 +++\n t/t0420-transfer-http-e-odb.sh | 3 +++\n t/t0470-read-object-http-e-odb.sh | 3 +++\n t/t0480-read-object-have-http-e-odb.sh | 3 +++\n 8 files changed, 54 insertions(+), 9 deletions(-)\n\ndiff --git a/external-odb.c b/external-odb.c\nindex 50c1cec50b..e3a05e24e3 100644\n--- a/external-odb.c\n+++ b/external-odb.c\n@@ -2,6 +2,7 @@\n #include \"external-odb.h\"\n #include \"odb-helper.h\"\n #include \"config.h\"\n+#include \"attr.h\"\n \n static struct odb_helper *helpers;\n static struct odb_helper **helpers_tail = &helpers;\n@@ -165,19 +166,37 @@ int external_odb_get_object(const unsigned char *sha1)\n \treturn external_odb_do_get_object(sha1);\n }\n \n+static int has_odb_attrs(struct odb_helper *o, const char *path)\n+{\n+\tstatic struct attr_check *check;\n+\n+\tif (!check)\n+\t\tcheck = attr_check_initl(\"odb\", NULL);\n+\n+\tif (!git_check_attr(path, check)) {\n+\t\tconst char *value = check->items[0].value;\n+\t\treturn value ? !strcmp(o->name, value) : 0;\n+\t}\n+\treturn 0;\n+}\n+\n int external_odb_put_object(const void *buf, size_t len,\n-\t\t\t const char *type, unsigned char *sha1)\n+\t\t\t const char *type, unsigned char *sha1,\n+\t\t\t const char *path)\n {\n \tstruct odb_helper *o;\n \n \texternal_odb_init();\n \n \t/* For now accept only blobs */\n-\tif (strcmp(type, \"blob\"))\n+\tif (!path || strcmp(type, \"blob\"))\n \t\treturn 1;\n \n \tfor (o = helpers; o; o = o->next) {\n-\t\tint r = odb_helper_put_object(o, buf, len, type, sha1);\n+\t\tint r;\n+\t\tif (!has_odb_attrs(o, path))\n+\t\t\tcontinue;\n+\t\tr = odb_helper_put_object(o, buf, len, type, sha1);\n \t\tif (r <= 0)\n \t\t\treturn r;\n \t}\ndiff --git a/external-odb.h b/external-odb.h\nindex 26bb931685..5a8936f417 100644\n--- a/external-odb.h\n+++ b/external-odb.h\n@@ -7,6 +7,7 @@ extern int external_odb_has_object(const unsigned char *sha1);\n extern int external_odb_get_object(const unsigned char *sha1);\n extern int external_odb_get_direct(const unsigned char *sha1);\n extern int external_odb_put_object(const void *buf, size_t len,\n-\t\t\t\t const char *type, unsigned char *sha1);\n+\t\t\t\t const char *type, unsigned char *sha1,\n+\t\t\t\t const char *path);\n \n #endif /* EXTERNAL_ODB_H */\ndiff --git a/sha1_file.c b/sha1_file.c\nindex 300029459f..d3f395e967 100644\n--- a/sha1_file.c\n+++ b/sha1_file.c\n@@ -1696,7 +1696,9 @@ static int freshen_packed_object(const unsigned char *sha1)\n \treturn 1;\n }\n \n-int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)\n+static int write_sha1_file_with_path(const void *buf, unsigned long len,\n+\t\t\t\t const char *type, unsigned char *sha1,\n+\t\t\t\t const char *path)\n {\n \tchar hdr[32];\n \tint hdrlen = sizeof(hdr);\n@@ -1705,13 +1707,19 @@ int write_sha1_file(const void *buf, unsigned long len, const char *type, unsign\n \t * it out into .git/objects/??/?{38} file.\n \t */\n \twrite_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);\n-\tif (!external_odb_put_object(buf, len, type, sha1))\n+\tif (!external_odb_put_object(buf, len, type, sha1, path))\n \t\treturn 0;\n \tif (freshen_packed_object(sha1) || freshen_loose_object(sha1))\n \t\treturn 0;\n \treturn write_loose_object(sha1, hdr, hdrlen, buf, len, 0);\n }\n \n+int write_sha1_file(const void *buf, unsigned long len,\n+\t\t const char *type, unsigned char *sha1)\n+{\n+\treturn write_sha1_file_with_path(buf, len, type, sha1, NULL);\n+}\n+\n int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,\n \t\t\t struct object_id *oid, unsigned flags)\n {\n@@ -1832,7 +1840,8 @@ static int index_mem(struct object_id *oid, void *buf, size_t size,\n \t}\n \n \tif (write_object)\n-\t\tret = write_sha1_file(buf, size, typename(type), oid->hash);\n+\t\tret = write_sha1_file_with_path(buf, size, typename(type),\n+\t\t\t\t\t\toid->hash, path);\n \telse\n \t\tret = hash_sha1_file(buf, size, typename(type), oid->hash);\n \tif (re_allocated)\n@@ -1854,8 +1863,9 @@ static int index_stream_convert_blob(struct object_id *oid, int fd,\n \t\t\t\t get_safe_crlf(flags));\n \n \tif (write_object)\n-\t\tret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),\n-\t\t\t\t oid->hash);\n+\t\tret = write_sha1_file_with_path(sbuf.buf, sbuf.len,\n+\t\t\t\t\t\ttypename(OBJ_BLOB),\n+\t\t\t\t\t\toid->hash, path);\n \telse\n \t\tret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),\n \t\t\t\t oid->hash);\ndiff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh\nindex f924de870f..944b7bb28f 100755\n--- a/t/t0400-external-odb.sh\n+++ b/t/t0400-external-odb.sh\n@@ -71,6 +71,9 @@ test_expect_success 'helper can add objects to alt repo' '\n \n test_expect_success 'commit adds objects to alt repo' '\n \ttest_config odb.magic.scriptCommand \"$HELPER\" &&\n+\techo \"* odb=magic\" >.gitattributes &&\n+\tGIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&\n+\tGIT_NO_EXTERNAL_ODB=1 git commit -m \"Add .gitattributes\" &&\n \ttest_commit three &&\n \thash3=$(git ls-tree HEAD | grep three.t | cut -f1 | cut -d\\ -f3) &&\n \tcontent=$(git -C alt-repo show \"$hash3\") &&\ndiff --git a/t/t0410-transfer-e-odb.sh b/t/t0410-transfer-e-odb.sh\nindex 065ec7d759..fd3e37918c 100755\n--- a/t/t0410-transfer-e-odb.sh\n+++ b/t/t0410-transfer-e-odb.sh\n@@ -111,6 +111,9 @@ test_expect_success 'setup other repo and its alternate repo' '\n '\n \n test_expect_success 'new blobs are put in first object store' '\n+\techo \"* odb=magic\" >.gitattributes &&\n+\tGIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&\n+\tGIT_NO_EXTERNAL_ODB=1 git commit -m \"Add .gitattributes\" &&\n \ttest_commit one &&\n \thash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\\ -f3) &&\n \tcontent=$(cd alt-repo1 && git show \"$hash1\") &&\ndiff --git a/t/t0420-transfer-http-e-odb.sh b/t/t0420-transfer-http-e-odb.sh\nindex f84fe950ec..d307af0457 100755\n--- a/t/t0420-transfer-http-e-odb.sh\n+++ b/t/t0420-transfer-http-e-odb.sh\n@@ -94,6 +94,9 @@ test_expect_success 'can delete uploaded files' '\n FILES_DIR=\"httpd/www/files\"\n \n test_expect_success 'new blobs are transfered to the http server' '\n+\techo \"* odb=magic\" >.gitattributes &&\n+\tGIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&\n+\tGIT_NO_EXTERNAL_ODB=1 git commit -m \"Add .gitattributes\" &&\n \ttest_commit one &&\n \thash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\\ -f3) &&\n \techo \"$hash1-4-blob\" >expected &&\ndiff --git a/t/t0470-read-object-http-e-odb.sh b/t/t0470-read-object-http-e-odb.sh\nindex 774528c04f..d814a43d59 100755\n--- a/t/t0470-read-object-http-e-odb.sh\n+++ b/t/t0470-read-object-http-e-odb.sh\n@@ -62,6 +62,9 @@ test_expect_success 'can delete uploaded files' '\n FILES_DIR=\"httpd/www/files\"\n \n test_expect_success 'new blobs are transfered to the http server' '\n+\techo \"* odb=magic\" >.gitattributes &&\n+\tGIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&\n+\tGIT_NO_EXTERNAL_ODB=1 git commit -m \"Add .gitattributes\" &&\n \ttest_commit one &&\n \thash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\\ -f3) &&\n \techo \"$hash1-4-blob\" >expected &&\ndiff --git a/t/t0480-read-object-have-http-e-odb.sh b/t/t0480-read-object-have-http-e-odb.sh\nindex 056a40f2bb..fe1fac5ef3 100755\n--- a/t/t0480-read-object-have-http-e-odb.sh\n+++ b/t/t0480-read-object-have-http-e-odb.sh\n@@ -62,6 +62,9 @@ test_expect_success 'can delete uploaded files' '\n FILES_DIR=\"httpd/www/files\"\n \n test_expect_success 'new blobs are transfered to the http server' '\n+\techo \"* odb=magic\" >.gitattributes &&\n+\tGIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&\n+\tGIT_NO_EXTERNAL_ODB=1 git commit -m \"Add .gitattributes\" &&\n \ttest_commit one &&\n \thash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\\ -f3) &&\n \techo \"$hash1-4-blob\" >expected &&\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335745","messageId":"20180103163403.11303-11-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 10/40] external-odb: implement external_odb_get_direct","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:33Z","receivedAt":"2018-01-03T16:36:17Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"This is implemented only in the promisor remote mode\nfor now by calling fetch_object().\n\nSigned-off-by: Christian Couder \n---\n external-odb.c | 15 +++++++++++++++\n external-odb.h | 1 +\n odb-helper.c | 13 +++++++++++++\n odb-helper.h | 3 ++-\n 4 files changed, 31 insertions(+), 1 deletion(-)\n\ndiff --git a/external-odb.c b/external-odb.c\nindex d26e63d8b1..5d0afb9762 100644\n--- a/external-odb.c\n+++ b/external-odb.c\n@@ -76,3 +76,18 @@ int external_odb_has_object(const unsigned char *sha1)\n \t\t\treturn 1;\n \treturn 0;\n }\n+\n+int external_odb_get_direct(const unsigned char *sha1)\n+{\n+\tstruct odb_helper *o;\n+\n+\texternal_odb_init();\n+\n+\tfor (o = helpers; o; o = o->next) {\n+\t\tif (odb_helper_get_direct(o, sha1) < 0)\n+\t\t\tcontinue;\n+\t\treturn 0;\n+\t}\n+\n+\treturn -1;\n+}\ndiff --git a/external-odb.h b/external-odb.h\nindex 9a3c2f01b3..fd6708163e 100644\n--- a/external-odb.h\n+++ b/external-odb.h\n@@ -4,5 +4,6 @@\n extern int has_external_odb(void);\n extern const char *external_odb_root(void);\n extern int external_odb_has_object(const unsigned char *sha1);\n+extern int external_odb_get_direct(const unsigned char *sha1);\n \n #endif /* EXTERNAL_ODB_H */\ndiff --git a/odb-helper.c b/odb-helper.c\nindex 1404393807..4b70b287af 100644\n--- a/odb-helper.c\n+++ b/odb-helper.c\n@@ -4,6 +4,7 @@\n #include \"odb-helper.h\"\n #include \"run-command.h\"\n #include \"sha1-lookup.h\"\n+#include \"fetch-object.h\"\n \n struct odb_helper *odb_helper_new(const char *name, int namelen)\n {\n@@ -52,3 +53,15 @@ int odb_helper_has_object(struct odb_helper *o, const unsigned char *sha1)\n \treturn !!odb_helper_lookup(o, sha1);\n }\n \n+int odb_helper_get_direct(struct odb_helper *o,\n+\t\t\t const unsigned char *sha1)\n+{\n+\tint res = 0;\n+\tuint64_t start = getnanotime();\n+\n+\tfetch_object(o->dealer, sha1);\n+\n+\ttrace_performance_since(start, \"odb_helper_get_direct\");\n+\n+\treturn res;\n+}\ndiff --git a/odb-helper.h b/odb-helper.h\nindex 9395e606ce..f4bc66b0ef 100644\n--- a/odb-helper.h\n+++ b/odb-helper.h\n@@ -20,5 +20,6 @@ struct odb_helper {\n extern struct odb_helper *odb_helper_new(const char *name, int namelen);\n extern int odb_helper_has_object(struct odb_helper *o,\n \t\t\t\t const unsigned char *sha1);\n-\n+extern int odb_helper_get_direct(struct odb_helper *o,\n+\t\t\t\t const unsigned char *sha1);\n #endif /* ODB_HELPER_H */\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335746","messageId":"20180103163403.11303-23-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 22/40] Add t0410 to test external ODB transfer","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:45Z","receivedAt":"2018-01-03T16:36:19Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"Signed-off-by: Christian Couder \n---\n t/t0410-transfer-e-odb.sh | 144 ++++++++++++++++++++++++++++++++++++++++++++++\n 1 file changed, 144 insertions(+)\n create mode 100755 t/t0410-transfer-e-odb.sh\n\ndiff --git a/t/t0410-transfer-e-odb.sh b/t/t0410-transfer-e-odb.sh\nnew file mode 100755\nindex 0000000000..065ec7d759\n--- /dev/null\n+++ b/t/t0410-transfer-e-odb.sh\n@@ -0,0 +1,144 @@\n+#!/bin/sh\n+\n+test_description='basic tests for transfering external ODBs'\n+\n+. ./test-lib.sh\n+\n+ORIG_SOURCE=\"$PWD/.git\"\n+export ORIG_SOURCE\n+\n+ALT_SOURCE1=\"$PWD/alt-repo1/.git\"\n+export ALT_SOURCE1\n+write_script odb-helper1 <<\\EOF\n+die() {\n+\tprintf >&2 \"%s\\n\" \"$@\"\n+\texit 1\n+}\n+GIT_DIR=$ALT_SOURCE1; export GIT_DIR\n+case \"$1\" in\n+init)\n+\techo \"capability=get_git_obj\"\n+\techo \"capability=have\"\n+\t;;\n+have)\n+\tgit cat-file --batch-check --batch-all-objects |\n+\tawk '{print $1 \" \" $3 \" \" $2}'\n+\t;;\n+get_git_obj)\n+\tcat \"$GIT_DIR\"/objects/$(echo $2 | sed 's#..#&/#')\n+\t;;\n+put_raw_obj)\n+\tsha1=\"$2\"\n+\tsize=\"$3\"\n+\tkind=\"$4\"\n+\twriten=$(git hash-object -w -t \"$kind\" --stdin)\n+\ttest \"$writen\" = \"$sha1\" || die \"bad sha1 passed '$sha1' vs writen '$writen'\"\n+\tref_hash=$(echo \"$sha1 $size $kind\" | GIT_DIR=$ORIG_SOURCE GIT_NO_EXTERNAL_ODB=1 git hash-object -w -t blob --stdin) || exit\n+\tGIT_DIR=$ORIG_SOURCE git update-ref refs/odbs/magic/\"$sha1\" \"$ref_hash\"\n+\t;;\n+*)\n+\tdie \"unknown command '$1'\"\n+\t;;\n+esac\n+EOF\n+HELPER1=\"\\\"$PWD\\\"/odb-helper1\"\n+\n+OTHER_SOURCE=\"$PWD/.git\"\n+export OTHER_SOURCE\n+\n+ALT_SOURCE2=\"$PWD/alt-repo2/.git\"\n+export ALT_SOURCE2\n+write_script odb-helper2 <<\\EOF\n+die() {\n+\tprintf >&2 \"%s\\n\" \"$@\"\n+\texit 1\n+}\n+GIT_DIR=$ALT_SOURCE2; export GIT_DIR\n+case \"$1\" in\n+init)\n+\techo \"capability=get_git_obj\"\n+\techo \"capability=have\"\n+\t;;\n+have)\n+\tGIT_DIR=$OTHER_SOURCE git for-each-ref --format='%(objectname)' refs/odbs/magic/ | GIT_DIR=$OTHER_SOURCE xargs git show\n+\t;;\n+get_git_obj)\n+\tOBJ_FILE=\"$GIT_DIR\"/objects/$(echo $2 | sed 's#..#&/#')\n+\tif ! test -f \"$OBJ_FILE\"\n+\tthen\n+\t\t# \"Download\" the missing object by copying it from alt-repo1\n+\t\tOBJ_DIR=$(echo $2 | sed 's/\\(..\\).*/\\1/')\n+\t\tOBJ_BASE=$(basename \"$OBJ_FILE\")\n+\t\tALT_OBJ_DIR1=\"$ALT_SOURCE1/objects/$OBJ_DIR\"\n+\t\tALT_OBJ_DIR2=\"$ALT_SOURCE2/objects/$OBJ_DIR\"\n+\t\tmkdir -p \"$ALT_OBJ_DIR2\" || die \"Could not mkdir '$ALT_OBJ_DIR2'\"\n+\t\tOBJ_SRC=\"$ALT_OBJ_DIR1/$OBJ_BASE\"\n+\t\tcp \"$OBJ_SRC\" \"$ALT_OBJ_DIR2\" ||\n+\t\tdie \"Could not cp '$OBJ_SRC' into '$ALT_OBJ_DIR2'\"\n+\tfi\n+\tcat \"$OBJ_FILE\" || die \"Could not cat '$OBJ_FILE'\"\n+\t;;\n+put_raw_obj)\n+\tsha1=\"$2\"\n+\tsize=\"$3\"\n+\tkind=\"$4\"\n+\twriten=$(git hash-object -w -t \"$kind\" --stdin)\n+\ttest \"$writen\" = \"$sha1\" || die \"bad sha1 passed '$sha1' vs writen '$writen'\"\n+\tref_hash=$(echo \"$sha1 $size $kind\" | GIT_DIR=$OTHER_SOURCE GIT_NO_EXTERNAL_ODB=1 git hash-object -w -t blob --stdin) || exit\n+\tGIT_DIR=$OTHER_SOURCE git update-ref refs/odbs/magic/\"$sha1\" \"$ref_hash\"\n+\t;;\n+*)\n+\tdie \"unknown command '$1'\"\n+\t;;\n+esac\n+EOF\n+HELPER2=\"\\\"$PWD\\\"/odb-helper2\"\n+\n+test_expect_success 'setup first alternate repo' '\n+\tgit init alt-repo1 &&\n+\ttest_commit zero &&\n+\tgit config odb.magic.scriptCommand \"$HELPER1\"\n+'\n+\n+test_expect_success 'setup other repo and its alternate repo' '\n+\tgit init other-repo &&\n+\tgit init alt-repo2 &&\n+\t(cd other-repo &&\n+\t git remote add origin .. &&\n+\t git pull origin master &&\n+\t git checkout master &&\n+\t git log)\n+'\n+\n+test_expect_success 'new blobs are put in first object store' '\n+\ttest_commit one &&\n+\thash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\\ -f3) &&\n+\tcontent=$(cd alt-repo1 && git show \"$hash1\") &&\n+\ttest \"$content\" = \"one\" &&\n+\ttest_commit two &&\n+\thash2=$(git ls-tree HEAD | grep two.t | cut -f1 | cut -d\\ -f3) &&\n+\tcontent=$(cd alt-repo1 && git show \"$hash2\") &&\n+\ttest \"$content\" = \"two\"\n+'\n+\n+test_expect_success 'other repo gets the blobs from object store' '\n+\t(cd other-repo &&\n+\t git fetch origin \"refs/odbs/magic/*:refs/odbs/magic/*\" &&\n+\t test_must_fail git cat-file blob \"$hash1\" &&\n+\t test_must_fail git cat-file blob \"$hash2\" &&\n+\t git config odb.magic.scriptCommand \"$HELPER2\" &&\n+\t git cat-file blob \"$hash1\" &&\n+\t git cat-file blob \"$hash2\"\n+\t)\n+'\n+\n+test_expect_success 'other repo gets everything else' '\n+\t(cd other-repo &&\n+\t git fetch origin &&\n+\t content=$(git show \"$hash1\") &&\n+\t test \"$content\" = \"one\" &&\n+\t content=$(git show \"$hash2\") &&\n+\t test \"$content\" = \"two\")\n+'\n+\n+test_done\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335747","messageId":"20180103163403.11303-21-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 20/40] external-odb: accept only blobs for now","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:43Z","receivedAt":"2018-01-03T16:36:21Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"The mechanism to decide which blobs should be sent to which\nexternal object database will be very simple for now.\nIf the external odb helper support any \"put_*\" instruction\nall the new blobs will be sent to it.\n\nSigned-off-by: Christian Couder \n---\n external-odb.c | 4 ++++\n 1 file changed, 4 insertions(+)\n\ndiff --git a/external-odb.c b/external-odb.c\nindex 337bdd2540..93971e9ce4 100644\n--- a/external-odb.c\n+++ b/external-odb.c\n@@ -159,6 +159,10 @@ int external_odb_put_object(const void *buf, size_t len,\n \n \texternal_odb_init();\n \n+\t/* For now accept only blobs */\n+\tif (strcmp(type, \"blob\"))\n+\t\treturn 1;\n+\n \tfor (o = helpers; o; o = o->next) {\n \t\tint r = odb_helper_put_object(o, buf, len, type, sha1);\n \t\tif (r <= 0)\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335748","messageId":"20180103163403.11303-20-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 19/40] external odb: add 'put_raw_obj' support","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:42Z","receivedAt":"2018-01-03T16:36:23Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"Add support for a 'put_raw_obj' capability/instruction to send new\nobjects to an external odb. Objects will be sent as they are (in\ntheir 'raw' format). They will not be converted to Git objects.\n\nFor now any new Git object (blob, tree, commit, ...) would be sent\nif 'put_raw_obj' is supported by an odb helper. This is not a great\ndefault, but let's leave it to following commits to tweak that.\n\nSigned-off-by: Christian Couder \n---\n external-odb.c | 15 +++++++++++++++\n external-odb.h | 2 ++\n odb-helper.c | 43 ++++++++++++++++++++++++++++++++++++++-----\n odb-helper.h | 3 +++\n sha1_file.c | 2 ++\n 5 files changed, 60 insertions(+), 5 deletions(-)\n\ndiff --git a/external-odb.c b/external-odb.c\nindex 2622c12853..337bdd2540 100644\n--- a/external-odb.c\n+++ b/external-odb.c\n@@ -151,3 +151,18 @@ int external_odb_get_direct(const unsigned char *sha1)\n \n \treturn -1;\n }\n+\n+int external_odb_put_object(const void *buf, size_t len,\n+\t\t\t const char *type, unsigned char *sha1)\n+{\n+\tstruct odb_helper *o;\n+\n+\texternal_odb_init();\n+\n+\tfor (o = helpers; o; o = o->next) {\n+\t\tint r = odb_helper_put_object(o, buf, len, type, sha1);\n+\t\tif (r <= 0)\n+\t\t\treturn r;\n+\t}\n+\treturn 1;\n+}\ndiff --git a/external-odb.h b/external-odb.h\nindex fb8b94972f..26bb931685 100644\n--- a/external-odb.h\n+++ b/external-odb.h\n@@ -6,5 +6,7 @@ extern const char *external_odb_root(void);\n extern int external_odb_has_object(const unsigned char *sha1);\n extern int external_odb_get_object(const unsigned char *sha1);\n extern int external_odb_get_direct(const unsigned char *sha1);\n+extern int external_odb_put_object(const void *buf, size_t len,\n+\t\t\t\t const char *type, unsigned char *sha1);\n \n #endif /* EXTERNAL_ODB_H */\ndiff --git a/odb-helper.c b/odb-helper.c\nindex ea642fd438..6f56f07b38 100644\n--- a/odb-helper.c\n+++ b/odb-helper.c\n@@ -72,9 +72,10 @@ static void prepare_helper_command(struct argv_array *argv, const char *cmd,\n \tstrbuf_release(&buf);\n }\n \n-__attribute__((format (printf,3,4)))\n+__attribute__((format (printf,4,5)))\n static int odb_helper_start(struct odb_helper *o,\n \t\t\t struct odb_helper_cmd *cmd,\n+\t\t\t int use_stdin,\n \t\t\t const char *fmt, ...)\n {\n \tva_list ap;\n@@ -91,7 +92,10 @@ static int odb_helper_start(struct odb_helper *o,\n \n \tcmd->child.argv = cmd->argv.argv;\n \tcmd->child.use_shell = 1;\n-\tcmd->child.no_stdin = 1;\n+\tif (use_stdin)\n+\t\tcmd->child.in = -1;\n+\telse\n+\t\tcmd->child.no_stdin = 1;\n \tcmd->child.out = -1;\n \n \tif (start_command(&cmd->child) < 0) {\n@@ -120,7 +124,7 @@ int odb_helper_init(struct odb_helper *o)\n \tFILE *fh;\n \tstruct strbuf line = STRBUF_INIT;\n \n-\tif (odb_helper_start(o, &cmd, \"init\") < 0)\n+\tif (odb_helper_start(o, &cmd, 0, \"init\") < 0)\n \t\treturn -1;\n \n \tfh = xfdopen(cmd.child.out, \"r\");\n@@ -180,7 +184,7 @@ static void odb_helper_load_have(struct odb_helper *o)\n \t\treturn;\n \to->have_valid = 1;\n \n-\tif (odb_helper_start(o, &cmd, \"have\") < 0)\n+\tif (odb_helper_start(o, &cmd, 0, \"have\") < 0)\n \t\treturn;\n \n \tfh = xfdopen(cmd.child.out, \"r\");\n@@ -235,7 +239,7 @@ int odb_helper_get_object(struct odb_helper *o, const unsigned char *sha1,\n \tif (!obj)\n \t\treturn -1;\n \n-\tif (odb_helper_start(o, &cmd, \"get_git_obj %s\", sha1_to_hex(sha1)) < 0)\n+\tif (odb_helper_start(o, &cmd, 0, \"get_git_obj %s\", sha1_to_hex(sha1)) < 0)\n \t\treturn -1;\n \n \tmemset(&stream, 0, sizeof(stream));\n@@ -335,3 +339,32 @@ int odb_helper_get_direct(struct odb_helper *o,\n \n \treturn res;\n }\n+\n+int odb_helper_put_object(struct odb_helper *o,\n+\t\t\t const void *buf, size_t len,\n+\t\t\t const char *type, unsigned char *sha1)\n+{\n+\tstruct odb_helper_cmd cmd;\n+\n+\tif (odb_helper_start(o, &cmd, 1, \"put_raw_obj %s %\"PRIuMAX\" %s\",\n+\t\t\t sha1_to_hex(sha1), (uintmax_t)len, type) < 0)\n+\t\treturn -1;\n+\n+\tdo {\n+\t\tint w = xwrite(cmd.child.in, buf, len);\n+\t\tif (w < 0) {\n+\t\t\terror(\"unable to write to odb helper '%s': %s\",\n+\t\t\t o->name, strerror(errno));\n+\t\t\tclose(cmd.child.in);\n+\t\t\tclose(cmd.child.out);\n+\t\t\todb_helper_finish(o, &cmd);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tlen -= w;\n+\t} while (len > 0);\n+\n+\tclose(cmd.child.in);\n+\tclose(cmd.child.out);\n+\todb_helper_finish(o, &cmd);\n+\treturn 0;\n+}\ndiff --git a/odb-helper.h b/odb-helper.h\nindex f8eac7f44c..4a9cc7f07b 100644\n--- a/odb-helper.h\n+++ b/odb-helper.h\n@@ -46,5 +46,8 @@ extern int odb_helper_get_object(struct odb_helper *o,\n \t\t\t\t int fd);\n extern int odb_helper_get_direct(struct odb_helper *o,\n \t\t\t\t const unsigned char *sha1);\n+extern int odb_helper_put_object(struct odb_helper *o,\n+\t\t\t\t const void *buf, size_t len,\n+\t\t\t\t const char *type, unsigned char *sha1);\n \n #endif /* ODB_HELPER_H */\ndiff --git a/sha1_file.c b/sha1_file.c\nindex f1d688c10a..8c873554b2 100644\n--- a/sha1_file.c\n+++ b/sha1_file.c\n@@ -1695,6 +1695,8 @@ int write_sha1_file(const void *buf, unsigned long len, const char *type, unsign\n \t * it out into .git/objects/??/?{38} file.\n \t */\n \twrite_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);\n+\tif (!external_odb_put_object(buf, len, type, sha1))\n+\t\treturn 0;\n \tif (freshen_packed_object(sha1) || freshen_loose_object(sha1))\n \t\treturn 0;\n \treturn write_loose_object(sha1, hdr, hdrlen, buf, len, 0);\n-- \n2.16.0.rc0.16.g82191dbc6c.dirty\n\n"},{"id":"335749","messageId":"20180103163403.11303-14-chriscool@tuxfamily.org","threadId":"47531","inReplyTo":"20180103163403.11303-1-chriscool@tuxfamily.org","subject":"[PATCH 13/40] gc: do not repack promisor packfiles","fromName":"Christian Couder","fromEmail":"christian.couder@gmail.com","sentAt":"2018-01-03T16:33:36Z","receivedAt":"2018-01-03T16:36:37Z","isPatch":true,"sender":{"key":"christian.couder@gmail.com","avatar":"https://avatars.githubusercontent.com/u/208954?v=4"},"body":"From: Jonathan Tan \n\nTeach gc to stop traversal at promisor objects, and to leave promisor\npackfiles alone. This has the effect of only repacking non-promisor\npackfiles, and preserves the distinction between promisor packfiles and\nnon-promisor packfiles.\n\nSigned-off-by: Jonathan Tan \nSigned-off-by: Jeff Hostetler \nSigned-off-by: Junio C Hamano \n---\n Documentation/git-pack-objects.txt | 11 ++++++++\n builtin/gc.c | 4 +++\n builtin/pack-objects.c | 37 ++++++++++++++++++++++++--\n builtin/prune.c | 7 +++++\n builtin/repack.c | 9 +++++--\n t/t0410-partial-clone.sh | 54 ++++++++++++++++++++++++++++++++++++--\n 6 files changed, 116 insertions(+), 6 deletions(-)\n\ndiff --git a/Documentation/git-pack-objects.txt b/Documentation/git-pack-objects.txt\nindex aa403d02f3..81bc490ac5 100644\n--- a/Documentation/git-pack-objects.txt\n+++ b/Documentation/git-pack-objects.txt\n@@ -255,6 +255,17 @@ a missing object is encountered. This is the default action.\n The form '--missing=allow-any' will allow object traversal to continue\n if a missing object is encountered. Missing objects will silently be\n omitted from the results.\n++\n+The form '--missing=allow-promisor' is like 'allow-any', but will only\n+allow object traversal to continue for EXPECTED promisor missing objects.\n+Unexpected missing object will raise an error.\n+\n+--exclude-promisor-objects::\n+\tOmit objects that are known to be in the promisor remote. (This\n+\toption has the purpose of operating only on locally created objects,\n+\tso that when we repack, we still maintain a distinction between\n+\tlocally created objects [without .promisor] and objects from the\n+\tpromisor remote [with .promisor].) This is used with partial clone.\n \n SEE ALSO\n --------\ndiff --git a/builtin/gc.c b/builtin/gc.c\nindex 3c5eae0edf..cef1461d1a 100644\n--- a/builtin/gc.c\n+++ b/builtin/gc.c\n@@ -20,6 +20,7 @@\n #include \"argv-array.h\"\n #include \"commit.h\"\n #include \"packfile.h\"\n+#include \"external-odb.h\"\n \n #define FAILED_RUN \"failed to run %s\"\n \n@@ -458,6 +459,9 @@ int cmd_gc(int argc, const char **argv, const char *prefix)\n \t\t\targv_array_push(&prune, prune_expire);\n \t\t\tif (quiet)\n \t\t\t\targv_array_push(&prune, \"--no-progress\");\n+\t\t\tif (has_external_odb())\n+\t\t\t\targv_array_push(&prune,\n+\t\t\t\t\t\t\"--exclude-promisor-objects\");\n \t\t\tif (run_command_v_opt(prune.argv, RUN_GIT_CMD))\n \t\t\t\treturn error(FAILED_RUN, prune.argv[0]);\n \t\t}\ndiff --git a/builtin/pack-objects.c b/builtin/pack-objects.c\nindex 6b9cfc289d..6c71552cdf 100644\n--- a/builtin/pack-objects.c\n+++ b/builtin/pack-objects.c\n@@ -75,6 +75,8 @@ static int use_bitmap_index = -1;\n static int write_bitmap_index;\n static uint16_t write_bitmap_options;\n \n+static int exclude_promisor_objects;\n+\n static unsigned long delta_cache_size = 0;\n static unsigned long max_delta_cache_size = 256 * 1024 * 1024;\n static unsigned long cache_max_small_delta_size = 1000;\n@@ -84,8 +86,9 @@ static unsigned long window_memory_limit = 0;\n static struct list_objects_filter_options filter_options;\n \n enum missing_action {\n-\tMA_ERROR = 0, /* fail if any missing objects are encountered */\n-\tMA_ALLOW_ANY, /* silently allow ALL missing objects */\n+\tMA_ERROR = 0, /* fail if any missing objects are encountered */\n+\tMA_ALLOW_ANY, /* silently allow ALL missing objects */\n+\tMA_ALLOW_PROMISOR, /* silently allow all missing PROMISOR objects */\n };\n static enum missing_action arg_missing_action;\n static show_object_fn fn_show_object;\n@@ -2578,6 +2581,20 @@ static void show_object__ma_allow_any(struct object *obj, const char *name, void\n \tshow_object(obj, name, data);\n }\n \n+static void show_object__ma_allow_promisor(struct object *obj, const char *name, void *data)\n+{\n+\tassert(arg_missing_action == MA_ALLOW_PROMISOR);\n+\n+\t/*\n+\t * Quietly ignore EXPECTED missing objects. This avoids problems with\n+\t * staging them now and getting an odd error later.\n+\t */\n+\tif (!has_object_file(&obj->oid) && is_promisor_object(&obj->oid))\n+\t\treturn;\n+\n+\tshow_object(obj, name, data);\n+}\n+\n static int option_parse_missing_action(const struct option *opt,\n \t\t\t\t const char *arg, int unset)\n {\n@@ -2592,10 +2609,18 @@ static int option_parse_missing_action(const struct option *opt,\n \n \tif (!strcmp(arg, \"allow-any\")) {\n \t\targ_missing_action = MA_ALLOW_ANY;\n+\t\tfetch_if_missing = 0;\n \t\tfn_show_object = show_object__ma_allow_any;\n \t\treturn 0;\n \t}\n \n+\tif (!strcmp(arg, \"allow-promisor\")) {\n+\t\targ_missing_action = MA_ALLOW_PROMISOR;\n+\t\tfetch_if_missing = 0;\n+\t\tfn_show_object = show_object__ma_allow_promisor;\n+\t\treturn 0;\n+\t}\n+\n \tdie(_(\"invalid value for --missing\"));\n \treturn 0;\n }\n@@ -3009,6 +3034,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)\n \t\t{ OPTION_CALLBACK, 0, \"missing\", NULL, N_(\"action\"),\n \t\t N_(\"handling for missing objects\"), PARSE_OPT_NONEG,\n \t\t option_parse_missing_action },\n+\t\tOPT_BOOL(0, \"exclude-promisor-objects\", &exclude_promisor_objects,\n+\t\t\t N_(\"do not pack objects in promisor packfiles\")),\n \t\tOPT_END(),\n \t};\n \n@@ -3054,6 +3081,12 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)\n \t\targv_array_push(&rp, \"--unpacked\");\n \t}\n \n+\tif (exclude_promisor_objects) {\n+\t\tuse_internal_rev_list = 1;\n+\t\tfetch_if_missing = 0;\n+\t\targv_array_push(&rp, \"--exclude-promisor-objects\");\n+\t}\n+\n \tif (!reuse_object)\n \t\treuse_delta = 0;\n \tif (pack_compression_level == -1)\ndiff --git a/builtin/prune.c b/builtin/prune.c\nindex d2fdae680a..4cfec82f40 100644\n--- a/builtin/prune.c\n+++ b/builtin/prune.c\n@@ -101,12 +101,15 @@ int cmd_prune(int argc, const char **argv, const char *prefix)\n {\n \tstruct rev_info revs;\n \tstruct progress *progress = NULL;\n+\tint exclude_promisor_objects = 0;\n \tconst struct option options[] = {\n \t\tOPT__DRY_RUN(&show_only, N_(\"do not remove, show only\")),\n \t\tOPT__VERBOSE(&verbose, N_(\"report pruned objects\")),\n \t\tOPT_BOOL(0, \"progress\", &show_progress, N_(\"show progress\")),\n \t\tOPT_EXPIRY_DATE(0, \"expire\", &expire,\n \t\t\t\tN_(\"expire objects older than