{"thread":{"id":"20431","subject":"[PATCH 4/5] administrative functions for rev-cache, and start of integration into git","startedAt":"2009-08-06T09:55:43Z","lastAt":"2009-08-06T09:55:43Z","messageCount":1,"participants":["Nick Edelen"],"isPatch":true,"patchVersion":1,"patchTotal":5},"messages":[{"id":"119768","messageId":"op.ux8i65metdk399@sirnot.private","threadId":"20431","inReplyTo":null,"subject":"[PATCH 4/5] administrative functions for rev-cache, and start of integration into git","fromName":"Nick Edelen","fromEmail":"sirnot@gmail.com","sentAt":"2009-08-06T09:55:43Z","receivedAt":"2009-08-06T09:55:43Z","isPatch":true,"sender":{"key":"sirnot@gmail.com","avatar":null},"body":"This patch, fourth, contains miscellaneous (maintenance) features:\n - support for cache slice fusion, index regeneration and object size caching\n - extra 'size' field added to commit, tag and blob objects, initialized in \n   parse_* functions and cache traversal\n - non-commit object generation refactored to take advantage of 'size' field\n - porcelain updated to support feature additions\n\nThe beginnings of integration into git are present in this patch, mainly \ncentered on caching object size; the object generation is refactored to more \nelegantly exploit this.  Fusion allows smaller (incremental) slices to be \ncoagulated into a larger slice, reducing overhead, while index regeneration \nenables repair or cleaning of the cache index.\n\nNote that tests for these features are included in the following patch, as they \ntake advantage of the rev-cache's integration into the revision walker.\n\nSigned-off-by: Nick Edelen <sirnot@gmail.com>\n\n---\n blob.c              |    1 +\n blob.h              |    1 +\n builtin-rev-cache.c |   49 ++++++-\n commit.c            |    1 +\n commit.h            |    1 +\n rev-cache.c         |  416 ++++++++++++++++++++++++++++++++++++++++++---------\n revision.h          |    4 +-\n 7 files changed, 397 insertions(+), 76 deletions(-)\n\ndiff --git a/blob.c b/blob.c\nindex bd7d078..e1bab3f 100644\n--- a/blob.c\n+++ b/blob.c\n@@ -21,6 +21,7 @@ struct blob *lookup_blob(const unsigned char *sha1)\n int parse_blob_buffer(struct blob *item, void *buffer, unsigned long size)\n {\n \titem->object.parsed = 1;\n+\titem->size = size;\n \treturn 0;\n }\n \ndiff --git a/blob.h b/blob.h\nindex ea5d9e9..d8db96a 100644\n--- a/blob.h\n+++ b/blob.h\n@@ -7,6 +7,7 @@ extern const char *blob_type;\n \n struct blob {\n \tstruct object object;\n+\tunsigned long size;\n };\n \n struct blob *lookup_blob(const unsigned char *sha1);\ndiff --git a/builtin-rev-cache.c b/builtin-rev-cache.c\nindex 2338871..44916f5 100755\n--- a/builtin-rev-cache.c\n+++ b/builtin-rev-cache.c\n@@ -4,6 +4,8 @@\n #include \"diff.h\"\n #include \"revision.h\"\n \n+#define DEFAULT_IGNORE_SLICE_SIZE\t\t5000000 /* in bytes */\n+\n /* porcelain for rev-cache.c */\n static int handle_add(int argc, const char *argv[]) /* args beyond this command */\n {\n@@ -21,7 +23,7 @@ static int handle_add(int argc, const char *argv[]) /* args beyond this command\n \t\tif (!strcmp(argv[i], \"--stdin\"))\n \t\t\tdostdin = 1;\n \t\telse if (!strcmp(argv[i], \"--fresh\"))\n-\t\t\tstarts_from_slices(&revs, UNINTERESTING);\n+\t\t\tstarts_from_slices(&revs, UNINTERESTING, 0, 0);\n \t\telse if (!strcmp(argv[i], \"--not\"))\n \t\t\tflags ^= UNINTERESTING;\n \t\telse if (!strcmp(argv[i], \"--legs\"))\n@@ -141,6 +143,47 @@ static int handle_walk(int argc, const char *argv[])\n \treturn 0;\n }\n \n+static int handle_fuse(int argc, const char *argv[])\n+{\n+\tstruct rev_info revs;\n+\tstruct rev_cache_info rci;\n+\tconst char *args[5];\n+\tint i, argn = 0;\n+\tchar add_all = 0;\n+\t\n+\tinit_revisions(&revs, 0);\n+\tinit_rci(&rci);\n+\targs[argn++] = \"rev-list\";\n+\t\n+\tfor (i = 0; i < argc; i++) {\n+\t\tif (!strcmp(argv[i], \"--all\")) {\n+\t\t\targs[argn++] = \"--all\";\n+\t\t\tsetup_revisions(argn, args, &revs, 0);\n+\t\t\tadd_all = 1;\n+\t\t} else if (!strcmp(argv[i], \"--noobjects\")) \n+\t\t\trci.objects = 0;\n+\t\telse if (!strncmp(argv[i], \"--ignore-size\", 13)) {\n+\t\t\tint sz = DEFAULT_IGNORE_SLICE_SIZE;\n+\t\t\t\n+\t\t\tif (argv[i][13] == '=')\n+\t\t\t\tsz = atoi(argv[i] + 14);\n+\t\t\t\n+\t\t\trci.ignore_size = sz;\n+\t\t} else \n+\t\t\tcontinue;\n+\t}\n+\t\n+\tif (!add_all)\n+\t\tstarts_from_slices(&revs, 0, 0, 0);\n+\t\n+\treturn coagulate_cache_slices(&rci, &revs);\n+}\n+\n+static int handle_index(int argc, const char *argv[])\n+{\n+\treturn regenerate_cache_index(0);\n+}\n+\n static int handle_help(void)\n {\n \tchar *usage = \"\\\n@@ -186,8 +229,12 @@ int cmd_rev_cache(int argc, const char *argv[], const char *prefix)\n \targv += 2;\n \tif (!strcmp(arg, \"add\"))\n \t\tr = handle_add(argc, argv);\n+\telse if (!strcmp(arg, \"fuse\"))\n+\t\tr = handle_fuse(argc, argv);\n \telse if (!strcmp(arg, \"walk\"))\n \t\tr = handle_walk(argc, argv);\n+\telse if (!strcmp(arg, \"index\"))\n+\t\tr = handle_index(argc, argv);\n \telse\n \t\treturn handle_help();\n \t\ndiff --git a/commit.c b/commit.c\nindex 65d223e..afe504f 100644\n--- a/commit.c\n+++ b/commit.c\n@@ -243,6 +243,7 @@ int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)\n \tif (item->object.parsed)\n \t\treturn 0;\n \titem->object.parsed = 1;\n+\titem->size = size;\n \ttail += size;\n \tif (tail <= bufptr + 46 || memcmp(bufptr, \"tree \", 5) || bufptr[45] != '\\n')\n \t\treturn error(\"bogus commit object %s\", sha1_to_hex(item->object.sha1));\ndiff --git a/commit.h b/commit.h\nindex ba9f638..0529fcf 100644\n--- a/commit.h\n+++ b/commit.h\n@@ -19,6 +19,7 @@ struct commit {\n \tstruct commit_list *parents;\n \tstruct tree *tree;\n \tchar *buffer;\n+\tunsigned long size;\n };\n \n extern int save_commit_buffer;\ndiff --git a/rev-cache.c b/rev-cache.c\nindex cd618cf..4cda858 100755\n--- a/rev-cache.c\n+++ b/rev-cache.c\n@@ -8,6 +8,7 @@\n #include \"diff.h\"\n #include \"revision.h\"\n #include \"run-command.h\"\n+#include \"string-list.h\"\n \n \n /* single index maps objects to cache files */\n@@ -88,6 +89,7 @@ static struct strbuf *g_buffer;\n #define IE_CAST(p)\t((struct index_entry *)(p))\n \n #define ACTUAL_OBJECT_ENTRY_SIZE(e)\t\t(OE_SIZE + PATH_SIZE((e)->merge_nr + (e)->split_nr) + (e)->size_size)\n+#define ENTRY_SIZE_OFFSET(e)\t\t\t(ACTUAL_OBJECT_ENTRY_SIZE(e) - (e)->size_size)\n \n #define SLOP\t\t5\n \n@@ -219,7 +221,6 @@ unsigned char *get_cache_slice(struct commit *commit)\n \t\treturn 0;\n \t\n \tie = search_index(commit->object.sha1);\n-\t\n \tif (ie && ie->cache_index < idx_head.cache_nr)\n \t\treturn idx_caches + ie->cache_index * 20;\n \t\n@@ -229,27 +230,46 @@ unsigned char *get_cache_slice(struct commit *commit)\n \n /* traversal */\n \n-static void handle_noncommit(struct rev_info *revs, struct object_entry *entry)\n+static unsigned long decode_size(unsigned char *str, int len);\n+\n+static void handle_noncommit(struct rev_info *revs, struct commit *commit, struct object_entry *entry)\n {\n-\tstruct object *obj = 0;\n+\tstruct blob *blob;\n+\tstruct tree *tree;\n+\tstruct object *obj;\n+\tunsigned long size;\n \t\n+\tsize = decode_size((unsigned char *)entry + ENTRY_SIZE_OFFSET(entry), entry->size_size);\n \tswitch (entry->type) {\n \tcase OBJ_TREE : \n-\t\tif (revs->tree_objects)\n-\t\t\tobj = (struct object *)lookup_tree(entry->sha1);\n+\t\tif (!revs->tree_objects)\n+\t\t\treturn;\n+\t\t\n+\t\ttree = lookup_tree(entry->sha1);\n+\t\tif (!tree)\n+\t\t\treturn;\n+\t\t\n+\t\ttree->size = size;\n+\t\tcommit->tree = tree;\n+\t\tobj = (struct object *)tree;\n \t\tbreak;\n+\t\n \tcase OBJ_BLOB : \n-\t\tif (revs->blob_objects)\n-\t\t\tobj = (struct object *)lookup_blob(entry->sha1);\n-\t\tbreak;\n-\tcase OBJ_TAG : \n-\t\tif (revs->tag_objects)\n-\t\t\tobj = (struct object *)lookup_tag(entry->sha1);\n+\t\tif (!revs->blob_objects)\n+\t\t\treturn;\n+\t\t\n+\t\tblob = lookup_blob(entry->sha1);\n+\t\tif (!blob)\n+\t\t\treturn;\n+\t\t\n+\t\tblob->size = size;\n+\t\tobj = (struct object *)blob;\n \t\tbreak;\n-\t}\n-\t\n-\tif (!obj)\n+\t\t\n+\tdefault : \n+\t\t/* tag objects aren't really supposed to be here */\n \t\treturn;\n+\t}\n \t\n \tobj->flags |= FACE_VALUE;\n \tadd_pending_object(revs, obj, \"\");\n@@ -262,7 +282,6 @@ static int setup_traversal(struct cache_slice_header *head, unsigned char *map,\n \tstruct commit_list *prev, *wp, **wpp;\n \tint retval;\n \t\n-\t/* printf(\"adding %s\\n\", sha1_to_hex(commit->object.sha1)); */\n \tiep = search_index(commit->object.sha1);\n \toep = OE_CAST(map + ntohl(iep->pos));\n \toep->include = 1;\n@@ -338,7 +357,7 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char\n \t\t/* add extra objects if necessary */\n \t\tif (entry->type != OBJ_COMMIT) {\n \t\t\tif (consume_children)\n-\t\t\t\thandle_noncommit(revs, entry);\n+\t\t\t\thandle_noncommit(revs, co, entry);\n \t\t\t\n \t\t\tcontinue;\n \t\t} else\n@@ -372,6 +391,8 @@ static int traverse_cache_slice_1(struct cache_slice_header *head, unsigned char\n \t\t\tif (last_objects[path]) {\n \t\t\t\tparse_commit(last_objects[path]);\n \t\t\t\t\n+\t\t\t\t/* we needn't worry about the unique field; that will be valid as \n+\t\t\t\t * long as we're not a end entry */\n \t\t\t\tlast_objects[path]->object.flags &= ~FACE_VALUE;\n \t\t\t\tlast_objects[path] = 0;\n \t\t\t}\n@@ -554,6 +575,68 @@ end:\n \n /* generation */\n \n+static int is_endpoint(struct commit *commit)\n+{\n+\tstruct commit_list *list = commit->parents;\n+\t\n+\twhile (list) {\n+\t\tif (!(list->item->object.flags & UNINTERESTING))\n+\t\t\treturn 0;\n+\t\t\n+\t\tlist = list->next;\n+\t}\n+\t\n+\treturn 1;\n+}\n+\n+/* ensures branch is self-contained: parents are either all interesting or all uninteresting */\n+static void make_legs(struct rev_info *revs)\n+{\n+\tstruct commit_list *list, **plist;\n+\tint total = 0;\n+\t\n+\t/* attach plist to end of commits list */\n+\tlist = revs->commits;\n+\twhile (list && list->next)\n+\t\tlist = list->next;\n+\t\n+\tif (list)\n+\t\tplist = &list->next;\n+\telse\n+\t\treturn;\n+\t\n+\t/* duplicates don't matter, as get_revision() ignores them */\n+\tfor (list = revs->commits; list; list = list->next) {\n+\t\tstruct commit *item = list->item;\n+\t\tstruct commit_list *parents = item->parents;\n+\t\t\n+\t\tif (item->object.flags & UNINTERESTING)\n+\t\t\tcontinue;\n+\t\tif (is_endpoint(item))\n+\t\t\tcontinue;\n+\t\t\n+\t\twhile (parents) {\n+\t\t\tstruct commit *p = parents->item;\n+\t\t\tparents = parents->next;\n+\t\t\t\n+\t\t\tif (!(p->object.flags & UNINTERESTING))\n+\t\t\t\tcontinue;\n+\t\t\t\n+\t\t\tp->object.flags &= ~UNINTERESTING;\n+\t\t\tparse_commit(p);\n+\t\t\tplist = &commit_list_insert(p, plist)->next;\n+\t\t\t\n+\t\t\tif (!(p->object.flags & SEEN))\n+\t\t\t\ttotal++;\n+\t\t}\n+\t}\n+\t\n+\tif (total)\n+\t\tsort_in_topological_order(&revs->commits, 1);\n+\t\n+}\n+\n+\n struct path_track {\n \tstruct commit *commit;\n \tint path; /* for decrementing paths */\n@@ -768,31 +851,97 @@ static void handle_paths(struct commit *commit, struct object_entry *object, str\n }\n \n \n-static void add_object_entry(const unsigned char *sha1, int type, struct object_entry *nothisone, \n+static int encode_size(unsigned long size, unsigned char *out)\n+{\n+\tint len = 0;\n+\t\n+\twhile (size) {\n+\t\t*out++ = (unsigned char)(size & 0xff);\n+\t\tsize >>= 8;\n+\t\tlen++;\n+\t}\n+\t\n+\treturn len;\n+}\n+\n+static unsigned long decode_size(unsigned char *str, int len)\n+{\n+\tunsigned long size = 0;\n+\tint shift = 0;\n+\t\n+\twhile (len--) {\n+\t\tsize |= (unsigned long)*str << shift;\n+\t\tshift += 8;\n+\t\tstr++;\n+\t}\n+\t\n+\treturn size;\n+}\n+\n+static void add_object_entry(const unsigned char *sha1, struct object_entry *entryp, \n \tstruct strbuf *merge_str, struct strbuf *split_str)\n {\n-\tstruct object_entry object;\n+\tstruct object_entry entry;\n+\tstruct object *obj;\n+\tunsigned char size_str[7];\n+\tunsigned long size;\n+\tenum object_type type;\n+\t\n+\tif (entryp)\n+\t\tsha1 = entryp->sha1;\n+\t\n+\tobj = lookup_object(sha1);\n+\tif (obj) {\n+\t\t/* it'd be smoother to have the size in the object... */\n+\t\tswitch (obj->type) {\n+\t\tcase OBJ_COMMIT : \n+\t\t\tsize = ((struct commit *)obj)->size;\n+\t\t\tbreak;\n+\t\tcase OBJ_TREE : \n+\t\t\tsize = ((struct tree *)obj)->size;\n+\t\t\tbreak;\n+\t\tcase OBJ_BLOB : \n+\t\t\tsize = ((struct blob *)obj)->size;\n+\t\t\tbreak;\n+\t\tdefault : \n+\t\t\t/* tags are potentially dynamic metadata; they don't really belong here */\n+\t\t\treturn;\n+\t\t}\n+\t\t\n+\t\ttype = obj->type;\n+\t}\n \t\n-\tif (!nothisone) {\n-\t\tmemset(&object, 0, sizeof(object));\n-\t\thashcpy(object.sha1, sha1);\n-\t\tobject.type = type;\n+\tif (!obj || !size) {\n+\t\tvoid *data = read_sha1_file(sha1, &type, &size);\n+\t\t\n+\t\tif (data)\n+\t\t\tfree(data);\n+\t}\n+\t\n+\tif (!entryp) {\n+\t\tmemset(&entry, 0, sizeof(entry));\n+\t\thashcpy(entry.sha1, sha1);\n+\t\tentry.type = type;\n \t\t\n \t\tif (merge_str)\n-\t\t\tobject.merge_nr = merge_str->len / PATH_WIDTH;\n+\t\t\tentry.merge_nr = merge_str->len / PATH_WIDTH;\n \t\tif (split_str)\n-\t\t\tobject.split_nr = split_str->len / PATH_WIDTH;\n+\t\t\tentry.split_nr = split_str->len / PATH_WIDTH;\n \t\t\n-\t\tnothisone = &object;\n+\t\tentryp = &entry;\n \t}\n \t\n-\tstrbuf_add(g_buffer, nothisone, sizeof(object));\n+\tentryp->size_size = encode_size(size, size_str);\n+\t\n+\t/* write the muvabitch */\n+\tstrbuf_add(g_buffer, entryp, sizeof(entry));\n \t\n-\tif (merge_str && merge_str->len)\n+\tif (merge_str)\n \t\tstrbuf_add(g_buffer, merge_str->buf, merge_str->len);\n-\tif (split_str && split_str->len)\n+\tif (split_str)\n \t\tstrbuf_add(g_buffer, split_str->buf, split_str->len);\n \t\n+\tstrbuf_add(g_buffer, size_str, entryp->size_size);\n }\n \n /* returns non-zero to continue parsing, 0 to skip */\n@@ -836,12 +985,7 @@ continue_loop:\n \n static int dump_tree_callback(const unsigned char *sha1, const char *path, unsigned int mode)\n {\n-\tunsigned char data[21];\n-\t\n-\thashcpy(data, sha1);\n-\tdata[20] = !!S_ISDIR(mode);\n-\t\n-\tstrbuf_add(g_buffer, data, 21);\n+\tstrbuf_add(g_buffer, sha1, 20);\n \t\n \treturn 1;\n }\n@@ -851,15 +995,10 @@ static void tree_addremove(struct diff_options *options,\n \tconst unsigned char *sha1,\n \tconst char *concatpath)\n {\n-\tunsigned char data[21];\n-\t\n \tif (whatnow != '+')\n \t\treturn;\n \t\n-\thashcpy(data, sha1);\n-\tdata[20] = !!S_ISDIR(mode);\n-\t\n-\tstrbuf_add(g_buffer, data, 21);\n+\tstrbuf_add(g_buffer, sha1, 20);\n }\n \n static void tree_change(struct diff_options *options,\n@@ -868,26 +1007,10 @@ static void tree_change(struct diff_options *options,\n \tconst unsigned char *new_sha1,\n \tconst char *concatpath)\n {\n-\tunsigned char data[21];\n-\t\n \tif (!hashcmp(old_sha1, new_sha1))\n \t\treturn;\n \t\n-\thashcpy(data, new_sha1);\n-\tdata[20] = !!S_ISDIR(new_mode);\n-\t\n-\tstrbuf_add(g_buffer, data, 21);\n-}\n-\n-static int sort_type_hash(const void *a, const void *b)\n-{\n-\tconst unsigned char *sa = (const unsigned char *)a, \n-\t\t*sb = (const unsigned char *)b;\n-\t\n-\tif (sa[20] == sb[20])\n-\t\treturn hashcmp(sa, sb);\n-\t\n-\treturn sa[20] > sb[20] ? -1 : 1;\n+\tstrbuf_add(g_buffer, new_sha1, 20);\n }\n \n static int add_unique_objects(struct commit *commit)\n@@ -898,6 +1021,7 @@ static int add_unique_objects(struct commit *commit)\n \tint i, j, next;\n \tchar is_first = 1;\n \t\n+\t/* ...no, calculate unique objects */\n \tstrbuf_init(&os, 0);\n \tstrbuf_init(&ost, 0);\n \torig_buf = g_buffer;\n@@ -917,20 +1041,20 @@ static int add_unique_objects(struct commit *commit)\n \t\t\n \t\tstrbuf_setlen(g_buffer, 0);\n \t\tdiff_tree_sha1(list->item->tree->object.sha1, commit->tree->object.sha1, \"\", &opts);\n-\t\tqsort(g_buffer->buf, g_buffer->len / 21, 21, (int (*)(const void *, const void *))hashcmp);\n+\t\tqsort(g_buffer->buf, g_buffer->len / 20, 20, (int (*)(const void *, const void *))hashcmp);\n \t\t\n \t\t/* take intersection */\n \t\tif (!is_first) {\n-\t\t\tfor (next = i = j = 0; i < os.len; i += 21) {\n+\t\t\tfor (next = i = j = 0; i < os.len; i += 20) {\n \t\t\t\twhile (j < ost.len && hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)) < 0)\n-\t\t\t\t\tj += 21;\n+\t\t\t\t\tj += 20;\n \t\t\t\t\n \t\t\t\tif (j >= ost.len || hashcmp((unsigned char *)(ost.buf + j), (unsigned char *)(os.buf + i)))\n \t\t\t\t\tcontinue;\n \t\t\t\t\n \t\t\t\tif (next != i)\n-\t\t\t\t\tmemcpy(os.buf + next, os.buf + i, 21);\n-\t\t\t\tnext += 21;\n+\t\t\t\t\tmemcpy(os.buf + next, os.buf + i, 20);\n+\t\t\t\tnext += 20;\n \t\t\t}\n \t\t\t\n \t\t\tif (next != i)\n@@ -939,22 +1063,21 @@ static int add_unique_objects(struct commit *commit)\n \t\t\tis_first = 0;\n \t}\n \t\n+\t/* no parents (!) */\n \tif (is_first) {\n \t\tg_buffer = &os;\n \t\tdump_tree(commit->tree, dump_tree_callback);\n \t}\n \t\n-\tif (os.len)\n-\t\tqsort(os.buf, os.len / 21, 21, sort_type_hash);\n-\t\n+\t/* the ordering of non-commit objects dosn't really matter, so we're not gonna bother */\n \tg_buffer = orig_buf;\n-\tfor (i = 0; i < os.len; i += 21)\n-\t\tadd_object_entry((unsigned char *)(os.buf + i), os.buf[i + 20] ? OBJ_TREE : OBJ_BLOB, 0, 0, 0);\n+\tfor (i = 0; i < os.len; i += 20)\n+\t\tadd_object_entry((unsigned char *)(os.buf + i), 0, 0, 0);\n \t\n \tstrbuf_release(&ost);\n \tstrbuf_release(&os);\n \t\n-\treturn 0;\n+\treturn i / 20;\n }\n \n static void init_revcache_directory(void)\n@@ -1046,6 +1169,9 @@ int make_cache_slice(struct rev_cache_info *rci,\n \tif (prepare_revision_walk(revs))\n \t\tdie(\"died preparing revision walk\");\n \t\n+\tif (rci->legs)\n+\t\tmake_legs(revs);\n+\t\n \tobject_nr = total_sz = 0;\n \twhile ((commit = get_revision(revs)) != 0) {\n \t\tstruct object_entry object;\n@@ -1067,12 +1193,12 @@ int make_cache_slice(struct rev_cache_info *rci,\n \t\t\n \t\tcommit->indegree = 0;\n \t\t\n-\t\tadd_object_entry(0, 0, &object, &merge_paths, &split_paths);\n+\t\tadd_object_entry(0, &object, &merge_paths, &split_paths);\n \t\tobject_nr++;\n \t\t\n-\t\tif (!(commit->object.flags & TREESAME)) {\n+\t\tif (rci->objects && !(commit->object.flags & TREESAME)) {\n \t\t\t/* add all unique children for this commit */\n-\t\t\tadd_object_entry(commit->tree->object.sha1, OBJ_TREE, 0, 0, 0);\n+\t\t\tadd_object_entry(commit->tree->object.sha1, 0, 0, 0);\n \t\t\tobject_nr++;\n \t\t\t\n \t\t\tif (!object.is_end)\n@@ -1260,7 +1386,7 @@ int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,\n \t\t * should we allow more intelligent overriding? */\n \t\tdate = ntohl(object_entry->date);\n \t\tif (date > idx_head.max_date) {\n- \t\t\tentry = 0;\n+\t\t\tentry = 0;\n \t\t\tif (date > max_date)\n \t\t\t\tmax_date = date;\n \t\t} else\n@@ -1316,8 +1442,7 @@ int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1,\n }\n \n \n-/* add start-commits from each cache slice (uninterestingness will be propogated) */\n-void starts_from_slices(struct rev_info *revs, unsigned int flags)\n+void starts_from_slices(struct rev_info *revs, unsigned int flags, unsigned char *which, int n)\n {\n \tstruct commit *commit;\n \tint i;\n@@ -1334,6 +1459,18 @@ void starts_from_slices(struct rev_info *revs, unsigned int flags)\n \t\tif (!entry->is_start)\n \t\t\tcontinue;\n \t\t\n+\t\t/* only include entries in 'which' slices */\n+\t\tif (n) {\n+\t\t\tint j;\n+\t\t\t\n+\t\t\tfor (j = 0; j < n; j++)\n+\t\t\t\tif (!hashcmp(idx_caches + entry->cache_index * 20, which + j * 20))\n+\t\t\t\t\tbreak;\n+\t\t\t\n+\t\t\tif (j == n)\n+\t\t\t\tcontinue;\n+\t\t}\n+\t\t\n \t\tcommit = lookup_commit(entry->sha1);\n \t\tif (!commit)\n \t\t\tcontinue;\n@@ -1343,3 +1480,134 @@ void starts_from_slices(struct rev_info *revs, unsigned int flags)\n \t}\n \t\n }\n+\n+\n+/* the most work-intensive attributes in the cache are the unique objects and size, both \n+ * of which can be re-used.  although path structures will be isomorphic, path generation is \n+ * not particularly expensive, and at any rate we need to re-sort the commits */\n+int coagulate_cache_slices(struct rev_cache_info *rci, struct rev_info *revs)\n+{\n+\tunsigned char cache_sha1[20];\n+\tchar base[PATH_MAX];\n+\tint fd, baselen, i;\n+\tstruct stat fi;\n+\tstruct string_list files = {0, 0, 0, 1}; /* dup */\n+\tstruct strbuf ignore;\n+\tDIR *dirh;\n+\t\n+\tstrbuf_init(&ignore, 0);\n+\tstrncpy(base, git_path(\"rev-cache\"), sizeof(base));\n+\tbaselen = strlen(base);\n+\t\n+\t/* enumerate files */\n+\tdirh = opendir(base);\n+\tif (dirh) {\n+\t\tstruct dirent *de;\n+\t\t\n+\t\twhile ((de = readdir(dirh))) {\n+\t\t\tif (de->d_name[0] == '.')\n+\t\t\t\tcontinue;\n+\t\t\t\n+\t\t\tbase[baselen] = '/';\n+\t\t\tstrncpy(base + baselen + 1, de->d_name, sizeof(base) - baselen - 1);\n+\t\t\t\n+\t\t\t/* _theoretically_ it is possible a slice < ignore_size to map objects not covered by, yet reachable from, \n+\t\t\t * a slice >= ignore_size, meaning that we could potentially delete an 'unfused' slice; but if that \n+\t\t\t * ever *did* happen their cache structure'd be so fucked up they might as well refuse the entire thing.\n+\t\t\t * and at any rate the worst it'd do is make rev-list revert to standard walking in that (small) bit.\n+\t\t\t */\n+\t\t\tif (rci->ignore_size) {\n+\t\t\t\tunsigned char sha1[20];\n+\t\t\t\t\n+\t\t\t\tif (stat(base, &fi))\n+\t\t\t\t\twarning(\"can't query file %s\\n\", base);\n+\t\t\t\telse if (fi.st_size >= rci->ignore_size && !get_sha1_hex(de->d_name, sha1)) {\n+\t\t\t\t\tstrbuf_add(&ignore, sha1, 20);\n+\t\t\t\t\tcontinue;\n+\t\t\t\t}\n+\t\t\t}\n+\t\t\t\n+\t\t\tstring_list_insert(base, &files);\n+\t\t}\n+\t\t\n+\t\tclosedir(dirh);\n+\t}\n+\t\n+\tif (ignore.len) {\n+\t\tstarts_from_slices(revs, UNINTERESTING, (unsigned char *)ignore.buf, ignore.len / 20);\n+\t\tstrbuf_release(&ignore);\n+\t}\n+\t\n+\trci->make_index = 0;\n+\tif (make_cache_slice(rci, revs, 0, 0, cache_sha1) < 0)\n+\t\tdie(\"can't make cache slice\");\n+\t\n+\t/* clean up time! */\n+\tcleanup_cache_slices();\n+\t\n+\tfor (i = 0; i < files.nr; i++) {\n+\t\tchar *name = files.items[i].string;\n+\t\t\n+\t\tfprintf(stderr, \"removing %s\\n\", name);\n+\t\tunlink_or_warn(name);\n+\t}\n+\t\n+\tstring_list_clear(&files, 0);\n+\t\n+\tfd = open(git_path(\"rev-cache/%s\", sha1_to_hex(cache_sha1)), O_RDWR);\n+\tif (fd < 0 || fstat(fd, &fi))\n+\t\tdie(\"what?  I can't open/query the cache I just generated\");\n+\t\n+\tif (make_cache_index(rci, cache_sha1, fd, fi.st_size) < 0)\n+\t\tdie(\"can't make new index\");\n+\t\n+\tclose(fd);\n+\t\n+\treturn 0;\n+}\n+\n+int regenerate_cache_index(struct rev_cache_info *rci)\n+{\n+\tDIR *dirh;\n+\tchar base[PATH_MAX];\n+\tint baselen;\n+\t\n+\t/* first remove old index if it exists */\n+\tunlink_or_warn(git_path(\"rev-cache/index\"));\n+\t\n+\tstrncpy(base, git_path(\"rev-cache\"), sizeof(base));\n+\tbaselen = strlen(base);\n+\t\n+\tdirh = opendir(base);\n+\tif (dirh) {\n+\t\tstruct dirent *de;\n+\t\tstruct stat fi;\n+\t\tint fd;\n+\t\tunsigned char sha1[20];\n+\t\t\n+\t\twhile ((de = readdir(dirh))) {\n+\t\t\tif (de->d_name[0] == '.')\n+\t\t\t\tcontinue;\n+\t\t\t\n+\t\t\tif (get_sha1_hex(de->d_name, sha1))\n+\t\t\t\tcontinue;\n+\t\t\t\n+\t\t\tbase[baselen] = '/';\n+\t\t\tstrncpy(base + baselen + 1, de->d_name, sizeof(base) - baselen - 1);\n+\t\t\t\n+\t\t\t/* open with RDWR because of mmap call in make_cache_index() */\n+\t\t\tfd = open(base, O_RDWR);\n+\t\t\tif (fd < 0 || fstat(fd, &fi))\n+\t\t\t\twarning(\"bad cache found [%s]; fuse recommended\", de->d_name);\n+\t\t\t\n+\t\t\tif (make_cache_index(rci, sha1, fd, fi.st_size) < 0)\n+\t\t\t\tdie(\"error writing cache\");\n+\t\t\t\n+\t\t\tclose(fd);\n+\t\t}\n+\t\t\n+\t\tclosedir(dirh);\n+\t}\n+\t\n+\treturn 0;\n+}\n\\ No newline at end of file\ndiff --git a/revision.h b/revision.h\nindex 200042a..d97ac8c 100644\n--- a/revision.h\n+++ b/revision.h\n@@ -207,6 +207,8 @@ extern int make_cache_slice(struct rev_cache_info *rci,\n extern int make_cache_index(struct rev_cache_info *rci, unsigned char *cache_sha1, \n \tint fd, unsigned int size);\n \n-extern void starts_from_slices(struct rev_info *revs, unsigned int flags);\n+extern void starts_from_slices(struct rev_info *revs, unsigned int flags, unsigned char *which, int n);\n+extern int coagulate_cache_slices(struct rev_cache_info *rci, struct rev_info *revs);\n+extern int regenerate_cache_index(struct rev_cache_info *rci);\n \n #endif\n-- \n"}]}