git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v4 06/24] read-cache: add index reading api

From
Thomas Gummerer <t.gummerer@gmail.com>
Date
Nov 27, 2013, 12:00 UTC
Message-ID
<1385553659-9928-7-git-send-email-t.gummerer@gmail.com>
In-Reply-To
<1385553659-9928-1-git-send-email-t.gummerer@gmail.com>

Add an api for access to the index file. Currently there is only a very basic api for accessing the index file, which only allows a full read of the index, and lets the users of the data filter it. The new index api gives the users the possibility to use only part of the index and provides functions for iterating over and accessing cache entries.

This simplifies future improvements to the in-memory format, as changes will be concentrated on one file, instead of the whole git source code.

Helped-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
---
 cache.h         | 41 ++++++++++++++++++++++++++++++++++++++++-
 read-cache-v2.c | 10 ++++++++--
 read-cache.c    | 47 +++++++++++++++++++++++++++++++++++++++++++----
 read-cache.h    |  3 ++-
 4 files changed, 93 insertions(+), 8 deletions(-)
diff --git a/cache.h b/cache.h
index 290e26d..38d57e7 100644
--- a/cache.h
+++ b/cache.h
@@ -127,7 +127,7 @@ struct cache_entry {
 	unsigned int ce_flags;
 	unsigned int ce_namelen;
 	unsigned char sha1[20];
-	struct cache_entry *next;
+	struct cache_entry *next; /* used by name_hash */
 	char name[FLEX_ARRAY]; /* more */
 };
 
@@ -260,6 +260,29 @@ static inline unsigned int canon_mode(unsigned int mode)
 
 #define cache_entry_size(len) (offsetof(struct cache_entry,name) + (len) + 1)
 
+/*
+ * Options by which the index should be filtered when read partially.
+ *
+ * pathspec: The pathspec which the index entries have to match
+ * seen: Used to return the seen parameter from match_pathspec()
+ * max_prefix_len: The common prefix length of the pathspecs
+ *
+ * read_staged: used to indicate if the conflicted entries (entries
+ *     with a stage) should be included
+ * read_cache_tree: used to indicate if the cache-tree should be read
+ * read_resolve_undo: used to indicate if the resolve undo data should
+ *     be read
+ */
+struct filter_opts {
+	const struct pathspec *pathspec;
+	char *seen;
+	int max_prefix_len;
+
+	int read_staged;
+	int read_cache_tree;
+	int read_resolve_undo;
+};
+
 struct index_state {
 	struct cache_entry **cache;
 	unsigned int version;
@@ -272,6 +295,7 @@ struct index_state {
 	struct hash_table name_hash;
 	struct hash_table dir_hash;
 	struct index_ops *ops;
+	struct filter_opts *filter_opts;
 };
 
 extern struct index_state the_index;
@@ -317,6 +341,12 @@ extern void free_name_hash(struct index_state *istate);
 #define unmerge_cache_entry_at(at) unmerge_index_entry_at(&the_index, at)
 #define unmerge_cache(pathspec) unmerge_index(&the_index, pathspec)
 #define read_blob_data_from_cache(path, sz) read_blob_data_from_index(&the_index, (path), (sz))
+
+/* index api */
+#define read_cache_filtered(opts) read_index_filtered(&the_index, (opts))
+#define read_cache_filtered_from(path, opts) read_index_filtered_from(&the_index, (path), (opts))
+#define for_each_cache_entry(fn, cb_data) \
+	for_each_index_entry(&the_index, (fn), (cb_data))
 #endif
 
 enum object_type {
@@ -449,6 +479,15 @@ extern void sanitize_stdfds(void);
 		} \
 	} while (0)
 
+/* index api */
+extern int read_index_filtered(struct index_state *, struct filter_opts *opts);
+extern int read_index_filtered_from(struct index_state *, const char *path, struct filter_opts *opts);
+
+typedef int each_cache_entry_fn(struct cache_entry *ce, void *);
+extern int for_each_index_entry(struct index_state *istate,
+				each_cache_entry_fn, void *);
+
+
 /* Initialize and use the cache information */
 extern void initialize_index(struct index_state *istate, int version);
 extern void change_index_version(struct index_state *istate, int version);
diff --git a/read-cache-v2.c b/read-cache-v2.c
index a7d076c..f884c10 100644
--- a/read-cache-v2.c
+++ b/read-cache-v2.c
@@ -3,6 +3,7 @@
 #include "resolve-undo.h"
 #include "cache-tree.h"
 #include "varint.h"
+#include "dir.h"
 
 /* Mask for the name length in ce_flags in the on-disk index */
 #define CE_NAMEMASK  (0x0fff)
@@ -202,8 +203,14 @@ static int read_index_extension(struct index_state *istate,
 	return 0;
 }
 
+/*
+ * The performance is the same if we read the whole index or only
+ * part of it, therefore we always read the whole index to avoid
+ * having to re-read it later.  The filter_opts will determine
+ * what part of the index is used when retrieving the cache-entries.
+ */
 static int read_index_v2(struct index_state *istate, void *mmap,
-			 unsigned long mmap_size)
+			 unsigned long mmap_size, struct filter_opts *opts)
 {
 	int i;
 	unsigned long src_offset;
@@ -229,7 +236,6 @@ static int read_index_v2(struct index_state *istate, void *mmap,
 		disk_ce = (struct ondisk_cache_entry *)((char *)mmap + src_offset);
 		ce = create_from_disk(disk_ce, &consumed, previous_name);
 		set_index_entry(istate, i, ce);
-
 		src_offset += consumed;
 	}
 	strbuf_release(&previous_name_buf);
diff --git a/read-cache.c b/read-cache.c
index 51be1bb..01f5397 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -1288,11 +1288,41 @@ static int verify_hdr_version(struct index_state *istate,
 
 int read_index(struct index_state *istate)
 {
-	return read_index_from(istate, get_index_file());
+	return read_index_filtered_from(istate, get_index_file(), NULL);
 }
 
-/* remember to discard_cache() before reading a different cache! */
-int read_index_from(struct index_state *istate, const char *path)
+int read_index_filtered(struct index_state *istate, struct filter_opts *opts)
+{
+	return read_index_filtered_from(istate, get_index_file(), opts);
+}
+
+/*
+ * Execute fn for each index entry which is currently in istate.  Data
+ * can be given to the function using the cb_data parameter.
+ */
+int for_each_index_entry(struct index_state *istate, each_cache_entry_fn fn, void *cb_data)
+{
+	int i, ret = 0;
+	struct filter_opts *opts = istate->filter_opts;
+
+	for (i = 0; i < istate->cache_nr; i++) {
+		struct cache_entry *ce = istate->cache[i];
+
+		if (opts && !opts->read_staged && ce_stage(ce))
+			continue;
+
+		if (opts && !match_pathspec_depth(opts->pathspec, ce->name, ce_namelen(ce),
+						  opts->max_prefix_len, opts->seen))
+			continue;
+
+		if ((ret = fn(istate->cache[i], cb_data)))
+			break;
+	}
+	return ret;
+}
+
+int read_index_filtered_from(struct index_state *istate, const char *path,
+			     struct filter_opts *opts)
 {
 	int fd, err, i;
 	struct stat_validity sv;
@@ -1307,6 +1337,7 @@ int read_index_from(struct index_state *istate, const char *path)
 	errno = ENOENT;
 	istate->timestamp.sec = 0;
 	istate->timestamp.nsec = 0;
+	istate->filter_opts = opts;
 	sv.sd = NULL;
 	for (i = 0; i < 50; i++) {
 		err = 0;
@@ -1337,7 +1368,7 @@ int read_index_from(struct index_state *istate, const char *path)
 		if (!err && istate->ops->verify_hdr(mmap, mmap_size) < 0)
 			err = 1;
 
-		if (!err && istate->ops->read_index(istate, mmap, mmap_size) < 0)
+		if (!err && istate->ops->read_index(istate, mmap, mmap_size, opts) < 0)
 			err = 1;
 		istate->timestamp.sec = sv.sd->sd_mtime.sec;
 		istate->timestamp.nsec = sv.sd->sd_mtime.nsec;
@@ -1350,6 +1381,13 @@ int read_index_from(struct index_state *istate, const char *path)
 	die("index file corrupt");
 }
 
+
+/* remember to discard_cache() before reading a different cache! */
+int read_index_from(struct index_state *istate, const char *path)
+{
+	return read_index_filtered_from(istate, path, NULL);
+}
+
 int is_index_unborn(struct index_state *istate)
 {
 	return (!istate->cache_nr && !istate->timestamp.sec);
@@ -1373,6 +1411,7 @@ int discard_index(struct index_state *istate)
 	istate->cache = NULL;
 	istate->cache_alloc = 0;
 	istate->ops = NULL;
+	istate->filter_opts = NULL;
 	return 0;
 }
 
diff --git a/read-cache.h b/read-cache.h
index ceedcae..f920546 100644
--- a/read-cache.h
+++ b/read-cache.h
@@ -28,7 +28,8 @@ struct cache_header {
 struct index_ops {
 	int (*match_stat_basic)(const struct cache_entry *ce, struct stat *st, int changed);
 	int (*verify_hdr)(void *mmap, unsigned long size);
-	int (*read_index)(struct index_state *istate, void *mmap, unsigned long mmap_size);
+	int (*read_index)(struct index_state *istate, void *mmap, unsigned long mmap_size,
+			  struct filter_opts *opts);
 	int (*write_index)(struct index_state *istate, int newfd);
 };
 

-- 
1.8.4.2
Previous: Thomas GummererNext: Thomas Gummerer
Message 7 of 41 in “Index-v5”
  1. 00/24 Index-v5Thomas Gummerer, Nov 27, 2013
  2. 01/24 t2104: Don't fail for index versions other than [23]Thomas Gummerer, Nov 27, 2013
  3. 02/24 read-cache: split index file version specific functionalityThomas Gummerer, Nov 27, 2013
  4. 03/24 read-cache: move index v2 specific functions to their own fileThomas Gummerer, Nov 27, 2013
  5. 04/24 read-cache: Re-read index if index file changedThomas Gummerer, Nov 27, 2013
  6. 05/24 add documentation for the index apiThomas Gummerer, Nov 27, 2013
  7. 06/24 read-cache: add index reading apiThomas Gummerer, Nov 27, 2013
  8. 07/24 make sure partially read index is not changedThomas Gummerer, Nov 27, 2013
  9. 08/24 grep.c: use index apiThomas Gummerer, Nov 27, 2013
  10. 09/24 ls-files.c: use index apiThomas Gummerer, Nov 27, 2013
  11. Duy NguyenNov 30, 2013
  12. Thomas GummererNov 30, 2013
  13. Antoine PelisseNov 30, 2013
  14. Thomas GummererNov 30, 2013
  15. 10/24 documentation: add documentation of the index-v5 file formatThomas Gummerer, Nov 27, 2013
  16. 11/24 read-cache: make in-memory format aware of stat_crcThomas Gummerer, Nov 27, 2013
  17. 12/24 read-cache: read index-v5Thomas Gummerer, Nov 27, 2013
  18. Duy NguyenNov 30, 2013
  19. Thomas GummererNov 30, 2013
  20. Antoine PelisseNov 30, 2013
  21. Thomas GummererNov 30, 2013
  22. Antoine PelisseNov 30, 2013
  23. Thomas GummererNov 30, 2013
  24. 13/24 read-cache: read resolve-undo dataThomas Gummerer, Nov 27, 2013
  25. 14/24 read-cache: read cache-tree in index-v5Thomas Gummerer, Nov 27, 2013
  26. 15/24 read-cache: write index-v5Thomas Gummerer, Nov 27, 2013
  27. 16/24 read-cache: write index-v5 cache-tree dataThomas Gummerer, Nov 27, 2013
  28. 17/24 read-cache: write resolve-undo data for index-v5Thomas Gummerer, Nov 27, 2013
  29. 18/24 update-index.c: rewrite index when index-version is givenThomas Gummerer, Nov 27, 2013
  30. 19/24 p0003-index.sh: add perf test for the index formatsThomas Gummerer, Nov 27, 2013
  31. 20/24 introduce GIT_INDEX_VERSION environment variableThomas Gummerer, Nov 27, 2013
  32. Eric SunshineNov 27, 2013
  33. Junio C HamanoNov 27, 2013
  34. Thomas GummererNov 28, 2013
  35. 21/24 test-lib: allow setting the index format versionThomas Gummerer, Nov 27, 2013
  36. 22/24 t1600: add index v5 specific testsThomas Gummerer, Nov 27, 2013
  37. 23/24 POC for partial writingThomas Gummerer, Nov 27, 2013
  38. Duy NguyenNov 30, 2013
  39. Thomas GummererNov 30, 2013
  40. 24/24 perf: add partial writing testThomas Gummerer, Nov 27, 2013
  41. Thomas GummererDec 9, 2013

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.