git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 01/14] Extend index to save more flags

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
Sep 20, 2008, 10:01 UTC
Message-ID
<1221904913-25887-2-git-send-email-pclouds@gmail.com>
In-Reply-To
<1221904913-25887-1-git-send-email-pclouds@gmail.com>

The on-disk format of index only saves 16 bit flags, nearly all have been used. The last bit (CE_EXTENDED) is used to for future extension.

This patch extends index entry format to save more flags in future. The new entry format will be used when CE_EXTENDED bit is 1.

Because older implementation may not understand CE_EXTENDED bit and misread the new format, if there is any extended entry in index, index header version will turn 3, which makes it incompatible for older git. If there is none, header version will return to 2 again.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 cache.h      |   58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 read-cache.c |   51 +++++++++++++++++++++++++++++++++++++++++----------
 2 files changed, 95 insertions(+), 14 deletions(-)
diff --git a/cache.h b/cache.h
index f4b8ddf..77b6eb3 100644
--- a/cache.h
+++ b/cache.h
@@ -109,6 +109,26 @@ struct ondisk_cache_entry {
 	char name[FLEX_ARRAY]; /* more */
 };
 
+/*
+ * This struct is used when CE_EXTENDED bit is 1
+ * The struct must match ondisk_cache_entry exactly from
+ * ctime till flags
+ */
+struct ondisk_cache_entry_extended {
+	struct cache_time ctime;
+	struct cache_time mtime;
+	unsigned int dev;
+	unsigned int ino;
+	unsigned int mode;
+	unsigned int uid;
+	unsigned int gid;
+	unsigned int size;
+	unsigned char sha1[20];
+	unsigned short flags;
+	unsigned short flags2;
+	char name[FLEX_ARRAY]; /* more */
+};
+
 struct cache_entry {
 	unsigned int ce_ctime;
 	unsigned int ce_mtime;
@@ -130,7 +150,15 @@ struct cache_entry {
 #define CE_VALID     (0x8000)
 #define CE_STAGESHIFT 12
 
-/* In-memory only */
+/*
+ * Range 0xFFFF0000 in ce_flags is divided into
+ * two parts: in-memory flags and on-disk ones.
+ * Flags in CE_EXTENDED_FLAGS will get saved on-disk
+ * if you want to save a new flag, add it in
+ * CE_EXTENDED_FLAGS
+ *
+ * In-memory only flags
+ */
 #define CE_UPDATE    (0x10000)
 #define CE_REMOVE    (0x20000)
 #define CE_UPTODATE  (0x40000)
@@ -140,6 +168,24 @@ struct cache_entry {
 #define CE_UNHASHED  (0x200000)
 
 /*
+ * Extended on-disk flags
+ */
+/* CE_EXTENDED2 is for future extension */
+#define CE_EXTENDED2 0x80000000
+
+#define CE_EXTENDED_FLAGS (0)
+
+/*
+ * Safeguard to avoid saving wrong flags:
+ *  - CE_EXTENDED2 won't get saved until its semantic is known
+ *  - Bits in 0x0000FFFF have been saved in ce_flags already
+ *  - Bits in 0x003F0000 are currently in-memory flags
+ */
+#if CE_EXTENDED_FLAGS & 0x80CFFFFF
+#error "CE_EXTENDED_FLAGS out of range"
+#endif
+
+/*
  * Copy the sha1 and stat state of a cache entry from one to
  * another. But we never change the name, or the hash state!
  */
@@ -171,7 +217,9 @@ static inline size_t ce_namelen(const struct cache_entry *ce)
 }
 
 #define ce_size(ce) cache_entry_size(ce_namelen(ce))
-#define ondisk_ce_size(ce) ondisk_cache_entry_size(ce_namelen(ce))
+#define ondisk_ce_size(ce) (((ce)->ce_flags & CE_EXTENDED) ? \
+			    ondisk_cache_entry_extended_size(ce_namelen(ce)) : \
+			    ondisk_cache_entry_size(ce_namelen(ce)))
 #define ce_stage(ce) ((CE_STAGEMASK & (ce)->ce_flags) >> CE_STAGESHIFT)
 #define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE)
 #define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE)
@@ -214,8 +262,10 @@ static inline int ce_to_dtype(const struct cache_entry *ce)
 	(S_ISREG(mode) ? (S_IFREG | ce_permissions(mode)) : \
 	S_ISLNK(mode) ? S_IFLNK : S_ISDIR(mode) ? S_IFDIR : S_IFGITLINK)
 
-#define cache_entry_size(len) ((offsetof(struct cache_entry,name) + (len) + 8) & ~7)
-#define ondisk_cache_entry_size(len) ((offsetof(struct ondisk_cache_entry,name) + (len) + 8) & ~7)
+#define flexible_size(STRUCT,len) ((offsetof(struct STRUCT,name) + (len) + 8) & ~7)
+#define cache_entry_size(len) flexible_size(cache_entry,len)
+#define ondisk_cache_entry_size(len) flexible_size(ondisk_cache_entry,len)
+#define ondisk_cache_entry_extended_size(len) flexible_size(ondisk_cache_entry_extended,len)
 
 struct index_state {
 	struct cache_entry **cache;
diff --git a/read-cache.c b/read-cache.c
index c5a8659..667c36b 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -1096,7 +1096,7 @@ static int verify_hdr(struct cache_header *hdr, unsigned long size)
 
 	if (hdr->hdr_signature != htonl(CACHE_SIGNATURE))
 		return error("bad signature");
-	if (hdr->hdr_version != htonl(2))
+	if (hdr->hdr_version != htonl(2) && hdr->hdr_version != htonl(3))
 		return error("bad index version");
 	SHA1_Init(&c);
 	SHA1_Update(&c, hdr, size - 20);
@@ -1131,6 +1131,7 @@ int read_index(struct index_state *istate)
 static void convert_from_disk(struct ondisk_cache_entry *ondisk, struct cache_entry *ce)
 {
 	size_t len;
+	const char *name;
 
 	ce->ce_ctime = ntohl(ondisk->ctime.sec);
 	ce->ce_mtime = ntohl(ondisk->mtime.sec);
@@ -1143,19 +1144,31 @@ static void convert_from_disk(struct ondisk_cache_entry *ondisk, struct cache_en
 	/* On-disk flags are just 16 bits */
 	ce->ce_flags = ntohs(ondisk->flags);
 
-	/* For future extension: we do not understand this entry yet */
-	if (ce->ce_flags & CE_EXTENDED)
-		die("Unknown index entry format");
 	hashcpy(ce->sha1, ondisk->sha1);
 
 	len = ce->ce_flags & CE_NAMEMASK;
+
+	if (ce->ce_flags & CE_EXTENDED) {
+		struct ondisk_cache_entry_extended *ondisk2;
+		int extended_flags;
+		ondisk2 = (struct ondisk_cache_entry_extended *)ondisk;
+		extended_flags = ntohs(ondisk2->flags2) << 16;
+		/* We do not yet understand any bit out of CE_EXTENDED_FLAGS */
+		if (extended_flags & ~CE_EXTENDED_FLAGS)
+			die("Unknown index entry format %08x", extended_flags);
+		ce->ce_flags |= extended_flags;
+		name = ondisk2->name;
+	}
+	else
+		name = ondisk->name;
+
 	if (len == CE_NAMEMASK)
-		len = strlen(ondisk->name);
+		len = strlen(name);
 	/*
 	 * NEEDSWORK: If the original index is crafted, this copy could
 	 * go unchecked.
 	 */
-	memcpy(ce->name, ondisk->name, len + 1);
+	memcpy(ce->name, name, len + 1);
 }
 
 static inline size_t estimate_cache_size(size_t ondisk_size, unsigned int entries)
@@ -1415,6 +1428,7 @@ static int ce_write_entry(SHA_CTX *c, int fd, struct cache_entry *ce)
 {
 	int size = ondisk_ce_size(ce);
 	struct ondisk_cache_entry *ondisk = xcalloc(1, size);
+	char *name;
 
 	ondisk->ctime.sec = htonl(ce->ce_ctime);
 	ondisk->ctime.nsec = 0;
@@ -1428,7 +1442,15 @@ static int ce_write_entry(SHA_CTX *c, int fd, struct cache_entry *ce)
 	ondisk->size = htonl(ce->ce_size);
 	hashcpy(ondisk->sha1, ce->sha1);
 	ondisk->flags = htons(ce->ce_flags);
-	memcpy(ondisk->name, ce->name, ce_namelen(ce));
+	if (ce->ce_flags & CE_EXTENDED) {
+		struct ondisk_cache_entry_extended *ondisk2;
+		ondisk2 = (struct ondisk_cache_entry_extended *)ondisk;
+		ondisk2->flags2 = htons((ce->ce_flags & CE_EXTENDED_FLAGS) >> 16);
+		name = ondisk2->name;
+	}
+	else
+		name = ondisk->name;
+	memcpy(name, ce->name, ce_namelen(ce));
 
 	return ce_write(c, fd, ondisk, size);
 }
@@ -1437,16 +1459,25 @@ int write_index(const struct index_state *istate, int newfd)
 {
 	SHA_CTX c;
 	struct cache_header hdr;
-	int i, err, removed;
+	int i, err, removed, extended;
 	struct cache_entry **cache = istate->cache;
 	int entries = istate->cache_nr;
 
-	for (i = removed = 0; i < entries; i++)
+	for (i = removed = extended = 0; i < entries; i++) {
 		if (cache[i]->ce_flags & CE_REMOVE)
 			removed++;
 
+		/* reduce extended entries if possible */
+		cache[i]->ce_flags &= ~CE_EXTENDED;
+		if (cache[i]->ce_flags & CE_EXTENDED_FLAGS) {
+			extended++;
+			cache[i]->ce_flags |= CE_EXTENDED;
+		}
+	}
+
 	hdr.hdr_signature = htonl(CACHE_SIGNATURE);
-	hdr.hdr_version = htonl(2);
+	/* for extended format, increase version so older git won't try to read it */
+	hdr.hdr_version = htonl(extended ? 3 : 2);
 	hdr.hdr_entries = htonl(entries - removed);
 
 	SHA1_Init(&c);

-- 
1.6.0.96.g2fad1.dirty
Previous: Nguyễn Thái Ngọc DuyNext: Nguyễn Thái Ngọc Duy
Message 2 of 37 in “Sparse checkout”
  1. 00/14 Sparse checkoutNguyễn Thái Ngọc Duy, Sep 20, 2008
  2. 01/14 Extend index to save more flagsNguyễn Thái Ngọc Duy, Sep 20, 2008
  3. 02/14 Introduce CE_NO_CHECKOUT bitNguyễn Thái Ngọc Duy, Sep 20, 2008
  4. 03/14 ls-files: add options to support sparse checkoutNguyễn Thái Ngọc Duy, Sep 20, 2008
  5. 04/14 update-index: refactor mark_valid() in preparation for new optionsNguyễn Thái Ngọc Duy, Sep 20, 2008
  6. 05/14 update-index: add --checkout/--no-checkout to update CE_NO_CHECKOUT bitNguyễn Thái Ngọc Duy, Sep 20, 2008
  7. 06/14 ls-files: Add tests for --sparse and friendsNguyễn Thái Ngọc Duy, Sep 20, 2008
  8. 07/14 Prevent diff machinery from examining worktree outside sparse checkoutNguyễn Thái Ngọc Duy, Sep 20, 2008
  9. 08/14 checkout_entry(): CE_NO_CHECKOUT on checked out entries.Nguyễn Thái Ngọc Duy, Sep 20, 2008
  10. 09/14 grep: skip files outside sparse checkout areaNguyễn Thái Ngọc Duy, Sep 20, 2008
  11. 10/14 ls-files: support "sparse patterns", used to form sparse checkout areasNguyễn Thái Ngọc Duy, Sep 20, 2008
  12. 11/14 unpack_trees(): add support for sparse checkoutNguyễn Thái Ngọc Duy, Sep 20, 2008
  13. 12/14 clone: support sparse checkout with --narrow-path optionNguyễn Thái Ngọc Duy, Sep 20, 2008
  14. 13/14 checkout: add new options to support sparse checkoutNguyễn Thái Ngọc Duy, Sep 20, 2008
  15. 14/14 wt-status: Show orphaned entries in "git status" outputNguyễn Thái Ngọc Duy, Sep 20, 2008
  16. Jakub NarebskiSep 20, 2008
  17. Junio C HamanoSep 20, 2008
  18. Junio C HamanoSep 20, 2008
  19. Nguyen Thai Ngoc DuySep 21, 2008
  20. Jakub NarebskiSep 21, 2008
  21. Santi BéjarSep 20, 2008
  22. Nguyen Thai Ngoc DuySep 20, 2008
  23. Jakub NarebskiSep 20, 2008
  24. Nguyen Thai Ngoc DuySep 20, 2008
  25. Jakub NarebskiSep 20, 2008
  26. Uwe Kleine-KönigSep 20, 2008
  27. Nguyen Thai Ngoc DuySep 20, 2008
  28. Junio C HamanoSep 20, 2008
  29. Nguyen Thai Ngoc DuySep 21, 2008
  30. Jakub NarebskiSep 21, 2008
  31. Nguyen Thai Ngoc DuySep 21, 2008
  32. Jakub NarebskiSep 21, 2008
  33. Santi BéjarSep 23, 2008
  34. Nguyen Thai Ngoc DuySep 23, 2008
  35. Nguyen Thai Ngoc DuySep 26, 2008
  36. Junio C HamanoSep 20, 2008
  37. Santi BéjarSep 23, 2008

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.