git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 07/19] streaming: create structure for loose object streams

From
Patrick Steinhardt <ps@pks.im>
Date
Nov 21, 2025, 07:40 UTC
Message-ID
<20251121-b4-pks-odb-read-stream-v2-7-ca8534963150@pks.im>
In-Reply-To
<20251121-b4-pks-odb-read-stream-v2-0-ca8534963150@pks.im>

As explained in a preceding commit, we want to get rid of the union of stream-type specific data in `struct odb_read_stream`. Create a new structure for loose object streams to move towards this design.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
---
 streaming.c | 85 ++++++++++++++++++++++++++++++++-----------------------------
 1 file changed, 44 insertions(+), 41 deletions(-)
diff --git a/streaming.c b/streaming.c
index 3af2f0c776..193405d11e 100644
--- a/streaming.c
+++ b/streaming.c
@@ -39,14 +39,6 @@ struct odb_read_stream {
 	enum { z_unused, z_used, z_done, z_error } z_state;
 
 	union {
-		struct {
-			void *mapped;
-			unsigned long mapsize;
-			char hdr[32];
-			int hdr_avail;
-			int hdr_used;
-		} loose;
-
 		struct {
 			struct packed_git *pack;
 			off_t pos;
@@ -165,11 +157,21 @@ static struct odb_read_stream *attach_stream_filter(struct odb_read_stream *st,
  *
  *****************************************************************/
 
-static ssize_t read_istream_loose(struct odb_read_stream *st, char *buf, size_t sz)
+struct odb_loose_read_stream {
+	struct odb_read_stream base;
+	void *mapped;
+	unsigned long mapsize;
+	char hdr[32];
+	int hdr_avail;
+	int hdr_used;
+};
+
+static ssize_t read_istream_loose(struct odb_read_stream *_st, char *buf, size_t sz)
 {
+	struct odb_loose_read_stream *st = (struct odb_loose_read_stream *)_st;
 	size_t total_read = 0;
 
-	switch (st->z_state) {
+	switch (st->base.z_state) {
 	case z_done:
 		return 0;
 	case z_error:
@@ -178,42 +180,43 @@ static ssize_t read_istream_loose(struct odb_read_stream *st, char *buf, size_t
 		break;
 	}
 
-	if (st->u.loose.hdr_used < st->u.loose.hdr_avail) {
-		size_t to_copy = st->u.loose.hdr_avail - st->u.loose.hdr_used;
+	if (st->hdr_used < st->hdr_avail) {
+		size_t to_copy = st->hdr_avail - st->hdr_used;
 		if (sz < to_copy)
 			to_copy = sz;
-		memcpy(buf, st->u.loose.hdr + st->u.loose.hdr_used, to_copy);
-		st->u.loose.hdr_used += to_copy;
+		memcpy(buf, st->hdr + st->hdr_used, to_copy);
+		st->hdr_used += to_copy;
 		total_read += to_copy;
 	}
 
 	while (total_read < sz) {
 		int status;
 
-		st->z.next_out = (unsigned char *)buf + total_read;
-		st->z.avail_out = sz - total_read;
-		status = git_inflate(&st->z, Z_FINISH);
+		st->base.z.next_out = (unsigned char *)buf + total_read;
+		st->base.z.avail_out = sz - total_read;
+		status = git_inflate(&st->base.z, Z_FINISH);
 
-		total_read = st->z.next_out - (unsigned char *)buf;
+		total_read = st->base.z.next_out - (unsigned char *)buf;
 
 		if (status == Z_STREAM_END) {
-			git_inflate_end(&st->z);
-			st->z_state = z_done;
+			git_inflate_end(&st->base.z);
+			st->base.z_state = z_done;
 			break;
 		}
 		if (status != Z_OK && (status != Z_BUF_ERROR || total_read < sz)) {
-			git_inflate_end(&st->z);
-			st->z_state = z_error;
+			git_inflate_end(&st->base.z);
+			st->base.z_state = z_error;
 			return -1;
 		}
 	}
 	return total_read;
 }
 
-static int close_istream_loose(struct odb_read_stream *st)
+static int close_istream_loose(struct odb_read_stream *_st)
 {
-	close_deflated_stream(st);
-	munmap(st->u.loose.mapped, st->u.loose.mapsize);
+	struct odb_loose_read_stream *st = (struct odb_loose_read_stream *)_st;
+	close_deflated_stream(&st->base);
+	munmap(st->mapped, st->mapsize);
 	return 0;
 }
 
@@ -222,7 +225,7 @@ static int open_istream_loose(struct odb_read_stream **out,
 			      const struct object_id *oid)
 {
 	struct object_info oi = OBJECT_INFO_INIT;
-	struct odb_read_stream *st;
+	struct odb_loose_read_stream *st;
 	struct odb_source *source;
 	unsigned long mapsize;
 	void *mapped;
@@ -244,8 +247,8 @@ static int open_istream_loose(struct odb_read_stream **out,
 	 */
 	CALLOC_ARRAY(st, 1);
 
-	switch (unpack_loose_header(&st->z, mapped, mapsize, st->u.loose.hdr,
-				    sizeof(st->u.loose.hdr))) {
+	switch (unpack_loose_header(&st->base.z, mapped, mapsize, st->hdr,
+				    sizeof(st->hdr))) {
 	case ULHR_OK:
 		break;
 	case ULHR_BAD:
@@ -253,26 +256,26 @@ static int open_istream_loose(struct odb_read_stream **out,
 		goto error;
 	}
 
-	oi.sizep = &st->size;
-	oi.typep = &st->type;
+	oi.sizep = &st->base.size;
+	oi.typep = &st->base.type;
 
-	if (parse_loose_header(st->u.loose.hdr, &oi) < 0 || st->type < 0)
+	if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
 		goto error;
 
-	st->u.loose.mapped = mapped;
-	st->u.loose.mapsize = mapsize;
-	st->u.loose.hdr_used = strlen(st->u.loose.hdr) + 1;
-	st->u.loose.hdr_avail = st->z.total_out;
-	st->z_state = z_used;
-	st->close = close_istream_loose;
-	st->read = read_istream_loose;
+	st->mapped = mapped;
+	st->mapsize = mapsize;
+	st->hdr_used = strlen(st->hdr) + 1;
+	st->hdr_avail = st->base.z.total_out;
+	st->base.z_state = z_used;
+	st->base.close = close_istream_loose;
+	st->base.read = read_istream_loose;
 
-	*out = st;
+	*out = &st->base;
 
 	return 0;
 error:
-	git_inflate_end(&st->z);
-	munmap(st->u.loose.mapped, st->u.loose.mapsize);
+	git_inflate_end(&st->base.z);
+	munmap(st->mapped, st->mapsize);
 	free(st);
 	return -1;
 }
-- 
2.52.0.rc2.482.gaa765fefd0.dirty
Previous: Patrick SteinhardtNext: Patrick Steinhardt
Message 46 of 85 in “Refactor object read streams to work via object sources”
  1. 00/18 Refactor object read streams to work via object sourcesPatrick Steinhardt, Nov 19, 2025
  2. 01/18 streaming: rename `git_istream` into `odb_read_stream`Patrick Steinhardt, Nov 19, 2025
  3. Justin ToblerNov 19, 2025
  4. Junio C HamanoNov 19, 2025
  5. Patrick SteinhardtNov 21, 2025
  6. 02/18 streaming: drop the `open()` callback functionPatrick Steinhardt, Nov 19, 2025
  7. Karthik NayakNov 19, 2025
  8. Justin ToblerNov 19, 2025
  9. Patrick SteinhardtNov 21, 2025
  10. 03/18 streaming: propagate final object type via the streamPatrick Steinhardt, Nov 19, 2025
  11. Justin ToblerNov 19, 2025
  12. Patrick SteinhardtNov 21, 2025
  13. 04/18 streaming: explicitly pass packfile info when streaming a packed objectPatrick Steinhardt, Nov 19, 2025
  14. 05/18 streaming: allocate stream inside the backend-specific logicPatrick Steinhardt, Nov 19, 2025
  15. Karthik NayakNov 19, 2025
  16. Patrick SteinhardtNov 21, 2025
  17. 06/18 streaming: create structure for in-core object streamsPatrick Steinhardt, Nov 19, 2025
  18. Karthik NayakNov 19, 2025
  19. Patrick SteinhardtNov 21, 2025
  20. 07/18 streaming: create structure for loose object streamsPatrick Steinhardt, Nov 19, 2025
  21. 08/18 streaming: create structure for packed object streamsPatrick Steinhardt, Nov 19, 2025
  22. 09/18 streaming: create structure for filtered object streamsPatrick Steinhardt, Nov 19, 2025
  23. 10/18 streaming: move zlib stream into backendsPatrick Steinhardt, Nov 19, 2025
  24. 11/18 packfile: introduce function to read object info from a storePatrick Steinhardt, Nov 19, 2025
  25. Karthik NayakNov 19, 2025
  26. Patrick SteinhardtNov 21, 2025
  27. 12/18 streaming: rely on object sources to create object streamPatrick Steinhardt, Nov 19, 2025
  28. Karthik NayakNov 19, 2025
  29. 13/18 streaming: get rid of `the_repository`Patrick Steinhardt, Nov 19, 2025
  30. 14/18 streaming: make the `odb_read_stream` definition publicPatrick Steinhardt, Nov 19, 2025
  31. Karthik NayakNov 19, 2025
  32. Patrick SteinhardtNov 21, 2025
  33. 15/18 streaming: move logic to read loose objects streams into backendPatrick Steinhardt, Nov 19, 2025
  34. 16/18 streaming: move logic to read packed objects streams into backendPatrick Steinhardt, Nov 19, 2025
  35. 17/18 streaming: refactor interface to be object-database-centricPatrick Steinhardt, Nov 19, 2025
  36. 18/18 streaming: move into object database subsystemPatrick Steinhardt, Nov 19, 2025
  37. 00/19 Refactor object read streams to work via object sourcesPatrick Steinhardt, Nov 21, 2025
  38. 01/19 streaming: rename `git_istream` into `odb_read_stream`Patrick Steinhardt, Nov 21, 2025
  39. 02/19 streaming: drop the `open()` callback functionPatrick Steinhardt, Nov 21, 2025
  40. Junio C HamanoNov 21, 2025
  41. Patrick SteinhardtNov 23, 2025
  42. 03/19 streaming: propagate final object type via the streamPatrick Steinhardt, Nov 21, 2025
  43. 04/19 streaming: explicitly pass packfile info when streaming a packed objectPatrick Steinhardt, Nov 21, 2025
  44. 05/19 streaming: allocate stream inside the backend-specific logicPatrick Steinhardt, Nov 21, 2025
  45. 06/19 streaming: create structure for in-core object streamsPatrick Steinhardt, Nov 21, 2025
  46. 07/19 streaming: create structure for loose object streamsPatrick Steinhardt, Nov 21, 2025
  47. 08/19 streaming: create structure for packed object streamsPatrick Steinhardt, Nov 21, 2025
  48. 09/19 streaming: create structure for filtered object streamsPatrick Steinhardt, Nov 21, 2025
  49. 10/19 streaming: move zlib stream into backendsPatrick Steinhardt, Nov 21, 2025
  50. 11/19 packfile: introduce function to read object info from a storePatrick Steinhardt, Nov 21, 2025
  51. 12/19 streaming: rely on object sources to create object streamPatrick Steinhardt, Nov 21, 2025
  52. Junio C HamanoNov 21, 2025
  53. Patrick SteinhardtNov 23, 2025
  54. 13/19 streaming: get rid of `the_repository`Patrick Steinhardt, Nov 21, 2025
  55. Junio C HamanoNov 21, 2025
  56. Patrick SteinhardtNov 23, 2025
  57. 14/19 streaming: make the `odb_read_stream` definition publicPatrick Steinhardt, Nov 21, 2025
  58. 15/19 streaming: move logic to read loose objects streams into backendPatrick Steinhardt, Nov 21, 2025
  59. 16/19 streaming: move logic to read packed objects streams into backendPatrick Steinhardt, Nov 21, 2025
  60. 17/19 streaming: refactor interface to be object-database-centricPatrick Steinhardt, Nov 21, 2025
  61. Junio C HamanoNov 22, 2025
  62. Patrick SteinhardtNov 23, 2025
  63. 18/19 streaming: move into object database subsystemPatrick Steinhardt, Nov 21, 2025
  64. Junio C HamanoNov 23, 2025
  65. 19/19 streaming: drop redundant type and size pointersPatrick Steinhardt, Nov 21, 2025
  66. 00/19 Refactor object read streams to work via object sourcesPatrick Steinhardt, Nov 23, 2025
  67. 01/19 streaming: rename `git_istream` into `odb_read_stream`Patrick Steinhardt, Nov 23, 2025
  68. 02/19 streaming: drop the `open()` callback functionPatrick Steinhardt, Nov 23, 2025
  69. 03/19 streaming: propagate final object type via the streamPatrick Steinhardt, Nov 23, 2025
  70. 04/19 streaming: explicitly pass packfile info when streaming a packed objectPatrick Steinhardt, Nov 23, 2025
  71. 05/19 streaming: allocate stream inside the backend-specific logicPatrick Steinhardt, Nov 23, 2025
  72. 06/19 streaming: create structure for in-core object streamsPatrick Steinhardt, Nov 23, 2025
  73. 07/19 streaming: create structure for loose object streamsPatrick Steinhardt, Nov 23, 2025
  74. 08/19 streaming: create structure for packed object streamsPatrick Steinhardt, Nov 23, 2025
  75. 09/19 streaming: create structure for filtered object streamsPatrick Steinhardt, Nov 23, 2025
  76. 10/19 streaming: move zlib stream into backendsPatrick Steinhardt, Nov 23, 2025
  77. 11/19 packfile: introduce function to read object info from a storePatrick Steinhardt, Nov 23, 2025
  78. 12/19 streaming: rely on object sources to create object streamPatrick Steinhardt, Nov 23, 2025
  79. 13/19 streaming: get rid of `the_repository`Patrick Steinhardt, Nov 23, 2025
  80. 14/19 streaming: make the `odb_read_stream` definition publicPatrick Steinhardt, Nov 23, 2025
  81. 15/19 streaming: move logic to read loose objects streams into backendPatrick Steinhardt, Nov 23, 2025
  82. 16/19 streaming: move logic to read packed objects streams into backendPatrick Steinhardt, Nov 23, 2025
  83. 17/19 streaming: refactor interface to be object-database-centricPatrick Steinhardt, Nov 23, 2025
  84. 18/19 streaming: move into object database subsystemPatrick Steinhardt, Nov 23, 2025
  85. 19/19 streaming: drop redundant type and size pointersPatrick Steinhardt, Nov 23, 2025

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.