git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 02/11] Factor out and export large blob writing code to arbitrary file handle

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
Feb 27, 2012, 07:55 UTC
Message-ID
<1330329315-11407-3-git-send-email-pclouds@gmail.com>
In-Reply-To
<1330329315-11407-1-git-send-email-pclouds@gmail.com>
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 cache.h |    3 +++
 entry.c |   39 ++++++++++++++++++++++++++-------------
 2 files changed, 29 insertions(+), 13 deletions(-)
diff --git a/cache.h b/cache.h
index e12b15f..6ce691b 100644
--- a/cache.h
+++ b/cache.h
@@ -937,6 +937,9 @@ struct checkout {
 		 refresh_cache:1;
 };
 
+extern int streaming_write_sha1(int fd, int seekable, const unsigned char *sha1,
+				enum object_type exp_type,
+				struct stream_filter *filter);
 extern int checkout_entry(struct cache_entry *ce, const struct checkout *state, char *topath);
 
 struct cache_def {
diff --git a/entry.c b/entry.c
index 852fea1..dde0d17 100644
--- a/entry.c
+++ b/entry.c
@@ -115,26 +115,20 @@ static int fstat_output(int fd, const struct checkout *state, struct stat *st)
 	return 0;
 }
 
-static int streaming_write_entry(struct cache_entry *ce, char *path,
-				 struct stream_filter *filter,
-				 const struct checkout *state, int to_tempfile,
-				 int *fstat_done, struct stat *statbuf)
+int streaming_write_sha1(int fd, int seekable, const unsigned char *sha1,
+			 enum object_type exp_type,
+			 struct stream_filter *filter)
 {
 	struct git_istream *st;
 	enum object_type type;
 	unsigned long sz;
 	int result = -1;
 	ssize_t kept = 0;
-	int fd = -1;
 
-	st = open_istream(ce->sha1, &type, &sz, filter);
+	st = open_istream(sha1, &type, &sz, filter);
 	if (!st)
 		return -1;
-	if (type != OBJ_BLOB)
-		goto close_and_exit;
-
-	fd = open_output_fd(path, ce, to_tempfile);
-	if (fd < 0)
+	if (exp_type != OBJ_ANY && type != exp_type)
 		goto close_and_exit;
 
 	for (;;) {
@@ -144,7 +138,7 @@ static int streaming_write_entry(struct cache_entry *ce, char *path,
 
 		if (!readlen)
 			break;
-		if (sizeof(buf) == readlen) {
+		if (seekable && sizeof(buf) == readlen) {
 			for (holeto = 0; holeto < readlen; holeto++)
 				if (buf[holeto])
 					break;
@@ -166,10 +160,29 @@ static int streaming_write_entry(struct cache_entry *ce, char *path,
 	if (kept && (lseek(fd, kept - 1, SEEK_CUR) == (off_t) -1 ||
 		     write(fd, "", 1) != 1))
 		goto close_and_exit;
-	*fstat_done = fstat_output(fd, state, statbuf);
+	result = 0;
 
 close_and_exit:
 	close_istream(st);
+	return result;
+}
+
+static int streaming_write_entry(struct cache_entry *ce, char *path,
+				 struct stream_filter *filter,
+				 const struct checkout *state, int to_tempfile,
+				 int *fstat_done, struct stat *statbuf)
+{
+	int result = -1;
+	int fd = open_output_fd(path, ce, to_tempfile);
+	if (fd < 0)
+		goto close_and_exit;
+
+	if (streaming_write_sha1(fd, 1, ce->sha1, OBJ_BLOB, filter))
+		goto close_and_exit;
+
+	*fstat_done = fstat_output(fd, state, statbuf);
+
+close_and_exit:
 	if (0 <= fd)
 		result = close(fd);
 	if (result && 0 <= fd)
-- 
1.7.3.1.256.g2539c.dirty
Previous: Peter BaumannNext: Junio C Hamano
Message 4 of 48 in “Large blob fixes”
  1. 00/11 Large blob fixesNguyễn Thái Ngọc Duy, Feb 27, 2012
  2. 01/11 Add more large blob test casesNguyễn Thái Ngọc Duy, Feb 27, 2012
  3. Peter BaumannFeb 27, 2012
  4. 02/11 Factor out and export large blob writing code to arbitrary file handleNguyễn Thái Ngọc Duy, Feb 27, 2012
  5. Junio C HamanoFeb 27, 2012
  6. Junio C HamanoFeb 27, 2012
  7. 03/11 cat-file: use streaming interface to print blobsNguyễn Thái Ngọc Duy, Feb 27, 2012
  8. Junio C HamanoFeb 27, 2012
  9. Nguyen Thai Ngoc DuyFeb 28, 2012
  10. 04/11 parse_object: special code path for blobs to avoid putting whole object in memoryNguyễn Thái Ngọc Duy, Feb 27, 2012
  11. 05/11 show: use streaming interface for showing blobsNguyễn Thái Ngọc Duy, Feb 27, 2012
  12. Junio C HamanoFeb 27, 2012
  13. 06/11 index-pack --verify: skip sha-1 collision testNguyễn Thái Ngọc Duy, Feb 27, 2012
  14. 07/11 index-pack: split second pass obj handling into own functionNguyễn Thái Ngọc Duy, Feb 27, 2012
  15. 08/11 index-pack: reduce memory usage when the pack has large blobsNguyễn Thái Ngọc Duy, Feb 27, 2012
  16. 09/11 pack-check: do not unpack blobsNguyễn Thái Ngọc Duy, Feb 27, 2012
  17. 10/11 archive: support streaming large files to a tar archiveNguyễn Thái Ngọc Duy, Feb 27, 2012
  18. 11/11 fsck: use streaming interface for writing lost-found blobsNguyễn Thái Ngọc Duy, Feb 27, 2012
  19. Junio C HamanoFeb 27, 2012
  20. Nguyen Thai Ngoc DuyFeb 28, 2012
  21. 00/10 Large blob fixesNguyễn Thái Ngọc Duy, Mar 4, 2012
  22. 01/10 Add more large blob test casesNguyễn Thái Ngọc Duy, Mar 4, 2012
  23. Junio C HamanoMar 6, 2012
  24. 02/10 streaming: make streaming-write-entry to be more reusableNguyễn Thái Ngọc Duy, Mar 4, 2012
  25. 03/10 cat-file: use streaming interface to print blobsNguyễn Thái Ngọc Duy, Mar 4, 2012
  26. Junio C HamanoMar 4, 2012
  27. Nguyen Thai Ngoc DuyMar 5, 2012
  28. 04/10 parse_object: special code path for blobs to avoid putting whole object in memoryNguyễn Thái Ngọc Duy, Mar 4, 2012
  29. 05/10 show: use streaming interface for showing blobsNguyễn Thái Ngọc Duy, Mar 4, 2012
  30. 06/10 index-pack: split second pass obj handling into own functionNguyễn Thái Ngọc Duy, Mar 4, 2012
  31. 07/10 index-pack: reduce memory usage when the pack has large blobsNguyễn Thái Ngọc Duy, Mar 4, 2012
  32. 08/10 pack-check: do not unpack blobsNguyễn Thái Ngọc Duy, Mar 4, 2012
  33. 09/10 archive: support streaming large files to a tar archiveNguyễn Thái Ngọc Duy, Mar 4, 2012
  34. 10/10 fsck: use streaming interface for writing lost-found blobsNguyễn Thái Ngọc Duy, Mar 4, 2012
  35. 00/11 Large blob fixesNguyễn Thái Ngọc Duy, Mar 5, 2012
  36. 01/11 Add more large blob test casesNguyễn Thái Ngọc Duy, Mar 5, 2012
  37. 02/11 streaming: make streaming-write-entry to be more reusableNguyễn Thái Ngọc Duy, Mar 5, 2012
  38. 03/11 cat-file: use streaming interface to print blobsNguyễn Thái Ngọc Duy, Mar 5, 2012
  39. 04/11 parse_object: special code path for blobs to avoid putting whole object in memoryNguyễn Thái Ngọc Duy, Mar 5, 2012
  40. Junio C HamanoMar 6, 2012
  41. 05/11 show: use streaming interface for showing blobsNguyễn Thái Ngọc Duy, Mar 5, 2012
  42. 06/11 index-pack: split second pass obj handling into own functionNguyễn Thái Ngọc Duy, Mar 5, 2012
  43. 07/11 index-pack: reduce memory usage when the pack has large blobsNguyễn Thái Ngọc Duy, Mar 5, 2012
  44. 08/11 pack-check: do not unpack blobsNguyễn Thái Ngọc Duy, Mar 5, 2012
  45. 09/11 archive: support streaming large files to a tar archiveNguyễn Thái Ngọc Duy, Mar 5, 2012
  46. Junio C HamanoMar 6, 2012
  47. 10/11 fsck: use streaming interface for writing lost-found blobsNguyễn Thái Ngọc Duy, Mar 5, 2012
  48. 11/11 update-server-info: respect core.bigfilethresholdNguyễn Thái Ngọc Duy, Mar 5, 2012

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.