git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 4/8] vcs-svn: Introduce fd_buffer routines

From
Jonathan Nieder <jrnieder@gmail.com>
Date
Nov 20, 2010, 19:26 UTC
Message-ID
<20101120192648.GE17823@burratino>
In-Reply-To
<20101120192153.GA17823@burratino>

The fd_buffer library is perhaps poorly named, because it does not manage its own buffer. These functions provide the relevant functionality from line_buffer for low-level, unbuffered (file descriptor) I/O.

The purpose is to preserve convenience while avoiding deadlock that could occur from too greedily reading ahead in the cat-blob-fd pipe. (An alternative approach to achieve the same effect would be to set the O_NONBLOCK flag on the pipe fd.)

Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
---
 Makefile              |    5 ++-
 vcs-svn/fast_export.c |   45 ++++++++++------------------------
 vcs-svn/fd_buffer.c   |   65 +++++++++++++++++++++++++++++++++++++++++++++++++
 vcs-svn/fd_buffer.h   |   19 ++++++++++++++
 4 files changed, 100 insertions(+), 34 deletions(-)
 create mode 100644 vcs-svn/fd_buffer.c
 create mode 100644 vcs-svn/fd_buffer.h
diff --git a/Makefile b/Makefile
index aa10288..844e3b4 100644
--- a/Makefile
+++ b/Makefile
@@ -1760,7 +1760,7 @@ ifndef NO_CURL
 endif
 XDIFF_OBJS = xdiff/xdiffi.o xdiff/xprepare.o xdiff/xutils.o xdiff/xemit.o \
 	xdiff/xmerge.o xdiff/xpatience.o
-VCSSVN_OBJS = vcs-svn/string_pool.o vcs-svn/line_buffer.o \
+VCSSVN_OBJS = vcs-svn/string_pool.o vcs-svn/line_buffer.o vcs-svn/fd_buffer.o \
 	vcs-svn/repo_tree.o vcs-svn/fast_export.o vcs-svn/svndump.o \
 	vcs-svn/sliding_window.o vcs-svn/svndiff.o
 OBJECTS := $(GIT_OBJS) $(XDIFF_OBJS) $(VCSSVN_OBJS)
@@ -1889,7 +1889,8 @@ xdiff-interface.o $(XDIFF_OBJS): \
 $(VCSSVN_OBJS): \
 	vcs-svn/obj_pool.h vcs-svn/trp.h vcs-svn/string_pool.h \
 	vcs-svn/line_buffer.h vcs-svn/repo_tree.h vcs-svn/fast_export.h \
-	vcs-svn/sliding_window.h vcs-svn/svndump.h vcs-svn/svndiff.h
+	vcs-svn/sliding_window.h vcs-svn/svndump.h vcs-svn/svndiff.h \
+	vcs-svn/fd_buffer.h
 endif
 
 exec_cmd.s exec_cmd.o: EXTRA_CPPFLAGS = \
diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c
index 0de498b..ebaab72 100644
--- a/vcs-svn/fast_export.c
+++ b/vcs-svn/fast_export.c
@@ -6,6 +6,7 @@
 #include "git-compat-util.h"
 #include "fast_export.h"
 #include "line_buffer.h"
+#include "fd_buffer.h"
 #include "repo_tree.h"
 #include "string_pool.h"
 #include "strbuf.h"
@@ -73,7 +74,7 @@ static int ends_with(const char *s, size_t len, const char *suffix)
 	return !memcmp(s + len - suffixlen, suffix, suffixlen);
 }
 
-static int parse_cat_response_line(const char *header, size_t *len)
+static int parse_cat_response_line(const char *header, off_t *len)
 {
 	size_t headerlen = strlen(header);
 	const char *type;
@@ -95,45 +96,25 @@ static int parse_cat_response_line(const char *header, size_t *len)
 static struct strbuf response_line = STRBUF_INIT;
 static const char *get_response_line(void)
 {
-	/*
-	 * NEEDSWORK: Does not actually need to read one byte at a time.
-	 * Some platforms have O_NONBLOCK.  On others we could read 8 chars
-	 * at a time until a potential appearance of " blob ".
-	 */
 	strbuf_reset(&response_line);
-	for (;;) {
-		char buf[1];
-		if (xread(REPORT_FILENO, buf, 1) < 0) {
-			error("cannot read cat-blob result: %s", strerror(errno));
-			return NULL;
-		}
-		if (*buf == '\n')
-			return response_line.buf;
-		strbuf_addch(&response_line, *buf);
-	}
+	if (fd_read_line(&response_line, REPORT_FILENO))
+		return NULL;
+	return response_line.buf;
 }
 
-static int copy_bytes(FILE *out, size_t len)
+static int copy_bytes(FILE *out, off_t len)
 {
-	char buf[4096];
-	ssize_t nread;
-	for (; len; len -= nread) {
-		nread = xread(REPORT_FILENO, buf,
-					len < sizeof(buf) ? len : sizeof(buf));
-		if (nread < 0)
-			return error("cannot copy cat-blob result: %s",
-					strerror(errno));
-		if (!nread)
-			return error("0-length read...");
-		if (fwrite(buf, 1, nread, out) != nread)
-			return error("cannot write cat-blob results: %s",
-					strerror(errno));
-	}
+	off_t ret = fd_copy_bytes(out, REPORT_FILENO, len);
+	if (!ret)
+		return error("read error: file ends early");
+	if (ret <= 0)
+		return error("cannot copy cat-blob result");
+	return 0;
 }
 
 static int fast_export_save_blob(FILE *out)
 {
-	size_t len = len;
+	off_t len = len;
 	const char *header, *tail;
 
 	header = get_response_line();
diff --git a/vcs-svn/fd_buffer.c b/vcs-svn/fd_buffer.c
new file mode 100644
index 0000000..3d2c1a1
--- /dev/null
+++ b/vcs-svn/fd_buffer.c
@@ -0,0 +1,65 @@
+/*
+ * Licensed under a two-clause BSD-style license.
+ * See LICENSE for details.
+ */
+
+#include "git-compat-util.h"
+#include "fd_buffer.h"
+#include "strbuf.h"
+
+/* Read a line without trailing newline. */
+int fd_read_line(struct strbuf *line, int fd)
+{
+	/*
+	 * NEEDSWORK: Does not actually need to read one byte at a time.
+	 * Some platforms have O_NONBLOCK.  On others we could read
+	 * several chars at a time until an appearance of a string known
+	 * to belong in the line (e.g., " blob ").
+	 */
+	for (;;) {
+		char buf[1];
+		if (xread(fd, buf, 1) < 0)
+			return error("cannot read line: %s", strerror(errno));
+		if (*buf == '\n')
+			return 0;
+		strbuf_addch(line, *buf);
+	}
+}
+
+int fd_read_binary(struct strbuf *buf, size_t len, int fd)
+{
+	char *p, *end;
+	strbuf_grow(buf, len);
+	p = buf->buf + buf->len;
+	end = p + len;
+
+	while (p != end) {
+		ssize_t nread = xread(fd, p, end - p);
+		if (nread < 0)
+			return error("read error: %s", strerror(errno));
+		if (!nread)	/* end of file */
+			break;
+		p += nread;
+	}
+	strbuf_setlen(buf, p - buf->buf);
+	return 0;
+}
+
+off_t fd_copy_bytes(FILE *out, int fd, off_t len)
+{
+	off_t total = 0;
+	while (len) {
+		char buf[4096];
+		ssize_t nread = xread(fd, buf,
+					len < sizeof(buf) ? len : sizeof(buf));
+		if (nread < 0)
+			return error("read error: %s", strerror(errno));
+		if (!nread)
+			return 0;
+		if (out && fwrite(buf, 1, nread, out) != nread)
+			return error("write error: %s", strerror(errno));
+		total += nread;
+		len -= nread;
+	}
+	return total;
+}
diff --git a/vcs-svn/fd_buffer.h b/vcs-svn/fd_buffer.h
new file mode 100644
index 0000000..9434ac3
--- /dev/null
+++ b/vcs-svn/fd_buffer.h
@@ -0,0 +1,19 @@
+#ifndef FD_BUFFER_H
+#define FD_BUFFER_H
+
+#include "strbuf.h"
+
+/* Low-level input helpers.  Usually line_buffer is a better choice. */
+
+extern int fd_read_line(struct strbuf *line, int fd);
+extern int fd_read_binary(struct strbuf *buf, size_t len, int fd);
+
+/* returns 0 for end of file */
+extern off_t fd_copy_bytes(FILE *out, int fd, off_t len);
+
+static inline off_t fd_skip_bytes(int fd, off_t len)
+{
+	return fd_copy_bytes(NULL, fd, len);
+}
+
+#endif

-- 
1.7.2.3
Previous: Jonathan NiederNext: Jonathan Nieder
Message 31 of 40 in “svn-fe: recognize v3 dumps”
  1. 0/2 svn-fe: recognize v3 dumpsJonathan Nieder, Nov 18, 2010
  2. 1/2 vcs-svn: Error out for v3 dumpsJonathan Nieder, Nov 18, 2010
  3. 2/2 vcs-svn: Allow simple v3 dumps (no deltas yet)Jonathan Nieder, Nov 18, 2010
  4. 0/15 svn-fe: support for property deltas (but not text deltas yet)Jonathan Nieder, Nov 20, 2010
  5. 01/15 vcs-svn: Check for errors from open()Jonathan Nieder, Nov 20, 2010
  6. 02/15 vcs-svn: Eliminate node_ctx.srcRev globalJonathan Nieder, Nov 20, 2010
  7. 03/15 vcs-svn: Eliminate node_ctx.mark globalJonathan Nieder, Nov 20, 2010
  8. 04/15 vcs-svn: Unclutter handle_node by introducing have_props varJonathan Nieder, Nov 20, 2010
  9. 05/15 vcs-svn: Use mark to indicate nodes with included textJonathan Nieder, Nov 20, 2010
  10. 06/15 vcs-svn: handle_node: Handle deletion case earlyJonathan Nieder, Nov 20, 2010
  11. 07/15 vcs-svn: Replace = Delete + AddJonathan Nieder, Nov 20, 2010
  12. 08/15 vcs-svn: Combine repo_replace and repo_modify functionsJonathan Nieder, Nov 20, 2010
  13. 09/15 vcs-svn: Delay read of per-path propertiesJonathan Nieder, Nov 20, 2010
  14. 10/15 vcs-svn: Reject path nodes without Node-actionJonathan Nieder, Nov 20, 2010
  15. Jonathan NiederNov 20, 2010
  16. 11/15 vcs-svn: More dump format sanity checksJonathan Nieder, Nov 20, 2010
  17. Jonathan NiederNov 30, 2010
  18. 0/2 intermittent repo_tree corruptionJonathan Nieder, Dec 5, 2010
  19. 1/2 treap: make treap_insert return inserted nodeJonathan Nieder, Dec 5, 2010
  20. vcs-svn: Allow change nodes for root of tree (/)Jonathan Nieder, Dec 6, 2010
  21. Jonathan NiederDec 6, 2010
  22. 12/15 vcs-svn: Make source easier to read on small screensJonathan Nieder, Nov 20, 2010
  23. 13/15 vcs-svn: Split off function for handling of individual propertiesJonathan Nieder, Nov 20, 2010
  24. 14/15 vcs-svn: Sharpen parsing of property linesJonathan Nieder, Nov 20, 2010
  25. 15/15 vcs-svn: Implement Prop-delta handlingJonathan Nieder, Nov 20, 2010
  26. 0/8 svn-fe: support for text deltasJonathan Nieder, Nov 20, 2010
  27. 1/8 svn-fe: Prepare for strbuf useJonathan Nieder, Nov 20, 2010
  28. 2/8 vcs-svn: Internal fast_export_save_blob helperJonathan Nieder, Nov 20, 2010
  29. 3/8 vcs-svn: Introduce repo_read_path to check the content at a pathJonathan Nieder, Nov 20, 2010
  30. Jonathan NiederMar 6, 2011
  31. 4/8 vcs-svn: Introduce fd_buffer routinesJonathan Nieder, Nov 20, 2010
  32. 5/8 vcs-svn: Read delta preimage from file descriptorJonathan Nieder, Nov 20, 2010
  33. 6/8 vcs-svn: Let caller set up sliding window for delta preimageJonathan Nieder, Nov 20, 2010
  34. Jonathan NiederNov 20, 2010
  35. 7/8 vcs-svn: Teach line_buffer about temporary filesJonathan Nieder, Nov 20, 2010
  36. 8/8 vcs-svn: Implement text-delta handlingJonathan Nieder, Nov 20, 2010
  37. 10/8 vcs-svn: Consume whole preimage when applying deltasJonathan Nieder, Dec 4, 2010
  38. 9/8 svn-fe: Test script for handling of dumps with --deltasJonathan Nieder, Nov 20, 2010
  39. Jonathan NiederDec 4, 2010
  40. 2/2 vcs-svn: fix intermittent repo_tree corruptionJonathan Nieder, Dec 5, 2010

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.