git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 04/16] vcs-svn: Teach line_buffer to handle multiple input files

From
Jonathan Nieder <jrnieder@gmail.com>
Date
Oct 11, 2010, 02:41 UTC
Message-ID
<20101011024106.GD1553@burratino>
In-Reply-To
<20101011023435.GA706@burratino>

Collect the line_buffer state in a newly public line_buffer struct. Callers can use multiple line_buffers to manage input from multiple files at a time.

The Subversion-format delta applier will use this to stream a delta and the preimage it applies to at the same time.

Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
---
 vcs-svn/line_buffer.txt |    5 +++--
 vcs-svn/fast_export.c   |    6 +++---
 vcs-svn/fast_export.h   |    5 ++++-
 vcs-svn/line_buffer.c   |   20 ++++++++------------
 vcs-svn/line_buffer.h   |   14 +++++++-------
 vcs-svn/svndump.c       |   29 ++++++++++++++++-------------
 test-line-buffer.c      |   17 +++++++++--------
 7 files changed, 50 insertions(+), 46 deletions(-)
diff --git a/vcs-svn/line_buffer.txt b/vcs-svn/line_buffer.txt
index 8906fb1..f8eaa4d 100644
--- a/vcs-svn/line_buffer.txt
+++ b/vcs-svn/line_buffer.txt
@@ -14,14 +14,15 @@ Calling sequence
 
 The calling program:
 
+ - initializes a `struct line_buffer` to LINE_BUFFER_INIT
  - specifies a file to read with `buffer_init`
  - processes input with `buffer_read_line`, `buffer_read_string`,
    `buffer_skip_bytes`, and `buffer_copy_bytes`
  - closes the file with `buffer_deinit`, perhaps to start over and
    read another file.
 
-Before exiting, the caller can use `buffer_reset` to deallocate
-resources for the benefit of profiling tools.
+When finished, the caller can use `buffer_reset` to deallocate
+resources.
 
 Functions
 ---------
diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c
index 6cfa256..260cf50 100644
--- a/vcs-svn/fast_export.c
+++ b/vcs-svn/fast_export.c
@@ -63,14 +63,14 @@ void fast_export_commit(uint32_t revision, uint32_t author, char *log,
 	printf("progress Imported commit %"PRIu32".\n\n", revision);
 }
 
-void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len)
+void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len, struct line_buffer *input)
 {
 	if (mode == REPO_MODE_LNK) {
 		/* svn symlink blobs start with "link " */
-		buffer_skip_bytes(5);
+		buffer_skip_bytes(input, 5);
 		len -= 5;
 	}
 	printf("blob\nmark :%"PRIu32"\ndata %"PRIu32"\n", mark, len);
-	buffer_copy_bytes(len);
+	buffer_copy_bytes(input, len);
 	fputc('\n', stdout);
 }
diff --git a/vcs-svn/fast_export.h b/vcs-svn/fast_export.h
index 2aaaea5..054e7d5 100644
--- a/vcs-svn/fast_export.h
+++ b/vcs-svn/fast_export.h
@@ -1,11 +1,14 @@
 #ifndef FAST_EXPORT_H_
 #define FAST_EXPORT_H_
 
+#include "line_buffer.h"
+
 void fast_export_delete(uint32_t depth, uint32_t *path);
 void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode,
 			uint32_t mark);
 void fast_export_commit(uint32_t revision, uint32_t author, char *log,
 			uint32_t uuid, uint32_t url, unsigned long timestamp);
-void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len);
+void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len,
+		      struct line_buffer *input);
 
 #endif
diff --git a/vcs-svn/line_buffer.c b/vcs-svn/line_buffer.c
index e7bc230..806932b 100644
--- a/vcs-svn/line_buffer.c
+++ b/vcs-svn/line_buffer.c
@@ -8,20 +8,16 @@
 #include "strbuf.h"
 
 #define COPY_BUFFER_LEN 4096
-static struct line_buffer buf_ = LINE_BUFFER_INIT;
-static struct line_buffer *buf;
 
-int buffer_init(const char *filename)
+int buffer_init(struct line_buffer *buf, const char *filename)
 {
-	buf = &buf_;
-
 	buf->infile = filename ? fopen(filename, "r") : stdin;
 	if (!buf->infile)
 		return -1;
 	return 0;
 }
 
-int buffer_deinit(void)
+int buffer_deinit(struct line_buffer *buf)
 {
 	int err;
 	if (buf->infile == stdin)
@@ -32,7 +28,7 @@ int buffer_deinit(void)
 }
 
 /* Read a line without trailing newline. */
-char *buffer_read_line(void)
+char *buffer_read_line(struct line_buffer *buf)
 {
 	char *end;
 	if (!fgets(buf->line_buffer, sizeof(buf->line_buffer), buf->infile))
@@ -53,14 +49,14 @@ char *buffer_read_line(void)
 	return buf->line_buffer;
 }
 
-char *buffer_read_string(uint32_t len)
+char *buffer_read_string(struct line_buffer *buf, uint32_t len)
 {
 	strbuf_reset(&buf->blob_buffer);
 	strbuf_fread(&buf->blob_buffer, len, buf->infile);
 	return ferror(buf->infile) ? NULL : buf->blob_buffer.buf;
 }
 
-void buffer_copy_bytes(uint32_t len)
+void buffer_copy_bytes(struct line_buffer *buf, uint32_t len)
 {
 	char byte_buffer[COPY_BUFFER_LEN];
 	uint32_t in;
@@ -70,13 +66,13 @@ void buffer_copy_bytes(uint32_t len)
 		len -= in;
 		fwrite(byte_buffer, 1, in, stdout);
 		if (ferror(stdout)) {
-			buffer_skip_bytes(len);
+			buffer_skip_bytes(buf, len);
 			return;
 		}
 	}
 }
 
-void buffer_skip_bytes(uint32_t len)
+void buffer_skip_bytes(struct line_buffer *buf, uint32_t len)
 {
 	char byte_buffer[COPY_BUFFER_LEN];
 	uint32_t in;
@@ -87,7 +83,7 @@ void buffer_skip_bytes(uint32_t len)
 	}
 }
 
-void buffer_reset(void)
+void buffer_reset(struct line_buffer *buf)
 {
 	strbuf_release(&buf->blob_buffer);
 }
diff --git a/vcs-svn/line_buffer.h b/vcs-svn/line_buffer.h
index 4ae1133..fb37390 100644
--- a/vcs-svn/line_buffer.h
+++ b/vcs-svn/line_buffer.h
@@ -12,12 +12,12 @@ struct line_buffer {
 };
 #define LINE_BUFFER_INIT {"", STRBUF_INIT, NULL}
 
-int buffer_init(const char *filename);
-int buffer_deinit(void);
-char *buffer_read_line(void);
-char *buffer_read_string(uint32_t len);
-void buffer_copy_bytes(uint32_t len);
-void buffer_skip_bytes(uint32_t len);
-void buffer_reset(void);
+int buffer_init(struct line_buffer *buf, const char *filename);
+int buffer_deinit(struct line_buffer *buf);
+char *buffer_read_line(struct line_buffer *buf);
+char *buffer_read_string(struct line_buffer *buf, uint32_t len);
+void buffer_copy_bytes(struct line_buffer *buf, uint32_t len);
+void buffer_skip_bytes(struct line_buffer *buf, uint32_t len);
+void buffer_reset(struct line_buffer *buf);
 
 #endif
diff --git a/vcs-svn/svndump.c b/vcs-svn/svndump.c
index 53d0215..3bba0fe 100644
--- a/vcs-svn/svndump.c
+++ b/vcs-svn/svndump.c
@@ -30,6 +30,8 @@
 /* Create memory pool for log messages */
 obj_pool_gen(log, char, 4096)
 
+static struct line_buffer input = LINE_BUFFER_INIT;
+
 static char* log_copy(uint32_t length, char *log)
 {
 	char *buffer;
@@ -113,14 +115,14 @@ static void read_props(void)
 	uint32_t key = ~0;
 	char *val = NULL;
 	char *t;
-	while ((t = buffer_read_line()) && strcmp(t, "PROPS-END")) {
+	while ((t = buffer_read_line(&input)) && strcmp(t, "PROPS-END")) {
 		if (!strncmp(t, "K ", 2)) {
 			len = atoi(&t[2]);
-			key = pool_intern(buffer_read_string(len));
-			buffer_read_line();
+			key = pool_intern(buffer_read_string(&input, len));
+			buffer_read_line(&input);
 		} else if (!strncmp(t, "V ", 2)) {
 			len = atoi(&t[2]);
-			val = buffer_read_string(len);
+			val = buffer_read_string(&input, len);
 			if (key == keys.svn_log) {
 				/* Value length excludes terminating nul. */
 				rev_ctx.log = log_copy(len + 1, val);
@@ -135,7 +137,7 @@ static void read_props(void)
 				node_ctx.type = REPO_MODE_LNK;
 			}
 			key = ~0;
-			buffer_read_line();
+			buffer_read_line(&input);
 		}
 	}
 }
@@ -177,9 +179,10 @@ static void handle_node(void)
 		node_ctx.type = node_ctx.srcMode;
 
 	if (node_ctx.mark)
-		fast_export_blob(node_ctx.type, node_ctx.mark, node_ctx.textLength);
+		fast_export_blob(node_ctx.type,
+				 node_ctx.mark, node_ctx.textLength, &input);
 	else if (node_ctx.textLength != LENGTH_UNKNOWN)
-		buffer_skip_bytes(node_ctx.textLength);
+		buffer_skip_bytes(&input, node_ctx.textLength);
 }
 
 static void handle_revision(void)
@@ -198,7 +201,7 @@ void svndump_read(const char *url)
 	uint32_t key;
 
 	reset_dump_ctx(pool_intern(url));
-	while ((t = buffer_read_line())) {
+	while ((t = buffer_read_line(&input))) {
 		val = strstr(t, ": ");
 		if (!val)
 			continue;
@@ -250,7 +253,7 @@ void svndump_read(const char *url)
 			node_ctx.propLength = atoi(val);
 		} else if (key == keys.content_length) {
 			len = atoi(val);
-			buffer_read_line();
+			buffer_read_line(&input);
 			if (active_ctx == REV_CTX) {
 				read_props();
 			} else if (active_ctx == NODE_CTX) {
@@ -258,7 +261,7 @@ void svndump_read(const char *url)
 				active_ctx = REV_CTX;
 			} else {
 				fprintf(stderr, "Unexpected content length header: %"PRIu32"\n", len);
-				buffer_skip_bytes(len);
+				buffer_skip_bytes(&input, len);
 			}
 		}
 	}
@@ -270,7 +273,7 @@ void svndump_read(const char *url)
 
 void svndump_init(const char *filename)
 {
-	buffer_init(filename);
+	buffer_init(&input, filename);
 	repo_init();
 	reset_dump_ctx(~0);
 	reset_rev_ctx(0);
@@ -285,7 +288,7 @@ void svndump_deinit(void)
 	reset_dump_ctx(~0);
 	reset_rev_ctx(0);
 	reset_node_ctx(NULL);
-	if (buffer_deinit())
+	if (buffer_deinit(&input))
 		fprintf(stderr, "Input error\n");
 	if (ferror(stdout))
 		fprintf(stderr, "Output error\n");
@@ -294,7 +297,7 @@ void svndump_deinit(void)
 void svndump_reset(void)
 {
 	log_reset();
-	buffer_reset();
+	buffer_reset(&input);
 	repo_reset();
 	reset_dump_ctx(~0);
 	reset_rev_ctx(0);
diff --git a/test-line-buffer.c b/test-line-buffer.c
index c11bf7f..f9af892 100644
--- a/test-line-buffer.c
+++ b/test-line-buffer.c
@@ -22,25 +22,26 @@ static uint32_t strtouint32(const char *s)
 
 int main(int argc, char *argv[])
 {
+	struct line_buffer buf = LINE_BUFFER_INIT;
 	char *s;
 
 	if (argc != 1)
 		usage("test-line-buffer < input.txt");
-	if (buffer_init(NULL))
+	if (buffer_init(&buf, NULL))
 		die_errno("open error");
-	while ((s = buffer_read_line())) {
-		s = buffer_read_string(strtouint32(s));
+	while ((s = buffer_read_line(&buf))) {
+		s = buffer_read_string(&buf, strtouint32(s));
 		fputs(s, stdout);
 		fputc('\n', stdout);
-		buffer_skip_bytes(1);
-		if (!(s = buffer_read_line()))
+		buffer_skip_bytes(&buf, 1);
+		if (!(s = buffer_read_line(&buf)))
 			break;
-		buffer_copy_bytes(strtouint32(s) + 1);
+		buffer_copy_bytes(&buf, strtouint32(s) + 1);
 	}
-	if (buffer_deinit())
+	if (buffer_deinit(&buf))
 		die("input error");
 	if (ferror(stdout))
 		die("output error");
-	buffer_reset();
+	buffer_reset(&buf);
 	return 0;
 }

-- 
1.7.2.3
Previous: Jonathan NiederNext: Jonathan Nieder
Message 49 of 79 in “Resurrect rr/svn-export”
  1. 0/8 Resurrect rr/svn-exportRamkumar Ramachandra, Jul 15, 2010
  2. 1/8 Export parse_date_basic() to convert a date string to timestampRamkumar Ramachandra, Jul 15, 2010
  3. Jonathan NiederJul 15, 2010
  4. Junio C HamanoJul 15, 2010
  5. 2/8 Introduce vcs-svn libRamkumar Ramachandra, Jul 15, 2010
  6. Jonathan NiederJul 15, 2010
  7. Ramkumar RamachandraJul 15, 2010
  8. 3/8 Add memory pool libraryRamkumar Ramachandra, Jul 15, 2010
  9. Jonathan NiederJul 15, 2010
  10. Ramkumar RamachandraJul 15, 2010
  11. 4/8 Add treap implementationRamkumar Ramachandra, Jul 15, 2010
  12. Jonathan NiederJul 15, 2010
  13. Ramkumar RamachandraJul 15, 2010
  14. 5/8 Add string-specific memory poolRamkumar Ramachandra, Jul 15, 2010
  15. 6/8 Add stream helper libraryRamkumar Ramachandra, Jul 15, 2010
  16. Jonathan NiederJul 15, 2010
  17. 7/8 Add infrastructure to write revisions in fast-export formatRamkumar Ramachandra, Jul 15, 2010
  18. Jonathan NiederJul 15, 2010
  19. 8/8 Add SVN dump parserRamkumar Ramachandra, Jul 15, 2010
  20. Jonathan NiederJul 15, 2010
  21. Jonathan NiederJul 15, 2010
  22. Jonathan NiederJul 16, 2010
  23. 3/9 Add memory pool libraryJonathan Nieder, Jul 16, 2010
  24. 4/9 Add treap implementationJonathan Nieder, Jul 16, 2010
  25. Jonathan NiederJul 16, 2010
  26. 0/10 rr/svn-export rerollJonathan Nieder, Aug 9, 2010
  27. 01/10 Export parse_date_basic() to convert a date string to timestampJonathan Nieder, Aug 9, 2010
  28. 02/10 Introduce vcs-svn libJonathan Nieder, Aug 9, 2010
  29. 03/10 Add memory pool libraryJonathan Nieder, Aug 9, 2010
  30. 04/10 Add treap implementationJonathan Nieder, Aug 9, 2010
  31. Junio C HamanoAug 12, 2010
  32. Jonathan NiederAug 12, 2010
  33. Jonathan NiederAug 12, 2010
  34. Junio C HamanoAug 12, 2010
  35. 05/10 Add string-specific memory poolJonathan Nieder, Aug 9, 2010
  36. Junio C HamanoAug 12, 2010
  37. Jonathan NiederAug 12, 2010
  38. 06/10 Add stream helper libraryJonathan Nieder, Aug 9, 2010
  39. 07/10 Infrastructure to write revisions in fast-export formatJonathan Nieder, Aug 9, 2010
  40. 08/10 SVN dump parserJonathan Nieder, Aug 9, 2010
  41. Junio C HamanoAug 12, 2010
  42. PATCH 09/10] Update svn-fe manualJonathan Nieder, Aug 9, 2010
  43. 10/10 svn-fe manual: Clarify warning about deltas in dump filesJonathan Nieder, Aug 9, 2010
  44. Ramkumar RamachandraAug 10, 2010
  45. Jonathan NiederAug 11, 2010
  46. 00/16 svn delta applierJonathan Nieder, Oct 11, 2010
  47. 01/16 vcs-svn: Eliminate global byte_buffer[] arrayJonathan Nieder, Oct 11, 2010
  48. 03/16 vcs-svn: Collect line_buffer data in a structJonathan Nieder, Oct 11, 2010
  49. 04/16 vcs-svn: Teach line_buffer to handle multiple input filesJonathan Nieder, Oct 11, 2010
  50. 05/16 vcs-svn: Make buffer_skip_bytes() report partial readsJonathan Nieder, Oct 11, 2010
  51. 06/16 vcs-svn: Improve support for reading large filesJonathan Nieder, Oct 11, 2010
  52. 07/16 vcs-svn: Add binary-safe read() functionJonathan Nieder, Oct 11, 2010
  53. 08/16 vcs-svn: Let callers peek ahead to find stream endJonathan Nieder, Oct 11, 2010
  54. 09/16 vcs-svn: Allow input errors to be detected earlyJonathan Nieder, Oct 11, 2010
  55. 10/16 vcs-svn: Allow character-oriented inputJonathan Nieder, Oct 11, 2010
  56. 11/16 vcs-svn: Add code to maintain a sliding view of a fileJonathan Nieder, Oct 11, 2010
  57. 12/16 vcs-svn: Learn to parse variable-length integersJonathan Nieder, Oct 11, 2010
  58. 13/16 vcs-svn: Learn to check for SVN\0 magicJonathan Nieder, Oct 11, 2010
  59. 14/16 compat: helper for detecting unsigned overflowJonathan Nieder, Oct 11, 2010
  60. 15/16 t9010 (svn-fe): Eliminate dependency on svn perl bindingsJonathan Nieder, Oct 11, 2010
  61. 02/16 vcs-svn: Replace buffer_read_string() memory pool with a strbufJonathan Nieder, Oct 11, 2010
  62. vcs-svn: Add svn delta parserJonathan Nieder, Oct 11, 2010
  63. 0/11 Building up the delta parserJonathan Nieder, Oct 13, 2010
  64. 01/11 fixup! vcs-svn: Learn to parse variable-length integersJonathan Nieder, Oct 13, 2010
  65. 02/11 vcs-svn: Skeleton of an svn delta parserJonathan Nieder, Oct 13, 2010
  66. 03/11 vcs-svn: Read the preimage while applying deltasJonathan Nieder, Oct 13, 2010
  67. Sam VilainOct 14, 2010
  68. Jonathan NiederOct 14, 2010
  69. 04/11 vcs-svn: Read inline data from deltasJonathan Nieder, Oct 13, 2010
  70. 05/11 vcs-svn: Read instructions from deltasJonathan Nieder, Oct 13, 2010
  71. 06/11 vcs-svn: Implement copyfrom_data delta instructionJonathan Nieder, Oct 13, 2010
  72. 07/11 vcs-svn: Check declared number of output bytesJonathan Nieder, Oct 13, 2010
  73. 08/11 vcs-svn: Reject deltas that do not consume all inline dataJonathan Nieder, Oct 13, 2010
  74. 09/11 vcs-svn: Let deltas use data from postimageJonathan Nieder, Oct 13, 2010
  75. 10/11 vcs-svn: Reject deltas that read past end of preimageJonathan Nieder, Oct 13, 2010
  76. 11/11 vcs-svn: Allow deltas to copy from preimageJonathan Nieder, Oct 13, 2010
  77. 11/11 vcs-svn: Allow deltas to copy from preimageJonathan Nieder, Oct 13, 2010
  78. Ramkumar RamachandraOct 18, 2010
  79. Jonathan NiederOct 18, 2010

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.