{"thread":{"id":"1991","subject":"[PATCH] Fix git+ssh's indefinite halts during long fetches","startedAt":"2005-10-01T18:39:42Z","lastAt":"2005-10-01T18:39:42Z","messageCount":1,"participants":["Dan Aloni"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"9593","messageId":"20051001183942.GA2099@localdomain","threadId":"1991","inReplyTo":null,"subject":"[PATCH] Fix git+ssh's indefinite halts during long fetches","fromName":"Dan Aloni","fromEmail":"da-x@monatomic.org","sentAt":"2005-10-01T18:39:42Z","receivedAt":"2005-10-01T18:39:42Z","isPatch":true,"sender":{"key":"da-x@monatomic.org","avatar":null},"body":"The problem with the old implementation is that the socket input buffers\nget full and then both ends halt waiting for each other. This take cares\nof it, by buffering at the fetching side while still trying to send. It's\nquite hackish but does the work (I managed to locally fetch the kernel with\nits ~85000 objects that sum up to 250MB).\n\nSigned-off-by: Dan Aloni <da-x@monatomic.org>\n\n---\ncommit 958b0c00525fb63276430783dccb18316cec73c9\ntree 4003cf1069e9ac4bbb8aade512047dd41a466ae2\nparent 60fb5b2c4d9e26204f480f8a18ae1ff0051a6440\nauthor Dan Aloni <da-x@monatomic.org> Sat, 01 Oct 2005 21:38:25 +0300\ncommitter Dan Aloni <da-x@monatomic.org> Sat, 01 Oct 2005 21:38:25 +0300\n\n cache.h     |   18 +++++++-\n sha1_file.c |  129 ++++++++++++++++++++++++++++++++++++++++++++++++-----------\n ssh-fetch.c |   76 ++++++++++++++++++++++++-----------\n 3 files changed, 174 insertions(+), 49 deletions(-)\n\ndiff --git a/cache.h b/cache.h\n--- a/cache.h\n+++ b/cache.h\n@@ -214,8 +214,22 @@ extern int check_sha1_signature(const un\n /* Read a tree into the cache */\n extern int read_tree(void *buffer, unsigned long size, int stage, const char **paths);\n \n-extern int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,\n-\t\t\t      size_t bufsize, size_t *bufposn);\n+struct input_segment {\n+\tunsigned char *buffer;\n+\tsize_t size;\n+\n+\tstruct input_segment *next;\n+\tstruct input_segment *prev;\n+};\n+\n+struct input_buffer {\n+\tstruct input_segment *first;\n+\tstruct input_segment *last;\n+};\n+\n+extern int read_input_buffers(int block, int fd_in, struct input_buffer *inputbuffer);\n+extern int write_sha1_from_fd(const unsigned char *sha1, int fd, struct input_buffer *inputbuffer);\n+\n extern int write_sha1_to_fd(int fd, const unsigned char *sha1);\n \n extern int has_sha1_pack(const unsigned char *sha1);\ndiff --git a/sha1_file.c b/sha1_file.c\n--- a/sha1_file.c\n+++ b/sha1_file.c\n@@ -1413,11 +1413,88 @@ int write_sha1_to_fd(int fd, const unsig\n \treturn 0;\n }\n \n-int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,\n-\t\t       size_t bufsize, size_t *bufposn)\n+int cut_last_input_segment(struct input_buffer *inputbuffer, struct input_segment *segment, size_t size_left)\n {\n-\tchar *filename = sha1_file_name(sha1);\n+\tif (size_left) {\n+\t\tmemmove(segment->buffer, segment->buffer + segment->size - size_left, size_left);\n+\t\tsegment->size = size_left;\n+\t\treturn 1;\n+\t}\n+\t\n+\tif (segment == inputbuffer->first) \n+\t\tinputbuffer->first = NULL;\n+\n+\tfree(segment->buffer);\n+\tsegment = segment->prev;\n+\tfree(inputbuffer->last);\n+\tinputbuffer->last = segment;\n+\treturn 0;\n+}\n+\n+int read_input_buffers(int block, int fd_in, struct input_buffer *inputbuffer)\n+{\n+\tunsigned long old_flags = 0;\n+\tsize_t bufsize;\n+\tint readsize;\n+\tstruct input_segment *segment;\n+\tint ret = 0;\n+\tunsigned char *buf;\n+\n+\tif (!block) {\n+\t\told_flags = fcntl(fd_in, F_GETFL);\n+\t\tfcntl(fd_in, F_SETFL, old_flags | O_NONBLOCK);\n+\t}\n+\n+\tdo {\n+\t\tbufsize = 0x10000;\n+\t\tbuf = malloc(bufsize);\n+\t\tif (!buf) {\n+\t\t\tret = -1;\n+\t\t\tbreak;\n+\t\t}\n+\t\t\n+\t\treadsize = read(fd_in, buf, bufsize);\n+\t\tif (readsize <= 0) {\n+\t\t\tfree(buf);\n+\t\t\tif (readsize == 0) {\n+\t\t\t\tret = -1;\n+\t\t\t}\n+\t\t\tbreak;\n+\t\t}\n+\t\t\t\n+\t\tbuf = realloc(buf, readsize);\n+\t\tsegment = (typeof(segment))(malloc(sizeof(*segment)));\n+\t\tif (!segment) {\n+\t\t\tfree(buf);\n+\t\t\tret = -1;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tsegment->size = readsize;\n+\t\tsegment->buffer = buf;\n+\t\tsegment->next = inputbuffer->first;\n+\t\tsegment->prev = NULL;\n+\n+\t\tif (inputbuffer->first == NULL) {\n+\t\t\tinputbuffer->last = segment;\n+\t\t} else {\n+\t\t\tinputbuffer->first->prev = segment;\n+\t\t}\n+\t\tinputbuffer->first = segment;\n+\t} while (!block);\n+\n+\tif (!block) {\n+\t\tfcntl(fd_in, F_SETFL, old_flags);\n+\t}\n+\n+\treturn ret;\n+}\n \n+extern int write_sha1_from_fd(const unsigned char *sha1, int fd, struct input_buffer *inputbuffer)\n+{\n+\tchar *filename = sha1_file_name(sha1);\n+\tint read_result_byte = 0;\n+\tsigned char result_byte;\n \tint local;\n \tz_stream stream;\n \tunsigned char real_sha1[20];\n@@ -1437,10 +1514,20 @@ int write_sha1_from_fd(const unsigned ch\n \tSHA1_Init(&c);\n \n \tdo {\n-\t\tssize_t size;\n-\t\tif (*bufposn) {\n-\t\t\tstream.avail_in = *bufposn;\n-\t\t\tstream.next_in = (unsigned char *) buffer;\n+\t\tstruct input_segment *segment = inputbuffer->last;\n+\t\tif (segment) {\n+\t\t\tif (!read_result_byte) {\n+\t\t\t\tint more_to_read;\n+\t\t\t\tresult_byte = segment->buffer[0];\n+\t\t\t\tread_result_byte = 1;\n+\n+\t\t\t\tmore_to_read = cut_last_input_segment(inputbuffer, segment, segment->size - 1);\n+\t\t\t\tif (!more_to_read)\n+\t\t\t\t\tcontinue;\n+\t\t\t}\n+\n+\t\t\tstream.avail_in = segment->size;\n+\t\t\tstream.next_in = segment->buffer;\n \t\t\tdo {\n \t\t\t\tstream.next_out = discard;\n \t\t\t\tstream.avail_out = sizeof(discard);\n@@ -1448,23 +1535,19 @@ int write_sha1_from_fd(const unsigned ch\n \t\t\t\tSHA1_Update(&c, discard, sizeof(discard) -\n \t\t\t\t\t    stream.avail_out);\n \t\t\t} while (stream.avail_in && ret == Z_OK);\n-\t\t\twrite(local, buffer, *bufposn - stream.avail_in);\n-\t\t\tmemmove(buffer, buffer + *bufposn - stream.avail_in,\n-\t\t\t\tstream.avail_in);\n-\t\t\t*bufposn = stream.avail_in;\n-\t\t\tif (ret != Z_OK)\n-\t\t\t\tbreak;\n-\t\t}\n-\t\tsize = read(fd, buffer + *bufposn, bufsize - *bufposn);\n-\t\tif (size <= 0) {\n-\t\t\tclose(local);\n-\t\t\tunlink(filename);\n-\t\t\tif (!size)\n+\n+\t\t\twrite(local, segment->buffer, segment->size - stream.avail_in);\n+\t\t\tcut_last_input_segment(inputbuffer, segment, stream.avail_in);\n+ \t\t\tif (ret != Z_OK)\n+ \t\t\t\tbreak;\n+\t\t} else {\n+\t\t\tret = read_input_buffers(1, fd, inputbuffer);\n+\t\t\tif (ret) {\n+\t\t\t\tclose(local);\n+\t\t\t\tunlink(filename);\n \t\t\t\treturn error(\"Connection closed?\");\n-\t\t\tperror(\"Reading from connection\");\n-\t\t\treturn -1;\n-\t\t}\n-\t\t*bufposn += size;\n+\t\t\t}\n+ \t\t}\n \t} while (1);\n \tinflateEnd(&stream);\n \ndiff --git a/ssh-fetch.c b/ssh-fetch.c\n--- a/ssh-fetch.c\n+++ b/ssh-fetch.c\n@@ -14,57 +14,85 @@\n #include \"fetch.h\"\n #include \"refs.h\"\n \n+#include <unistd.h>\n+#include <fcntl.h>\n+#include <sys/time.h>\n+#include <sys/types.h>\n+#include <unistd.h>\n+\n static int fd_in;\n static int fd_out;\n \n static unsigned char remote_version = 0;\n static unsigned char local_version = 1;\n+static struct input_buffer inputbuffer = {NULL, };\n \n-static ssize_t force_write(int fd, void *buffer, size_t length)\n+static ssize_t force_write(void *buffer, size_t length)\n {\n+\tfd_set rfds, wfds;\n \tssize_t ret = 0;\n+\tint retval;\n+\tstruct timeval tv;\n+\n+\tFD_ZERO(&rfds);\n+\tFD_ZERO(&wfds);\n+\tFD_SET(fd_in, &rfds);\n+\tFD_SET(fd_out, &wfds);\n+\n \twhile (ret < length) {\n-\t\tssize_t size = write(fd, buffer + ret, length - ret);\n-\t\tif (size < 0) {\n-\t\t\treturn size;\n+\t\ttv.tv_sec = 1;\n+\t\ttv.tv_usec = 0;\n+\t\t\n+\t\tretval = select(FD_SETSIZE, &rfds, &wfds, NULL, &tv);\n+\t\tif (!retval)\n+\t\t\tcontinue;\n+\n+\t\tif (FD_ISSET(fd_in, &rfds)) {\n+\t\t\tread_input_buffers(0, fd_in, &inputbuffer);\n+\t\t\tcontinue;\n \t\t}\n-\t\tif (size == 0) {\n-\t\t\treturn ret;\n+\n+\t\tif (FD_ISSET(fd_out, &wfds)) {\n+\t\t\tssize_t size;\n+\t\t\tunsigned long old_flags = 0;\n+\n+\t\t\told_flags = fcntl(fd_out, F_GETFL);\n+\t\t\tfcntl(fd_out, F_SETFL, old_flags | O_NONBLOCK);\n+\t\t\tsize = write(fd_out, buffer + ret, length - ret);\n+\t\t\tfcntl(fd_out, F_SETFL, old_flags);\n+\t\t\t\n+\t\t\tif (size < 0) {\n+\t\t\t\treturn size;\n+\t\t\t}\n+\t\t\tif (size == 0) {\n+\t\t\t\treturn ret;\n+\t\t\t}\t\t\n+\t\n+\t\t\tret += size;\n \t\t}\n-\t\tret += size;\n \t}\n+\n \treturn ret;\n }\n \n void prefetch(unsigned char *sha1)\n {\n \tchar type = 'o';\n-\tforce_write(fd_out, &type, 1);\n-\tforce_write(fd_out, sha1, 20);\n+\tread_input_buffers(0, fd_in, &inputbuffer);\n+\tforce_write(&type, 1);\n+\tforce_write(sha1, 20);\n \t//memcpy(requested + 20 * prefetches++, sha1, 20);\n }\n \n-static char conn_buf[4096];\n-static size_t conn_buf_posn = 0;\n-\n int fetch(unsigned char *sha1)\n {\n \tint ret;\n-\tsigned char remote;\n \n-\tif (conn_buf_posn) {\n-\t\tremote = conn_buf[0];\n-\t\tmemmove(conn_buf, conn_buf + 1, --conn_buf_posn);\n-\t} else {\n-\t\tif (read(fd_in, &remote, 1) < 1)\n-\t\t\treturn -1;\n-\t}\n-\t//fprintf(stderr, \"Got %d\\n\", remote);\n-\tif (remote < 0)\n-\t\treturn remote;\n-\tret = write_sha1_from_fd(sha1, fd_in, conn_buf, 4096, &conn_buf_posn);\n+\tread_input_buffers(0, fd_in, &inputbuffer);\n+\tret = write_sha1_from_fd(sha1, fd_in, &inputbuffer);\n \tif (!ret)\n \t\tpull_say(\"got %s\\n\", sha1_to_hex(sha1));\n+\n \treturn ret;\n }\n \n\n-- \nDan Aloni\nda-x@monatomic.org, da-x@colinux.org, da-x@gmx.net\n"}]}