git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 13/14] remote-curl: resume interrupted bundle transfers

From
Jeff King <peff@peff.net>
Date
Nov 10, 2011, 07:53 UTC
Message-ID
<20111110075353.GM27950@sigill.intra.peff.net>
In-Reply-To
<20111110074330.GA27925@sigill.intra.peff.net>

If we have a bundle file from a previous fetch that matches this URL, then we should resume the transfer where we left off.

Signed-off-by: Jeff King <peff@peff.net>
---
The second half of the diff is hard to read because I re-indent a big
chunk. It's much easier to see what's going on with "diff -b".
 remote-curl.c |   64 +++++++++++++++++++++++++++++++++++++++++++++++---------
 1 files changed, 53 insertions(+), 11 deletions(-)
diff --git a/remote-curl.c b/remote-curl.c
index 6b0820e..43ad1b6 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -9,6 +9,7 @@
 #include "sideband.h"
 #include "bundle.h"
 #include "progress.h"
+#include "dir.h"
 
 static struct remote *remote;
 static const char *url; /* always ends with a trailing slash */
@@ -171,6 +172,32 @@ static int get_refs_from_url(const char *url, struct strbuf *out, int options,
 	return ret;
 }
 
+static int resume_bundle(const char *url, const char *tmpname)
+{
+	struct get_refs_cb_data data;
+	int ret;
+
+	data.fh = fopen(tmpname, "ab");
+	if (!data.fh)
+		die_errno("unable to open %s", tmpname);
+
+	data.is_bundle = 1;
+	data.total = ftell(data.fh);
+	if (options.progress) {
+		data.progress = start_progress("Resuming bundle", 0);
+		display_progress(data.progress, 0);
+		display_throughput(data.progress, data.total);
+	}
+
+	ret = http_get_callback(url, get_refs_callback, &data, data.total, 0);
+
+	if (fclose(data.fh))
+		die_errno("unable to write to %s", tmpname);
+	stop_progress(&data.progress);
+
+	return ret;
+}
+
 static const char *url_to_bundle_tmpfile(const char *url)
 {
 	struct strbuf buf = STRBUF_INIT;
@@ -210,20 +237,35 @@ static int get_refs_from_url(const char *url, struct strbuf *out, int options,
 	free_discovery(last);
 
 	filename = url_to_bundle_tmpfile(url);
+	if (file_exists(filename)) {
+		struct strbuf trimmed = STRBUF_INIT;
 
-	strbuf_addf(&buffer, "%sinfo/refs", url);
-	if (!prefixcmp(url, "http://") || !prefixcmp(url, "https://")) {
-		is_http = 1;
-		if (!strchr(url, '?'))
-			strbuf_addch(&buffer, '?');
-		else
-			strbuf_addch(&buffer, '&');
-		strbuf_addf(&buffer, "service=%s", service);
+		strbuf_addstr(&trimmed, url);
+		while (trimmed.len > 0 && trimmed.buf[trimmed.len-1] == '/')
+			strbuf_setlen(&trimmed, trimmed.len - 1);
+		refs_url = strbuf_detach(&trimmed, NULL);
+
+		http_ret = resume_bundle(refs_url, filename);
+		is_bundle = 1;
 	}
-	refs_url = strbuf_detach(&buffer, NULL);
+	else
+		http_ret = HTTP_MISSING_TARGET;
+
+	if (http_ret != HTTP_OK && http_ret != HTTP_NOAUTH) {
+		strbuf_addf(&buffer, "%sinfo/refs", url);
+		if (!prefixcmp(url, "http://") || !prefixcmp(url, "https://")) {
+			is_http = 1;
+			if (!strchr(url, '?'))
+				strbuf_addch(&buffer, '?');
+			else
+				strbuf_addch(&buffer, '&');
+			strbuf_addf(&buffer, "service=%s", service);
+		}
+		refs_url = strbuf_detach(&buffer, NULL);
 
-	http_ret = get_refs_from_url(refs_url, &buffer, HTTP_NO_CACHE,
-				     filename, &is_bundle);
+		http_ret = get_refs_from_url(refs_url, &buffer, HTTP_NO_CACHE,
+					     filename, &is_bundle);
+	}
 
 	/* try again with "plain" url (no ? or & appended) */
 	if (http_ret != HTTP_OK && http_ret != HTTP_NOAUTH) {
-- 
1.7.7.2.7.g9f96f
Previous: Jeff KingNext: Jeff King
Message 17 of 23 in “resumable network bundles”
  1. 0/14 resumable network bundlesJeff King, Nov 10, 2011
  2. Jeff KingNov 10, 2011
  3. 01/14 t/lib-httpd: check for NO_CURLJeff King, Nov 10, 2011
  4. 02/14 http: turn off curl signalsJeff King, Nov 10, 2011
  5. Daniel StenbergNov 10, 2011
  6. Jeff KingNov 11, 2011
  7. 03/14 http: refactor http_request functionJeff King, Nov 10, 2011
  8. 04/14 http: add a public function for arbitrary-callback requestJeff King, Nov 10, 2011
  9. 05/14 remote-curl: use http callback for requesting refsJeff King, Nov 10, 2011
  10. 06/14 transport: factor out bundle to ref list conversionJeff King, Nov 10, 2011
  11. 07/14 bundle: add is_bundle_buf helperJeff King, Nov 10, 2011
  12. 08/14 remote-curl: free "discovery" objectJeff King, Nov 10, 2011
  13. 09/14 remote-curl: auto-detect bundles when fetching refsJeff King, Nov 10, 2011
  14. 10/14 remote-curl: try base $URL after $URL/info/refsJeff King, Nov 10, 2011
  15. 11/14 progress: allow pure-throughput progress metersJeff King, Nov 10, 2011
  16. 12/14 remote-curl: show progress for bundle downloadsJeff King, Nov 10, 2011
  17. 13/14 remote-curl: resume interrupted bundle transfersJeff King, Nov 10, 2011
  18. 14/14 clone: give advice on how to resume a failed cloneJeff King, Nov 10, 2011
  19. Junio C HamanoNov 10, 2011
  20. Jeff KingNov 11, 2011
  21. David Michael BarrNov 11, 2011
  22. Tay Ray ChuanNov 12, 2011
  23. Jeff KingNov 12, 2011

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.