[PATCH 07/11] Resumable clone: add resumable download to http/curl

From

KWKevin Wern <kevin.m.wern@gmail.com>

Date

Sep 16, 2016, 00:12 UTC

Message-ID

<1473984742-12516-8-git-send-email-kevin.m.wern@gmail.com>

In-Reply-To

<1473984742-12516-1-git-send-email-kevin.m.wern@gmail.com>

Signed-off-by: Kevin Wern <kevin.m.wern@gmail.com>
---
 http.c        | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 http.h        |  7 ++++-
 remote-curl.c | 27 +++++++++++++++++++
 3 files changed, 118 insertions(+), 2 deletions(-)

diff --git a/http.c b/http.c
index 1d5e3bb..93d6324 100644
--- a/http.c
+++ b/http.c
@@ -10,6 +10,8 @@
 #include "pkt-line.h"
 #include "gettext.h"
 #include "transport.h"
+#include "progress.h"
+#include "dir.h"
 
 #if LIBCURL_VERSION_NUM >= 0x070a08
 long int git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER;
@@ -1136,7 +1138,10 @@ static int handle_curl_result(struct slot_results *results)
 				curl_easy_strerror(results->curl_result),
 				sizeof(curl_errorstr));
 #endif
-		return HTTP_ERROR;
+		if (results->http_code >= 400)
+			return HTTP_ERROR;
+		else
+			return HTTP_ERROR_RESUMABLE;
 	}
 }
 
@@ -1365,6 +1370,40 @@ static void http_opt_request_remainder(CURL *curl, off_t pos)
 #define HTTP_REQUEST_STRBUF	0
 #define HTTP_REQUEST_FILE	1
 
+static int bytes_to_rounded_kb(double bytes)
+{
+	return (int) (bytes + 512)/1024;
+}
+
+int progress_func(void *data, double total_to_download, double now_downloaded,
+		  double total_to_upload, double now_uploadeded)
+{
+	struct progress **progress = data;
+	int kilobytes = total_to_download >= 1024;
+
+	if (total_to_download <= 0.0) {
+		return 0;
+	}
+	if (kilobytes) {
+		now_downloaded = bytes_to_rounded_kb(now_downloaded);
+		total_to_download = bytes_to_rounded_kb(total_to_download);
+	}
+	if (!*progress && now_downloaded < total_to_download) {
+		if (total_to_download > 1024)
+			*progress = start_progress("Downloading (KB)",
+						   total_to_download);
+		else
+			*progress = start_progress("Downloading (B)",
+						   total_to_download);
+	}
+	display_progress(*progress, now_downloaded);
+	if (now_downloaded == total_to_download) {
+		stop_progress(progress);
+	}
+	return 0;
+}
+
+
 static int http_request(const char *url,
 			void *result, int target,
 			const struct http_get_options *options)
@@ -1373,6 +1412,7 @@ static int http_request(const char *url,
 	struct slot_results results;
 	struct curl_slist *headers = NULL;
 	struct strbuf buf = STRBUF_INIT;
+	struct progress *progress = NULL;
 	const char *accept_language;
 	int ret;
 
@@ -1389,6 +1429,16 @@ static int http_request(const char *url,
 			off_t posn = ftello(result);
 			curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 					 fwrite);
+			if (options && options->progress) {
+				curl_easy_setopt(slot->curl,
+						 CURLOPT_NOPROGRESS, 0);
+				curl_easy_setopt(slot->curl,
+						 CURLOPT_PROGRESSFUNCTION,
+						 progress_func);
+				curl_easy_setopt(slot->curl,
+						 CURLOPT_PROGRESSDATA,
+						 &progress);
+			}
 			if (posn > 0)
 				http_opt_request_remainder(slot->curl, posn);
 		} else
@@ -1559,6 +1609,40 @@ cleanup:
 	return ret;
 }
 
+int http_download_primer(const char *url, const char *out_file)
+{
+	int ret = 0, try_count = HTTP_TRY_COUNT;
+	struct http_get_options options = {0};
+	options.progress = 1;
+
+	if (file_exists(out_file)) {
+		fprintf(stderr,
+			"File already downloaded: '%s', skipping...\n",
+			out_file);
+		return ret;
+	}
+
+	do {
+		if (try_count != HTTP_TRY_COUNT) {
+			fprintf(stderr, "Connection interrupted for some "
+				"reason, retrying (%d attempts left)\n",
+				try_count);
+			struct timeval time = {10, 0}; // 1s
+			select(0, NULL, NULL, NULL, &time);
+		}
+		ret = http_get_file(url, out_file, &options);
+		try_count--;
+	} while (try_count > 0 && ret == HTTP_ERROR_RESUMABLE);
+
+	if (ret != HTTP_OK) {
+		error("Unable to get resource: %s", url);
+		ret = -1;
+	}
+
+	return ret;
+}
+
+
 int http_fetch_ref(const char *base, struct ref *ref)
 {
 	struct http_get_options options = {0};
diff --git a/http.h b/http.h
index 4ef4bbd..6a7ce7b 100644
--- a/http.h
+++ b/http.h
@@ -138,7 +138,8 @@ extern char *get_remote_object_url(const char *url, const char *hex,
 /* Options for http_get_*() */
 struct http_get_options {
 	unsigned no_cache:1,
-		 keep_error:1;
+		 keep_error:1,
+		 progress:1;
 
 	/* If non-NULL, returns the content-type of the response. */
 	struct strbuf *content_type;
@@ -172,6 +173,7 @@ struct http_get_options {
 #define HTTP_START_FAILED	3
 #define HTTP_REAUTH	4
 #define HTTP_NOAUTH	5
+#define HTTP_ERROR_RESUMABLE	6
 
 /*
  * Requests a URL and stores the result in a strbuf.
@@ -180,6 +182,9 @@ struct http_get_options {
  */
 int http_get_strbuf(const char *url, struct strbuf *result, struct http_get_options *options);
 
+#define HTTP_TRY_COUNT 5
+int http_download_primer(const char *url, const char *out_file);
+
 extern int http_fetch_ref(const char *base, struct ref *ref);
 
 /* Helpers for fetching packs */
diff --git a/remote-curl.c b/remote-curl.c
index 8ebb587..051ba52 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -394,6 +394,30 @@ static void prime_clone(void)
 	free(result_full);
 }
 
+static void download_primer(const char *url, const char *base_dir)
+{
+	char *slash_ptr = strchr(url, '/'), *out_file;
+	struct strbuf out_path = STRBUF_INIT;
+	do {
+		out_file = slash_ptr + 1;
+	} while (slash_ptr = strchr(out_file, '/'));
+	strbuf_addf(&out_path, "%s/%s", base_dir, out_file);
+	if (!http_download_primer(url, out_path.buf))
+		printf("%s\n", out_path.buf);
+	printf("\n");
+	fflush(stdout);
+}
+
+static void parse_download_primer(struct strbuf *buf)
+{
+	const char *remote_url;
+	if (skip_prefix(buf->buf, "download-primer ", &remote_url)) {
+		char *base_path;
+		base_path = strchr(remote_url, ' ');
+		*base_path++ = '\0';
+		download_primer(remote_url, base_path);
+	}
+}
 
 static struct discovery *discover_refs(const char *service, int for_push)
 {
@@ -1105,6 +1129,8 @@ int main(int argc, const char **argv)
 		} else if (!strcmp(buf.buf, "list") || starts_with(buf.buf, "list ")) {
 			int for_push = !!strstr(buf.buf + 4, "for-push");
 			output_refs(get_refs(for_push));
+		} else if (starts_with(buf.buf, "download-primer")) {
+			parse_download_primer(&buf);
 		} else if (!strcmp(buf.buf, "prime-clone")) {
 			prime_clone();
 		} else if (starts_with(buf.buf, "push ")) {
@@ -1132,6 +1158,7 @@ int main(int argc, const char **argv)
 			printf("fetch\n");
 			printf("option\n");
 			printf("push\n");
+			printf("download-primer\n");
 			printf("prime-clone\n");
 			printf("check-connectivity\n");
 			printf("\n");

-- 
2.7.4