threads / patch / 52930

patchremote-curl: show progress for fetches over dumb HTTP

Subject: [PATCH RESEND] remote-curl: show progress for fetches over dumb HTTP

## tl;dr

3 messages between Mar 3, 2020 and Mar 5, 2020. Diffs are folded; open one to read it.

replies: 2people: 3as markdown or json

René Scharfe· Mar 3, 2020, 20:55 UTC · lore

Fetching over dumb HTTP transport doesn't show any progress, even with the option --progress. If the connection is slow or there is a lot of data to get then this can take a long time while the user is left to wonder if git got stuck.

We don't know the number of objects to fetch at the outset, but we can count the ones we got. Show an open-ended progress indicator based on that number if the user asked for it.

Signed-off-by: René Scharfe <l.s.r@web.de>
---
Original submission:
https://lore.kernel.org/git/9ed26e7e-c19c-cdb2-0710-3b91bf31291b@web.de/
 remote-curl.c |  1 +
 walker.c      | 13 ++++++++++++-
 walker.h      |  1 +
 3 files changed, 14 insertions(+), 1 deletion(-)
Show changes to 3 files +14 −2

remote-curl.c, walker.c, walker.h

diff --git a/remote-curl.c b/remote-curl.c
index 8eb96152f5..e4cd321844 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -1026,6 +1026,7 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch)

 	walker = get_http_walker(url.buf);
 	walker->get_verbosely = options.verbosity >= 3;
+	walker->get_progress = options.progress;
 	walker->get_recover = 0;
 	ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
 	walker_free(walker);
diff --git a/walker.c b/walker.c
index bb010f7a2b..4984bf8b3d 100644
--- a/walker.c
+++ b/walker.c
@@ -8,6 +8,7 @@
 #include "tag.h"
 #include "blob.h"
 #include "refs.h"
+#include "progress.h"

 static struct object_id current_commit_oid;

@@ -162,6 +163,11 @@ static int process(struct walker *walker, struct object *obj)
 static int loop(struct walker *walker)
 {
 	struct object_list *elem;
+	struct progress *progress = NULL;
+	uint64_t nr = 0;
+
+	if (walker->get_progress)
+		progress = start_delayed_progress(_("Fetching objects"), 0);

 	while (process_queue) {
 		struct object *obj = process_queue->item;
@@ -176,15 +182,20 @@ static int loop(struct walker *walker)
 		 */
 		if (! (obj->flags & TO_SCAN)) {
 			if (walker->fetch(walker, obj->oid.hash)) {
+				stop_progress(&progress);
 				report_missing(obj);
 				return -1;
 			}
 		}
 		if (!obj->type)
 			parse_object(the_repository, &obj->oid);
-		if (process_object(walker, obj))
+		if (process_object(walker, obj)) {
+			stop_progress(&progress);
 			return -1;
+		}
+		display_progress(progress, ++nr);
 	}
+	stop_progress(&progress);
 	return 0;
 }

diff --git a/walker.h b/walker.h
index 6d8ae00e5b..d40b016bab 100644
--- a/walker.h
+++ b/walker.h
@@ -10,6 +10,7 @@ struct walker {
 	int (*fetch)(struct walker *, unsigned char *sha1);
 	void (*cleanup)(struct walker *);
 	int get_verbosely;
+	int get_progress;
 	int get_recover;

 	int corrupt_object_found;
--
2.25.1
Junio C Hamano· Mar 3, 2020, 21:41 UTC · re: René Scharfe · lore

Re: [PATCH RESEND] remote-curl: show progress for fetches over dumb HTTP

René Scharfe <l.s.r@web.de> writes:
Show 13 quoted lines
> Fetching over dumb HTTP transport doesn't show any progress, even with
> the option --progress.  If the connection is slow or there is a lot of
> data to get then this can take a long time while the user is left to
> wonder if git got stuck.
>
> We don't know the number of objects to fetch at the outset, but we can
> count the ones we got.  Show an open-ended progress indicator based on
> that number if the user asked for it.
>
> Signed-off-by: René Scharfe <l.s.r@web.de>
> ---
> Original submission:
> https://lore.kernel.org/git/9ed26e7e-c19c-cdb2-0710-3b91bf31291b@web.de/
Thanks.
Show 78 quoted lines
>
>  remote-curl.c |  1 +
>  walker.c      | 13 ++++++++++++-
>  walker.h      |  1 +
>  3 files changed, 14 insertions(+), 1 deletion(-)
>
> diff --git a/remote-curl.c b/remote-curl.c
> index 8eb96152f5..e4cd321844 100644
> --- a/remote-curl.c
> +++ b/remote-curl.c
> @@ -1026,6 +1026,7 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch)
>
>  	walker = get_http_walker(url.buf);
>  	walker->get_verbosely = options.verbosity >= 3;
> +	walker->get_progress = options.progress;
>  	walker->get_recover = 0;
>  	ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
>  	walker_free(walker);
> diff --git a/walker.c b/walker.c
> index bb010f7a2b..4984bf8b3d 100644
> --- a/walker.c
> +++ b/walker.c
> @@ -8,6 +8,7 @@
>  #include "tag.h"
>  #include "blob.h"
>  #include "refs.h"
> +#include "progress.h"
>
>  static struct object_id current_commit_oid;
>
> @@ -162,6 +163,11 @@ static int process(struct walker *walker, struct object *obj)
>  static int loop(struct walker *walker)
>  {
>  	struct object_list *elem;
> +	struct progress *progress = NULL;
> +	uint64_t nr = 0;
> +
> +	if (walker->get_progress)
> +		progress = start_delayed_progress(_("Fetching objects"), 0);
>
>  	while (process_queue) {
>  		struct object *obj = process_queue->item;
> @@ -176,15 +182,20 @@ static int loop(struct walker *walker)
>  		 */
>  		if (! (obj->flags & TO_SCAN)) {
>  			if (walker->fetch(walker, obj->oid.hash)) {
> +				stop_progress(&progress);
>  				report_missing(obj);
>  				return -1;
>  			}
>  		}
>  		if (!obj->type)
>  			parse_object(the_repository, &obj->oid);
> -		if (process_object(walker, obj))
> +		if (process_object(walker, obj)) {
> +			stop_progress(&progress);
>  			return -1;
> +		}
> +		display_progress(progress, ++nr);
>  	}
> +	stop_progress(&progress);
>  	return 0;
>  }
>
> diff --git a/walker.h b/walker.h
> index 6d8ae00e5b..d40b016bab 100644
> --- a/walker.h
> +++ b/walker.h
> @@ -10,6 +10,7 @@ struct walker {
>  	int (*fetch)(struct walker *, unsigned char *sha1);
>  	void (*cleanup)(struct walker *);
>  	int get_verbosely;
> +	int get_progress;
>  	int get_recover;
>
>  	int corrupt_object_found;
> --
> 2.25.1
Manish Devgan· Mar 5, 2020, 07:03 UTC · re: René Scharfe · lore

Re: [PATCH RESEND] remote-curl: show progress for fetches over dumb HTTP

On Wed, Mar 4, 2020 at 2:25 AM René Scharfe <l.s.r@web.de> wrote:
Show 92 quoted lines
>
> Fetching over dumb HTTP transport doesn't show any progress, even with
> the option --progress.  If the connection is slow or there is a lot of
> data to get then this can take a long time while the user is left to
> wonder if git got stuck.
>
> We don't know the number of objects to fetch at the outset, but we can
> count the ones we got.  Show an open-ended progress indicator based on
> that number if the user asked for it.
>
> Signed-off-by: René Scharfe <l.s.r@web.de>
> ---
> Original submission:
> https://lore.kernel.org/git/9ed26e7e-c19c-cdb2-0710-3b91bf31291b@web.de/
>
>  remote-curl.c |  1 +
>  walker.c      | 13 ++++++++++++-
>  walker.h      |  1 +
>  3 files changed, 14 insertions(+), 1 deletion(-)
>
> diff --git a/remote-curl.c b/remote-curl.c
> index 8eb96152f5..e4cd321844 100644
> --- a/remote-curl.c
> +++ b/remote-curl.c
> @@ -1026,6 +1026,7 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch)
>
>         walker = get_http_walker(url.buf);
>         walker->get_verbosely = options.verbosity >= 3;
> +       walker->get_progress = options.progress;
>         walker->get_recover = 0;
>         ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
>         walker_free(walker);
> diff --git a/walker.c b/walker.c
> index bb010f7a2b..4984bf8b3d 100644
> --- a/walker.c
> +++ b/walker.c
> @@ -8,6 +8,7 @@
>  #include "tag.h"
>  #include "blob.h"
>  #include "refs.h"
> +#include "progress.h"
>
>  static struct object_id current_commit_oid;
>
> @@ -162,6 +163,11 @@ static int process(struct walker *walker, struct object *obj)
>  static int loop(struct walker *walker)
>  {
>         struct object_list *elem;
> +       struct progress *progress = NULL;
> +       uint64_t nr = 0;
> +
> +       if (walker->get_progress)
> +               progress = start_delayed_progress(_("Fetching objects"), 0);
>
>         while (process_queue) {
>                 struct object *obj = process_queue->item;
> @@ -176,15 +182,20 @@ static int loop(struct walker *walker)
>                  */
>                 if (! (obj->flags & TO_SCAN)) {
>                         if (walker->fetch(walker, obj->oid.hash)) {
> +                               stop_progress(&progress);
>                                 report_missing(obj);
>                                 return -1;
>                         }
>                 }
>                 if (!obj->type)
>                         parse_object(the_repository, &obj->oid);
> -               if (process_object(walker, obj))
> +               if (process_object(walker, obj)) {
> +                       stop_progress(&progress);
>                         return -1;
> +               }
> +               display_progress(progress, ++nr);
>         }
> +       stop_progress(&progress);
>         return 0;
>  }
>
> diff --git a/walker.h b/walker.h
> index 6d8ae00e5b..d40b016bab 100644
> --- a/walker.h
> +++ b/walker.h
> @@ -10,6 +10,7 @@ struct walker {
>         int (*fetch)(struct walker *, unsigned char *sha1);
>         void (*cleanup)(struct walker *);
>         int get_verbosely;
> +       int get_progress;
>         int get_recover;
>
>         int corrupt_object_found;
> --
> 2.25.1
Thanks. This works for me.

Regards Manish Devgan

← back to recent threads