git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 08/16] upload-pack: let pack-objects do the object counting in shallow case

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
Jul 20, 2013, 09:58 UTC
Message-ID
<1374314290-5976-9-git-send-email-pclouds@gmail.com>
In-Reply-To
<1374314290-5976-1-git-send-email-pclouds@gmail.com>

Remove the duplicate object counting code in upload-pack, dump out all register_shallow()'d SHA-1 into a temporary shallow file and feed it to pack-objects. The end result is the same, except with less code, and fewer bytes sending over pipe to pack-objects.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 t/t5530-upload-pack-error.sh |   3 -
 upload-pack.c                | 128 ++++++++++---------------------------------
 2 files changed, 30 insertions(+), 101 deletions(-)
diff --git a/t/t5530-upload-pack-error.sh b/t/t5530-upload-pack-error.sh
index c983d36..3932e79 100755
--- a/t/t5530-upload-pack-error.sh
+++ b/t/t5530-upload-pack-error.sh
@@ -54,9 +54,6 @@ test_expect_success 'upload-pack fails due to error in rev-list' '
 	printf "0032want %s\n0034shallow %s00000009done\n0000" \
 		$(git rev-parse HEAD) $(git rev-parse HEAD^) >input &&
 	test_must_fail git upload-pack . <input >/dev/null 2>output.err &&
-	# pack-objects survived
-	grep "Total.*, reused" output.err &&
-	# but there was an error, which must have been in rev-list
 	grep "bad tree object" output.err
 '
 
diff --git a/upload-pack.c b/upload-pack.c
index c377a3e..c3e68ae 100644
--- a/upload-pack.c
+++ b/upload-pack.c
@@ -35,7 +35,6 @@ static int no_done;
 static int use_thin_pack, use_ofs_delta, use_include_tag;
 static int no_progress, daemon_mode;
 static int allow_tip_sha1_in_want;
-static int shallow_nr;
 static struct object_array have_obj;
 static struct object_array want_obj;
 static struct object_array extra_edge_obj;
@@ -68,87 +67,32 @@ static ssize_t send_client_data(int fd, const char *data, ssize_t sz)
 	return sz;
 }
 
-static FILE *pack_pipe = NULL;
-static void show_commit(struct commit *commit, void *data)
-{
-	if (commit->object.flags & BOUNDARY)
-		fputc('-', pack_pipe);
-	if (fputs(sha1_to_hex(commit->object.sha1), pack_pipe) < 0)
-		die("broken output pipe");
-	fputc('\n', pack_pipe);
-	fflush(pack_pipe);
-	free(commit->buffer);
-	commit->buffer = NULL;
-}
-
-static void show_object(struct object *obj,
-			const struct name_path *path, const char *component,
-			void *cb_data)
-{
-	show_object_with_name(pack_pipe, obj, path, component);
-}
-
-static void show_edge(struct commit *commit)
-{
-	fprintf(pack_pipe, "-%s\n", sha1_to_hex(commit->object.sha1));
-}
-
-static int do_rev_list(int in, int out, void *user_data)
-{
-	int i;
-	struct rev_info revs;
-
-	pack_pipe = xfdopen(out, "w");
-	init_revisions(&revs, NULL);
-	revs.tag_objects = 1;
-	revs.tree_objects = 1;
-	revs.blob_objects = 1;
-	if (use_thin_pack)
-		revs.edge_hint = 1;
-
-	for (i = 0; i < want_obj.nr; i++) {
-		struct object *o = want_obj.objects[i].item;
-		/* why??? */
-		o->flags &= ~UNINTERESTING;
-		add_pending_object(&revs, o, NULL);
-	}
-	for (i = 0; i < have_obj.nr; i++) {
-		struct object *o = have_obj.objects[i].item;
-		o->flags |= UNINTERESTING;
-		add_pending_object(&revs, o, NULL);
-	}
-	setup_revisions(0, NULL, &revs, NULL);
-	if (prepare_revision_walk(&revs))
-		die("revision walk setup failed");
-	mark_edges_uninteresting(revs.commits, &revs, show_edge);
-	if (use_thin_pack)
-		for (i = 0; i < extra_edge_obj.nr; i++)
-			fprintf(pack_pipe, "-%s\n", sha1_to_hex(
-					extra_edge_obj.objects[i].item->sha1));
-	traverse_commit_list(&revs, show_commit, show_object, NULL);
-	fflush(pack_pipe);
-	fclose(pack_pipe);
-	return 0;
-}
-
 static void create_pack_file(void)
 {
-	struct async rev_list;
 	struct child_process pack_objects;
 	char data[8193], progress[128];
 	char abort_msg[] = "aborting due to possible repository "
 		"corruption on the remote side.";
 	int buffered = -1;
 	ssize_t sz;
-	const char *argv[10];
-	int arg = 0;
+	const char *argv[12];
+	int i, arg = 0;
+	FILE *pipe_fd;
+	static struct lock_file shallow_lock;
+	const char *alternate_shallow_file = NULL;
+
+	setup_alternate_shallow(&shallow_lock,
+				&alternate_shallow_file,
+				NULL, 0);
+	if (*alternate_shallow_file) {
+		argv[arg++] = "--shallow-file";
+		argv[arg++] = alternate_shallow_file;
+	}
 
 	argv[arg++] = "pack-objects";
-	if (!shallow_nr) {
-		argv[arg++] = "--revs";
-		if (use_thin_pack)
-			argv[arg++] = "--thin";
-	}
+	argv[arg++] = "--revs";
+	if (use_thin_pack)
+		argv[arg++] = "--thin";
 
 	argv[arg++] = "--stdout";
 	if (!no_progress)
@@ -169,28 +113,18 @@ static void create_pack_file(void)
 	if (start_command(&pack_objects))
 		die("git upload-pack: unable to fork git-pack-objects");
 
-	if (shallow_nr) {
-		memset(&rev_list, 0, sizeof(rev_list));
-		rev_list.proc = do_rev_list;
-		rev_list.out = pack_objects.in;
-		if (start_async(&rev_list))
-			die("git upload-pack: unable to fork git-rev-list");
-	}
-	else {
-		FILE *pipe_fd = xfdopen(pack_objects.in, "w");
-		int i;
+	pipe_fd = xfdopen(pack_objects.in, "w");
 
-		for (i = 0; i < want_obj.nr; i++)
-			fprintf(pipe_fd, "%s\n",
-				sha1_to_hex(want_obj.objects[i].item->sha1));
-		fprintf(pipe_fd, "--not\n");
-		for (i = 0; i < have_obj.nr; i++)
-			fprintf(pipe_fd, "%s\n",
-				sha1_to_hex(have_obj.objects[i].item->sha1));
-		fprintf(pipe_fd, "\n");
-		fflush(pipe_fd);
-		fclose(pipe_fd);
-	}
+	for (i = 0; i < want_obj.nr; i++)
+		fprintf(pipe_fd, "%s\n",
+			sha1_to_hex(want_obj.objects[i].item->sha1));
+	fprintf(pipe_fd, "--not\n");
+	for (i = 0; i < have_obj.nr; i++)
+		fprintf(pipe_fd, "%s\n",
+			sha1_to_hex(have_obj.objects[i].item->sha1));
+	fprintf(pipe_fd, "\n");
+	fflush(pipe_fd);
+	fclose(pipe_fd);
 
 
 	/* We read from pack_objects.err to capture stderr output for
@@ -290,8 +224,9 @@ static void create_pack_file(void)
 		error("git upload-pack: git-pack-objects died with error.");
 		goto fail;
 	}
-	if (shallow_nr && finish_async(&rev_list))
-		goto fail;	/* error was already reported */
+
+	if (*alternate_shallow_file)
+		rollback_lock_file(&shallow_lock);
 
 	/* flush the data */
 	if (0 <= buffered) {
@@ -575,7 +510,6 @@ static void receive_needs(void)
 	int depth = 0;
 	int has_non_tip = 0;
 
-	shallow_nr = 0;
 	for (;;) {
 		struct object *o;
 		const char *features;
@@ -679,7 +613,6 @@ static void receive_needs(void)
 				packet_write(1, "shallow %s",
 						sha1_to_hex(object->sha1));
 				register_shallow(object->sha1);
-				shallow_nr++;
 			}
 			result = result->next;
 		}
@@ -715,7 +648,6 @@ static void receive_needs(void)
 				register_shallow(shallows.objects[i].item->sha1);
 		}
 
-	shallow_nr += shallows.nr;
 	free(shallows.objects);
 }
 
-- 
1.8.2.83.gc99314b
Previous: Nguyễn Thái Ngọc DuyNext: Nguyễn Thái Ngọc Duy
Message 22 of 44 in “First class shallow clone”
  1. 0/7 First class shallow cloneNguyễn Thái Ngọc Duy, Jul 17, 2013
  2. 1/7 transport.h: remove send_pack prototype, already defined in send-pack.hNguyễn Thái Ngọc Duy, Jul 17, 2013
  3. 2/7 {receive,upload}-pack: advertise shallow graft informationNguyễn Thái Ngọc Duy, Jul 17, 2013
  4. 3/7 connect.c: teach get_remote_heads to parse "shallow" linesNguyễn Thái Ngọc Duy, Jul 17, 2013
  5. Junio C HamanoJul 20, 2013
  6. 4/7 Move setup_alternate_shallow and write_shallow_commits to shallow.cNguyễn Thái Ngọc Duy, Jul 17, 2013
  7. 5/7 fetch-pack: support fetching from a shallow repositoryNguyễn Thái Ngọc Duy, Jul 17, 2013
  8. Junio C HamanoJul 20, 2013
  9. 6/7 {send,receive}-pack: support pushing from a shallow cloneNguyễn Thái Ngọc Duy, Jul 17, 2013
  10. 7/7 send-pack: support pushing to a shallow cloneNguyễn Thái Ngọc Duy, Jul 17, 2013
  11. 00/16 First class shallow cloneNguyễn Thái Ngọc Duy, Jul 20, 2013
  12. 01/16 send-pack: forbid pushing from a shallow repositoryNguyễn Thái Ngọc Duy, Jul 20, 2013
  13. 02/16 {receive,upload}-pack: advertise shallow graft informationNguyễn Thái Ngọc Duy, Jul 20, 2013
  14. 03/16 connect.c: teach get_remote_heads to parse "shallow" linesNguyễn Thái Ngọc Duy, Jul 20, 2013
  15. 04/16 Move setup_alternate_shallow and write_shallow_commits to shallow.cNguyễn Thái Ngọc Duy, Jul 20, 2013
  16. 05/16 fetch-pack: support fetching from a shallow repositoryNguyễn Thái Ngọc Duy, Jul 20, 2013
  17. Philip OakleyJul 22, 2013
  18. Duy NguyenJul 23, 2013
  19. Duy NguyenJul 23, 2013
  20. 06/16 {send,receive}-pack: support pushing from a shallow cloneNguyễn Thái Ngọc Duy, Jul 20, 2013
  21. 07/16 send-pack: support pushing to a shallow cloneNguyễn Thái Ngọc Duy, Jul 20, 2013
  22. 08/16 upload-pack: let pack-objects do the object counting in shallow caseNguyễn Thái Ngọc Duy, Jul 20, 2013
  23. 09/16 pack-protocol.txt: a bit about smart httpNguyễn Thái Ngọc Duy, Jul 20, 2013
  24. 10/16 Add document for command arguments for supporting smart httpNguyễn Thái Ngọc Duy, Jul 20, 2013
  25. 11/16 {fetch,upload}-pack: support fetching from a shallow clone via smart httpNguyễn Thái Ngọc Duy, Jul 20, 2013
  26. 12/16 receive-pack: support pushing to a shallow clone via httpNguyễn Thái Ngọc Duy, Jul 20, 2013
  27. 13/16 send-pack: support pushing from a shallow clone via httpNguyễn Thái Ngọc Duy, Jul 20, 2013
  28. 14/16 git-clone.txt: remove shallow clone limitationsNguyễn Thái Ngọc Duy, Jul 20, 2013
  29. 15/16 config: add core.noshallow to prevent turning a repo into a shallow oneNguyễn Thái Ngọc Duy, Jul 20, 2013
  30. Philip OakleyJul 22, 2013
  31. Duy NguyenJul 23, 2013
  32. Junio C HamanoJul 23, 2013
  33. Philip OakleyJul 23, 2013
  34. 16/16 clone: use git protocol for cloning shallow repo locallyNguyễn Thái Ngọc Duy, Jul 20, 2013
  35. Philip OakleyJul 22, 2013
  36. Duy NguyenJul 23, 2013
  37. Junio C HamanoJul 23, 2013
  38. Duy NguyenJul 23, 2013
  39. Philip OakleyJul 23, 2013
  40. Duy NguyenJul 24, 2013
  41. Philip OakleyJul 24, 2013
  42. Piotr KrukowieckiJul 24, 2013
  43. Duy NguyenJul 24, 2013
  44. Piotr KrukowieckiJul 24, 2013

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.