git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v6 12/40] external odb: add 'put_raw_obj' support

From
Christian Couder <christian.couder@gmail.com>
Date
Sep 16, 2017, 08:07 UTC
Message-ID
<20170916080731.13925-13-chriscool@tuxfamily.org>
In-Reply-To
<20170916080731.13925-1-chriscool@tuxfamily.org>

Add support for a 'put_raw_obj' capability/instruction to send new objects to an external odb. Objects will be sent as they are (in their 'raw' format). They will not be converted to Git objects.

For now any new Git object (blob, tree, commit, ...) would be sent if 'put_raw_obj' is supported by an odb helper. This is not a great default, but let's leave it to following commits to tweak that.

Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
 external-odb.c | 15 +++++++++++++++
 external-odb.h |  2 ++
 odb-helper.c   | 43 ++++++++++++++++++++++++++++++++++++++-----
 odb-helper.h   |  3 +++
 sha1_file.c    |  2 ++
 5 files changed, 60 insertions(+), 5 deletions(-)
diff --git a/external-odb.c b/external-odb.c
index 0f0de170b8..82fac702e8 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -118,3 +118,18 @@ int external_odb_get_object(const unsigned char *sha1)
 
 	return -1;
 }
+
+int external_odb_put_object(const void *buf, size_t len,
+			    const char *type, unsigned char *sha1)
+{
+	struct odb_helper *o;
+
+	external_odb_init();
+
+	for (o = helpers; o; o = o->next) {
+		int r = odb_helper_put_object(o, buf, len, type, sha1);
+		if (r <= 0)
+			return r;
+	}
+	return 1;
+}
diff --git a/external-odb.h b/external-odb.h
index dc5635f452..d369dfdf6f 100644
--- a/external-odb.h
+++ b/external-odb.h
@@ -4,5 +4,7 @@
 extern const char *external_odb_root(void);
 extern int external_odb_has_object(const unsigned char *sha1);
 extern int external_odb_get_object(const unsigned char *sha1);
+extern int external_odb_put_object(const void *buf, size_t len,
+				   const char *type, unsigned char *sha1);
 
 #endif /* EXTERNAL_ODB_H */
diff --git a/odb-helper.c b/odb-helper.c
index 9375eca58f..39d20fdfd7 100644
--- a/odb-helper.c
+++ b/odb-helper.c
@@ -71,9 +71,10 @@ static void prepare_helper_command(struct argv_array *argv, const char *cmd,
 	strbuf_release(&buf);
 }
 
-__attribute__((format (printf,3,4)))
+__attribute__((format (printf,4,5)))
 static int odb_helper_start(struct odb_helper *o,
 			    struct odb_helper_cmd *cmd,
+			    int use_stdin,
 			    const char *fmt, ...)
 {
 	va_list ap;
@@ -90,7 +91,10 @@ static int odb_helper_start(struct odb_helper *o,
 
 	cmd->child.argv = cmd->argv.argv;
 	cmd->child.use_shell = 1;
-	cmd->child.no_stdin = 1;
+	if (use_stdin)
+		cmd->child.in = -1;
+	else
+		cmd->child.no_stdin = 1;
 	cmd->child.out = -1;
 
 	if (start_command(&cmd->child) < 0) {
@@ -119,7 +123,7 @@ int odb_helper_init(struct odb_helper *o)
 	FILE *fh;
 	struct strbuf line = STRBUF_INIT;
 
-	if (odb_helper_start(o, &cmd, "init") < 0)
+	if (odb_helper_start(o, &cmd, 0, "init") < 0)
 		return -1;
 
 	fh = xfdopen(cmd.child.out, "r");
@@ -179,7 +183,7 @@ static void odb_helper_load_have(struct odb_helper *o)
 		return;
 	o->have_valid = 1;
 
-	if (odb_helper_start(o, &cmd, "have") < 0)
+	if (odb_helper_start(o, &cmd, 0, "have") < 0)
 		return;
 
 	fh = xfdopen(cmd.child.out, "r");
@@ -234,7 +238,7 @@ int odb_helper_get_object(struct odb_helper *o, const unsigned char *sha1,
 	if (!obj)
 		return -1;
 
-	if (odb_helper_start(o, &cmd, "get_git_obj %s", sha1_to_hex(sha1)) < 0)
+	if (odb_helper_start(o, &cmd, 0, "get_git_obj %s", sha1_to_hex(sha1)) < 0)
 		return -1;
 
 	memset(&stream, 0, sizeof(stream));
@@ -321,3 +325,32 @@ int odb_helper_get_object(struct odb_helper *o, const unsigned char *sha1,
 
 	return 0;
 }
+
+int odb_helper_put_object(struct odb_helper *o,
+			  const void *buf, size_t len,
+			  const char *type, unsigned char *sha1)
+{
+	struct odb_helper_cmd cmd;
+
+	if (odb_helper_start(o, &cmd, 1, "put_raw_obj %s %"PRIuMAX" %s",
+			     sha1_to_hex(sha1), (uintmax_t)len, type) < 0)
+		return -1;
+
+	do {
+		int w = xwrite(cmd.child.in, buf, len);
+		if (w < 0) {
+			error("unable to write to odb helper '%s': %s",
+			      o->name, strerror(errno));
+			close(cmd.child.in);
+			close(cmd.child.out);
+			odb_helper_finish(o, &cmd);
+			return -1;
+		}
+		len -= w;
+	} while (len > 0);
+
+	close(cmd.child.in);
+	close(cmd.child.out);
+	odb_helper_finish(o, &cmd);
+	return 0;
+}
diff --git a/odb-helper.h b/odb-helper.h
index 5f28a6e512..0571ba09cb 100644
--- a/odb-helper.h
+++ b/odb-helper.h
@@ -35,5 +35,8 @@ extern int odb_helper_has_object(struct odb_helper *o,
 extern int odb_helper_get_object(struct odb_helper *o,
 				 const unsigned char *sha1,
 				 int fd);
+extern int odb_helper_put_object(struct odb_helper *o,
+				 const void *buf, size_t len,
+				 const char *type, unsigned char *sha1);
 
 #endif /* ODB_HELPER_H */
diff --git a/sha1_file.c b/sha1_file.c
index 4a4f5df5ec..d0155e392f 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -1613,6 +1613,8 @@ int write_sha1_file(const void *buf, unsigned long len, const char *type, unsign
 	 * it out into .git/objects/??/?{38} file.
 	 */
 	write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
+	if (!external_odb_put_object(buf, len, type, sha1))
+		return 0;
 	if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
 		return 0;
 	return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);

-- 
2.14.1.576.g3f707d88cd
Previous: Christian CouderNext: Christian Couder
Message 36 of 49 in “Add initial experimental external ODB support”
  1. 00/40 Add initial experimental external ODB supportChristian Couder, Sep 16, 2017
  2. 01/40 builtin/clone: get rid of 'value' strbufChristian Couder, Sep 16, 2017
  3. 04/40 t0021/rot13-filter: improve error messageChristian Couder, Sep 16, 2017
  4. 05/40 t0021/rot13-filter: add packet_initialize()Christian Couder, Sep 16, 2017
  5. 07/40 Add Git/Packet.pm from parts of t0021/rot13-filter.plChristian Couder, Sep 16, 2017
  6. 11/40 t0400: add 'put_raw_obj' instruction to odb-helper scriptChristian Couder, Sep 16, 2017
  7. 10/40 odb-helper: add odb_helper_init() to send 'init' instructionChristian Couder, Sep 16, 2017
  8. 16/40 Add t0410 to test external ODB transferChristian Couder, Sep 16, 2017
  9. 14/40 t0400: add test for external odb write supportChristian Couder, Sep 16, 2017
  10. 19/40 lib-httpd: add list.shChristian Couder, Sep 16, 2017
  11. 22/40 pack-objects: don't pack objects in external odbsChristian Couder, Sep 16, 2017
  12. 26/40 odb-helper: add init_object_process()Christian Couder, Sep 16, 2017
  13. 29/40 odb-helper: add put_object_process()Christian Couder, Sep 16, 2017
  14. 30/40 Add t0470 to test passing raw objectsChristian Couder, Sep 16, 2017
  15. 31/40 odb-helper: add have_object_process()Christian Couder, Sep 16, 2017
  16. 32/40 Add t0480 to test "have" capability and raw objectsChristian Couder, Sep 16, 2017
  17. 33/40 external-odb: use 'odb=magic' attribute to mark odb blobsChristian Couder, Sep 16, 2017
  18. 38/40 Add tests for 'clone --initial-refspec'Christian Couder, Sep 16, 2017
  19. 39/40 Add t0430 to test cloning using bundlesChristian Couder, Sep 16, 2017
  20. 40/40 Doc/external-odb: explain transfering objects and metadataChristian Couder, Sep 16, 2017
  21. 37/40 clone: disable external odb before initial cloneChristian Couder, Sep 16, 2017
  22. 35/40 clone: add 'initial' param to write_remote_refs()Christian Couder, Sep 16, 2017
  23. 36/40 clone: add --initial-refspec optionChristian Couder, Sep 16, 2017
  24. 34/40 Add Documentation/technical/external-odb.txtChristian Couder, Sep 16, 2017
  25. 25/40 odb-helper: add 'script_mode' to 'struct odb_helper'Christian Couder, Sep 16, 2017
  26. 28/40 Add t0460 to test passing git objectsChristian Couder, Sep 16, 2017
  27. 23/40 Add t0420 to test transfer to HTTP external odbChristian Couder, Sep 16, 2017
  28. 21/40 odb-helper: add odb_helper_get_raw_object()Christian Couder, Sep 16, 2017
  29. 20/40 lib-httpd: add apache-e-odb.confChristian Couder, Sep 16, 2017
  30. 18/40 lib-httpd: add upload.shChristian Couder, Sep 16, 2017
  31. 24/40 external-odb: add 'get_direct' supportChristian Couder, Sep 16, 2017
  32. 27/40 Add t0450 to test 'get_direct' mechanismChristian Couder, Sep 16, 2017
  33. 15/40 Add GIT_NO_EXTERNAL_ODB env variableChristian Couder, Sep 16, 2017
  34. 13/40 external-odb: accept only blobs for nowChristian Couder, Sep 16, 2017
  35. 17/40 lib-httpd: pass config file to start_httpd()Christian Couder, Sep 16, 2017
  36. 12/40 external odb: add 'put_raw_obj' supportChristian Couder, Sep 16, 2017
  37. 06/40 t0021/rot13-filter: add capability functionsChristian Couder, Sep 16, 2017
  38. 09/40 Add initial external odb supportChristian Couder, Sep 16, 2017
  39. Jonathan TanSep 19, 2017
  40. Christian CouderSep 27, 2017
  41. Jonathan TanSep 29, 2017
  42. Ben PeartOct 2, 2017
  43. Christian CouderOct 3, 2017
  44. Jonathan TanOct 4, 2017
  45. 03/40 t0021/rot13-filter: improve 'if .. elsif .. else' styleChristian Couder, Sep 16, 2017
  46. 08/40 sha1_file: prepare for external odbsChristian Couder, Sep 16, 2017
  47. 02/40 t0021/rot13-filter: refactor packet reading functionsChristian Couder, Sep 16, 2017
  48. Ben PeartOct 2, 2017
  49. Christian CouderOct 3, 2017

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.