git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 39/40] external-odb: use 'odb=magic' attribute to mark odb blobs

From
Christian Couder <christian.couder@gmail.com>
Date
Jan 3, 2018, 16:34 UTC
Message-ID
<20180103163403.11303-40-chriscool@tuxfamily.org>
In-Reply-To
<20180103163403.11303-1-chriscool@tuxfamily.org>

To tell which blobs should be sent to the "magic" external odb, let's require that the blobs be marked using the 'odb=magic' attribute.

Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
 external-odb.c                         | 25 ++++++++++++++++++++++---
 external-odb.h                         |  3 ++-
 sha1_file.c                            | 20 +++++++++++++++-----
 t/t0400-external-odb.sh                |  3 +++
 t/t0410-transfer-e-odb.sh              |  3 +++
 t/t0420-transfer-http-e-odb.sh         |  3 +++
 t/t0470-read-object-http-e-odb.sh      |  3 +++
 t/t0480-read-object-have-http-e-odb.sh |  3 +++
 8 files changed, 54 insertions(+), 9 deletions(-)
diff --git a/external-odb.c b/external-odb.c
index 50c1cec50b..e3a05e24e3 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -2,6 +2,7 @@
 #include "external-odb.h"
 #include "odb-helper.h"
 #include "config.h"
+#include "attr.h"
 
 static struct odb_helper *helpers;
 static struct odb_helper **helpers_tail = &helpers;
@@ -165,19 +166,37 @@ int external_odb_get_object(const unsigned char *sha1)
 	return external_odb_do_get_object(sha1);
 }
 
+static int has_odb_attrs(struct odb_helper *o, const char *path)
+{
+	static struct attr_check *check;
+
+	if (!check)
+		check = attr_check_initl("odb", NULL);
+
+	if (!git_check_attr(path, check)) {
+		const char *value = check->items[0].value;
+		return value ? !strcmp(o->name, value) : 0;
+	}
+	return 0;
+}
+
 int external_odb_put_object(const void *buf, size_t len,
-			    const char *type, unsigned char *sha1)
+			    const char *type, unsigned char *sha1,
+			    const char *path)
 {
 	struct odb_helper *o;
 
 	external_odb_init();
 
 	/* For now accept only blobs */
-	if (strcmp(type, "blob"))
+	if (!path || strcmp(type, "blob"))
 		return 1;
 
 	for (o = helpers; o; o = o->next) {
-		int r = odb_helper_put_object(o, buf, len, type, sha1);
+		int r;
+		if (!has_odb_attrs(o, path))
+			continue;
+		r = odb_helper_put_object(o, buf, len, type, sha1);
 		if (r <= 0)
 			return r;
 	}
diff --git a/external-odb.h b/external-odb.h
index 26bb931685..5a8936f417 100644
--- a/external-odb.h
+++ b/external-odb.h
@@ -7,6 +7,7 @@ extern int external_odb_has_object(const unsigned char *sha1);
 extern int external_odb_get_object(const unsigned char *sha1);
 extern int external_odb_get_direct(const unsigned char *sha1);
 extern int external_odb_put_object(const void *buf, size_t len,
-				   const char *type, unsigned char *sha1);
+				   const char *type, unsigned char *sha1,
+				   const char *path);
 
 #endif /* EXTERNAL_ODB_H */
diff --git a/sha1_file.c b/sha1_file.c
index 300029459f..d3f395e967 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -1696,7 +1696,9 @@ static int freshen_packed_object(const unsigned char *sha1)
 	return 1;
 }
 
-int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)
+static int write_sha1_file_with_path(const void *buf, unsigned long len,
+				     const char *type, unsigned char *sha1,
+				     const char *path)
 {
 	char hdr[32];
 	int hdrlen = sizeof(hdr);
@@ -1705,13 +1707,19 @@ int write_sha1_file(const void *buf, unsigned long len, const char *type, unsign
 	 * it out into .git/objects/??/?{38} file.
 	 */
 	write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
-	if (!external_odb_put_object(buf, len, type, sha1))
+	if (!external_odb_put_object(buf, len, type, sha1, path))
 		return 0;
 	if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
 		return 0;
 	return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
 }
 
+int write_sha1_file(const void *buf, unsigned long len,
+		    const char *type, unsigned char *sha1)
+{
+	return write_sha1_file_with_path(buf, len, type, sha1, NULL);
+}
+
 int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
 			     struct object_id *oid, unsigned flags)
 {
@@ -1832,7 +1840,8 @@ static int index_mem(struct object_id *oid, void *buf, size_t size,
 	}
 
 	if (write_object)
-		ret = write_sha1_file(buf, size, typename(type), oid->hash);
+		ret = write_sha1_file_with_path(buf, size, typename(type),
+						oid->hash, path);
 	else
 		ret = hash_sha1_file(buf, size, typename(type), oid->hash);
 	if (re_allocated)
@@ -1854,8 +1863,9 @@ static int index_stream_convert_blob(struct object_id *oid, int fd,
 				 get_safe_crlf(flags));
 
 	if (write_object)
-		ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
-				      oid->hash);
+		ret = write_sha1_file_with_path(sbuf.buf, sbuf.len,
+						typename(OBJ_BLOB),
+						oid->hash, path);
 	else
 		ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
 				     oid->hash);
diff --git a/t/t0400-external-odb.sh b/t/t0400-external-odb.sh
index f924de870f..944b7bb28f 100755
--- a/t/t0400-external-odb.sh
+++ b/t/t0400-external-odb.sh
@@ -71,6 +71,9 @@ test_expect_success 'helper can add objects to alt repo' '
 
 test_expect_success 'commit adds objects to alt repo' '
 	test_config odb.magic.scriptCommand "$HELPER" &&
+	echo "* odb=magic" >.gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
 	test_commit three &&
 	hash3=$(git ls-tree HEAD | grep three.t | cut -f1 | cut -d\  -f3) &&
 	content=$(git -C alt-repo show "$hash3") &&
diff --git a/t/t0410-transfer-e-odb.sh b/t/t0410-transfer-e-odb.sh
index 065ec7d759..fd3e37918c 100755
--- a/t/t0410-transfer-e-odb.sh
+++ b/t/t0410-transfer-e-odb.sh
@@ -111,6 +111,9 @@ test_expect_success 'setup other repo and its alternate repo' '
 '
 
 test_expect_success 'new blobs are put in first object store' '
+	echo "* odb=magic" >.gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
 	test_commit one &&
 	hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\  -f3) &&
 	content=$(cd alt-repo1 && git show "$hash1") &&
diff --git a/t/t0420-transfer-http-e-odb.sh b/t/t0420-transfer-http-e-odb.sh
index f84fe950ec..d307af0457 100755
--- a/t/t0420-transfer-http-e-odb.sh
+++ b/t/t0420-transfer-http-e-odb.sh
@@ -94,6 +94,9 @@ test_expect_success 'can delete uploaded files' '
 FILES_DIR="httpd/www/files"
 
 test_expect_success 'new blobs are transfered to the http server' '
+	echo "* odb=magic" >.gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
 	test_commit one &&
 	hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\  -f3) &&
 	echo "$hash1-4-blob" >expected &&
diff --git a/t/t0470-read-object-http-e-odb.sh b/t/t0470-read-object-http-e-odb.sh
index 774528c04f..d814a43d59 100755
--- a/t/t0470-read-object-http-e-odb.sh
+++ b/t/t0470-read-object-http-e-odb.sh
@@ -62,6 +62,9 @@ test_expect_success 'can delete uploaded files' '
 FILES_DIR="httpd/www/files"
 
 test_expect_success 'new blobs are transfered to the http server' '
+	echo "* odb=magic" >.gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
 	test_commit one &&
 	hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\  -f3) &&
 	echo "$hash1-4-blob" >expected &&
diff --git a/t/t0480-read-object-have-http-e-odb.sh b/t/t0480-read-object-have-http-e-odb.sh
index 056a40f2bb..fe1fac5ef3 100755
--- a/t/t0480-read-object-have-http-e-odb.sh
+++ b/t/t0480-read-object-have-http-e-odb.sh
@@ -62,6 +62,9 @@ test_expect_success 'can delete uploaded files' '
 FILES_DIR="httpd/www/files"
 
 test_expect_success 'new blobs are transfered to the http server' '
+	echo "* odb=magic" >.gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
 	test_commit one &&
 	hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\  -f3) &&
 	echo "$hash1-4-blob" >expected &&

-- 
2.16.0.rc0.16.g82191dbc6c.dirty
Previous: Christian CouderNext: Christian Couder
Message 27 of 51 in “Promisor remotes and external ODB support”
  1. 00/40 Promisor remotes and external ODB supportChristian Couder, Jan 3, 2018
  2. 08/40 index-pack: refactor writing of .keep filesChristian Couder, Jan 3, 2018
  3. 12/40 rev-list: support termination at promisor objectsChristian Couder, Jan 3, 2018
  4. 11/40 sha1_file: support lazily fetching missing objectsChristian Couder, Jan 3, 2018
  5. 09/40 introduce fetch-object: fetch one promisor objectChristian Couder, Jan 3, 2018
  6. 18/40 t0400: add 'put_raw_obj' instruction to odb-helper scriptChristian Couder, Jan 3, 2018
  7. 21/40 t0400: add test for external odb write supportChristian Couder, Jan 3, 2018
  8. 24/40 lib-httpd: add upload.shChristian Couder, Jan 3, 2018
  9. 25/40 lib-httpd: add list.shChristian Couder, Jan 3, 2018
  10. 23/40 lib-httpd: pass config file to start_httpd()Christian Couder, Jan 3, 2018
  11. 27/40 odb-helper: add odb_helper_get_raw_object()Christian Couder, Jan 3, 2018
  12. 29/40 Add t0420 to test transfer to HTTP external odbChristian Couder, Jan 3, 2018
  13. 26/40 lib-httpd: add apache-e-odb.confChristian Couder, Jan 3, 2018
  14. 28/40 pack-objects: don't pack objects in external odbsChristian Couder, Jan 3, 2018
  15. Jeff HostetlerJan 4, 2018
  16. Christian CouderMar 19, 2018
  17. 30/40 external-odb: add 'get_direct' supportChristian Couder, Jan 3, 2018
  18. 31/40 odb-helper: add 'script_mode' to 'struct odb_helper'Christian Couder, Jan 3, 2018
  19. 32/40 odb-helper: add init_object_process()Christian Couder, Jan 3, 2018
  20. 34/40 Add t0460 to test passing git objectsChristian Couder, Jan 3, 2018
  21. 33/40 Add t0450 to test 'get_direct' mechanismChristian Couder, Jan 3, 2018
  22. 40/40 Add Documentation/technical/external-odb.txtChristian Couder, Jan 3, 2018
  23. 35/40 odb-helper: add put_object_process()Christian Couder, Jan 3, 2018
  24. 36/40 Add t0470 to test passing raw objectsChristian Couder, Jan 3, 2018
  25. 37/40 odb-helper: add have_object_process()Christian Couder, Jan 3, 2018
  26. 38/40 Add t0480 to test "have" capability and raw objectsChristian Couder, Jan 3, 2018
  27. 39/40 external-odb: use 'odb=magic' attribute to mark odb blobsChristian Couder, Jan 3, 2018
  28. 10/40 external-odb: implement external_odb_get_directChristian Couder, Jan 3, 2018
  29. Jeff HostetlerJan 4, 2018
  30. Christian CouderJan 15, 2018
  31. 22/40 Add t0410 to test external ODB transferChristian Couder, Jan 3, 2018
  32. 20/40 external-odb: accept only blobs for nowChristian Couder, Jan 3, 2018
  33. 19/40 external odb: add 'put_raw_obj' supportChristian Couder, Jan 3, 2018
  34. 13/40 gc: do not repack promisor packfilesChristian Couder, Jan 3, 2018
  35. 16/40 odb-helper: add 'enum odb_helper_type'Christian Couder, Jan 3, 2018
  36. 17/40 odb-helper: add odb_helper_init() to send 'init' instructionChristian Couder, Jan 3, 2018
  37. 14/40 sha1_file: prepare for external odbsChristian Couder, Jan 3, 2018
  38. Jeff HostetlerJan 4, 2018
  39. Christian CouderJan 16, 2018
  40. 07/40 fsck: support promisor objects as CLI argumentChristian Couder, Jan 3, 2018
  41. 15/40 external-odb: add script mode supportChristian Couder, Jan 3, 2018
  42. Jeff HostetlerJan 4, 2018
  43. Christian CouderMar 19, 2018
  44. 06/40 fsck: support referenced promisor objectsChristian Couder, Jan 3, 2018
  45. 05/40 fsck: support refs pointing to promisor objectsChristian Couder, Jan 3, 2018
  46. 02/40 Add GIT_NO_EXTERNAL_ODB env variableChristian Couder, Jan 3, 2018
  47. 01/40 Add initial external odb supportChristian Couder, Jan 3, 2018
  48. Jeff HostetlerJan 4, 2018
  49. Christian CouderJan 15, 2018
  50. 03/40 external-odb: add has_external_odb()Christian Couder, Jan 3, 2018
  51. 04/40 fsck: introduce promisor objectsChristian Couder, Jan 3, 2018

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.