git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 02/19] convert: add [async_]convert_to_working_tree_ca() variants

From
Matheus Tavares <matheus.bernardino@usp.br>
Date
Sep 22, 2020, 22:49 UTC
Message-ID
<313c3bcbebb9460d62cc29692964111565043de0.1600814153.git.matheus.bernardino@usp.br>
In-Reply-To
<cover.1600814153.git.matheus.bernardino@usp.br>
From: Jeff Hostetler <jeffhost@microsoft.com>

Separate the attribute gathering from the actual conversion by adding _ca() variants of the conversion functions. These variants receive a precomputed 'struct conv_attrs', not relying, thus, on a index state. They will be used in a future patch adding parallel checkout support, for two reasons:

- We will already load the conversion attributes in checkout_entry(),
  before conversion, to decide whether a path is eligible for parallel
  checkout. Therefore, it would be wasteful to load them again later,
  for the actual conversion.
- The parallel workers will be responsible for reading, converting and
  writing blobs to the working tree. They won't have access to the main
  process' index state, so they cannot load the attributes. Instead,
  they will receive the preloaded ones and call the _ca() variant of
  the conversion functions. Furthermore, the attributes machinery is
  optimized to handle paths in sequential order, so it's better to leave
  it for the main process, anyway.
Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com>
[matheus.bernardino: squash, remove one function definition and reword]
Signed-off-by: Matheus Tavares <matheus.bernardino@usp.br>
---
 convert.c | 50 ++++++++++++++++++++++++++++++++++++--------------
 convert.h |  9 +++++++++
 2 files changed, 45 insertions(+), 14 deletions(-)
diff --git a/convert.c b/convert.c
index 941a845692..55bcce891c 100644
--- a/convert.c
+++ b/convert.c
@@ -1447,7 +1447,7 @@ void convert_to_git_filter_fd(const struct index_state *istate,
 	ident_to_git(dst->buf, dst->len, dst, ca.ident);
 }
 
-static int convert_to_working_tree_internal(const struct index_state *istate,
+static int convert_to_working_tree_internal(const struct conv_attrs *ca,
 					    const char *path, const char *src,
 					    size_t len, struct strbuf *dst,
 					    int normalizing,
@@ -1455,11 +1455,8 @@ static int convert_to_working_tree_internal(const struct index_state *istate,
 					    struct delayed_checkout *dco)
 {
 	int ret = 0, ret_filter = 0;
-	struct conv_attrs ca;
-
-	convert_attrs(istate, &ca, path);
 
-	ret |= ident_to_worktree(src, len, dst, ca.ident);
+	ret |= ident_to_worktree(src, len, dst, ca->ident);
 	if (ret) {
 		src = dst->buf;
 		len = dst->len;
@@ -1469,24 +1466,24 @@ static int convert_to_working_tree_internal(const struct index_state *istate,
 	 * is a smudge or process filter (even if the process filter doesn't
 	 * support smudge).  The filters might expect CRLFs.
 	 */
-	if ((ca.drv && (ca.drv->smudge || ca.drv->process)) || !normalizing) {
-		ret |= crlf_to_worktree(src, len, dst, ca.crlf_action);
+	if ((ca->drv && (ca->drv->smudge || ca->drv->process)) || !normalizing) {
+		ret |= crlf_to_worktree(src, len, dst, ca->crlf_action);
 		if (ret) {
 			src = dst->buf;
 			len = dst->len;
 		}
 	}
 
-	ret |= encode_to_worktree(path, src, len, dst, ca.working_tree_encoding);
+	ret |= encode_to_worktree(path, src, len, dst, ca->working_tree_encoding);
 	if (ret) {
 		src = dst->buf;
 		len = dst->len;
 	}
 
 	ret_filter = apply_filter(
-		path, src, len, -1, dst, ca.drv, CAP_SMUDGE, meta, dco);
-	if (!ret_filter && ca.drv && ca.drv->required)
-		die(_("%s: smudge filter %s failed"), path, ca.drv->name);
+		path, src, len, -1, dst, ca->drv, CAP_SMUDGE, meta, dco);
+	if (!ret_filter && ca->drv && ca->drv->required)
+		die(_("%s: smudge filter %s failed"), path, ca->drv->name);
 
 	return ret | ret_filter;
 }
@@ -1497,7 +1494,9 @@ int async_convert_to_working_tree(const struct index_state *istate,
 				  const struct checkout_metadata *meta,
 				  void *dco)
 {
-	return convert_to_working_tree_internal(istate, path, src, len, dst, 0, meta, dco);
+	struct conv_attrs ca;
+	convert_attrs(istate, &ca, path);
+	return convert_to_working_tree_internal(&ca, path, src, len, dst, 0, meta, dco);
 }
 
 int convert_to_working_tree(const struct index_state *istate,
@@ -1505,13 +1504,36 @@ int convert_to_working_tree(const struct index_state *istate,
 			    size_t len, struct strbuf *dst,
 			    const struct checkout_metadata *meta)
 {
-	return convert_to_working_tree_internal(istate, path, src, len, dst, 0, meta, NULL);
+	struct conv_attrs ca;
+	convert_attrs(istate, &ca, path);
+	return convert_to_working_tree_internal(&ca, path, src, len, dst, 0, meta, NULL);
+}
+
+int async_convert_to_working_tree_ca(const struct conv_attrs *ca,
+				     const char *path, const char *src,
+				     size_t len, struct strbuf *dst,
+				     const struct checkout_metadata *meta,
+				     void *dco)
+{
+	return convert_to_working_tree_internal(ca, path, src, len, dst, 0, meta, dco);
+}
+
+int convert_to_working_tree_ca(const struct conv_attrs *ca,
+			       const char *path, const char *src,
+			       size_t len, struct strbuf *dst,
+			       const struct checkout_metadata *meta)
+{
+	return convert_to_working_tree_internal(ca, path, src, len, dst, 0, meta, NULL);
 }
 
 int renormalize_buffer(const struct index_state *istate, const char *path,
 		       const char *src, size_t len, struct strbuf *dst)
 {
-	int ret = convert_to_working_tree_internal(istate, path, src, len, dst, 1, NULL, NULL);
+	struct conv_attrs ca;
+	int ret;
+
+	convert_attrs(istate, &ca, path);
+	ret = convert_to_working_tree_internal(&ca, path, src, len, dst, 1, NULL, NULL);
 	if (ret) {
 		src = dst->buf;
 		len = dst->len;
diff --git a/convert.h b/convert.h
index aeb4a1be9a..46d537d1ae 100644
--- a/convert.h
+++ b/convert.h
@@ -100,11 +100,20 @@ int convert_to_working_tree(const struct index_state *istate,
 			    const char *path, const char *src,
 			    size_t len, struct strbuf *dst,
 			    const struct checkout_metadata *meta);
+int convert_to_working_tree_ca(const struct conv_attrs *ca,
+			       const char *path, const char *src,
+			       size_t len, struct strbuf *dst,
+			       const struct checkout_metadata *meta);
 int async_convert_to_working_tree(const struct index_state *istate,
 				  const char *path, const char *src,
 				  size_t len, struct strbuf *dst,
 				  const struct checkout_metadata *meta,
 				  void *dco);
+int async_convert_to_working_tree_ca(const struct conv_attrs *ca,
+				     const char *path, const char *src,
+				     size_t len, struct strbuf *dst,
+				     const struct checkout_metadata *meta,
+				     void *dco);
 int async_query_available_blobs(const char *cmd,
 				struct string_list *available_paths);
 int renormalize_buffer(const struct index_state *istate,

-- 
2.28.0
Previous: Matheus TavaresNext: Matheus Tavares
Message 30 of 61 in “[RFC] Parallel checkout”
  1. 00/21 [RFC] Parallel checkoutMatheus Tavares, Aug 10, 2020
  2. 01/21 convert: make convert_attrs() and convert structs publicMatheus Tavares, Aug 10, 2020
  3. 02/21 convert: add [async_]convert_to_working_tree_ca() variantsMatheus Tavares, Aug 10, 2020
  4. 03/21 convert: add get_stream_filter_ca() variantMatheus Tavares, Aug 10, 2020
  5. 04/21 convert: add conv_attrs classificationMatheus Tavares, Aug 10, 2020
  6. 05/21 entry: extract a header file for entry.c functionsMatheus Tavares, Aug 10, 2020
  7. 06/21 entry: make fstat_output() and read_blob_entry() publicMatheus Tavares, Aug 10, 2020
  8. 07/21 entry: extract cache_entry update from write_entry()Matheus Tavares, Aug 10, 2020
  9. 08/21 entry: move conv_attrs lookup up to checkout_entry()Matheus Tavares, Aug 10, 2020
  10. 09/21 entry: add checkout_entry_ca() which takes preloaded conv_attrsMatheus Tavares, Aug 10, 2020
  11. 10/21 unpack-trees: add basic support for parallel checkoutMatheus Tavares, Aug 10, 2020
  12. 11/21 parallel-checkout: make it truly parallelMatheus Tavares, Aug 10, 2020
  13. Jeff HostetlerAug 19, 2020
  14. Matheus Tavares BernardinoAug 20, 2020
  15. Jeff HostetlerAug 20, 2020
  16. 12/21 parallel-checkout: add configuration optionsMatheus Tavares, Aug 10, 2020
  17. 13/21 parallel-checkout: support progress displayingMatheus Tavares, Aug 10, 2020
  18. 14/21 make_transient_cache_entry(): optionally alloc from mem_poolMatheus Tavares, Aug 10, 2020
  19. 15/21 builtin/checkout.c: complete parallel checkout supportMatheus Tavares, Aug 10, 2020
  20. 16/21 checkout-index: add parallel checkout supportMatheus Tavares, Aug 10, 2020
  21. 17/21 parallel-checkout: avoid stat() calls in workersMatheus Tavares, Aug 10, 2020
  22. 18/21 entry: use is_dir_sep() when checking leading dirsMatheus Tavares, Aug 10, 2020
  23. 19/21 symlinks: make has_dirs_only_path() track FL_NOENTMatheus Tavares, Aug 10, 2020
  24. 20/21 parallel-checkout: create leading dirs in workersMatheus Tavares, Aug 10, 2020
  25. 21/21 parallel-checkout: skip checking the working tree on cloneMatheus Tavares, Aug 10, 2020
  26. Jeff HostetlerAug 12, 2020
  27. 00/19 Parallel Checkout (part I)Matheus Tavares, Sep 22, 2020
  28. 01/19 convert: make convert_attrs() and convert structs publicMatheus Tavares, Sep 22, 2020
  29. 03/19 convert: add get_stream_filter_ca() variantMatheus Tavares, Sep 22, 2020
  30. 02/19 convert: add [async_]convert_to_working_tree_ca() variantsMatheus Tavares, Sep 22, 2020
  31. 04/19 convert: add conv_attrs classificationMatheus Tavares, Sep 22, 2020
  32. 05/19 entry: extract a header file for entry.c functionsMatheus Tavares, Sep 22, 2020
  33. 06/19 entry: make fstat_output() and read_blob_entry() publicMatheus Tavares, Sep 22, 2020
  34. 07/19 entry: extract cache_entry update from write_entry()Matheus Tavares, Sep 22, 2020
  35. 08/19 entry: move conv_attrs lookup up to checkout_entry()Matheus Tavares, Sep 22, 2020
  36. Jeff HostetlerOct 1, 2020
  37. Jeff HostetlerOct 1, 2020
  38. 09/19 entry: add checkout_entry_ca() which takes preloaded conv_attrsMatheus Tavares, Sep 22, 2020
  39. 10/19 unpack-trees: add basic support for parallel checkoutMatheus Tavares, Sep 22, 2020
  40. parallel-checkout: drop unused checkout state parameterJeff King, Oct 5, 2020
  41. Matheus Tavares BernardinoOct 5, 2020
  42. Jeff KingOct 5, 2020
  43. 11/19 parallel-checkout: make it truly parallelMatheus Tavares, Sep 22, 2020
  44. Martin ÅgrenSep 29, 2020
  45. Matheus Tavares BernardinoSep 30, 2020
  46. 12/19 parallel-checkout: support progress displayingMatheus Tavares, Sep 22, 2020
  47. 13/19 make_transient_cache_entry(): optionally alloc from mem_poolMatheus Tavares, Sep 22, 2020
  48. 14/19 builtin/checkout.c: complete parallel checkout supportMatheus Tavares, Sep 22, 2020
  49. 15/19 checkout-index: add parallel checkout supportMatheus Tavares, Sep 22, 2020
  50. 16/19 parallel-checkout: add tests for basic operationsMatheus Tavares, Sep 22, 2020
  51. Jonathan NiederOct 20, 2020
  52. Taylor BlauOct 20, 2020
  53. Matheus Tavares BernardinoOct 20, 2020
  54. Junio C HamanoOct 20, 2020
  55. Matheus Tavares BernardinoOct 20, 2020
  56. Jonathan NiederOct 20, 2020
  57. Junio C HamanoOct 20, 2020
  58. 17/19 parallel-checkout: add tests related to clone collisionsMatheus Tavares, Sep 22, 2020
  59. 18/19 parallel-checkout: add tests related to .gitattributesMatheus Tavares, Sep 22, 2020
  60. 19/19 ci: run test round with parallel-checkout enabledMatheus Tavares, Sep 22, 2020
  61. Jeff HostetlerOct 1, 2020

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.