{"thread":{"id":"26974","subject":"[RFC PATCH] convert: add functions to check if we can bypass conversion","startedAt":"2011-04-03T09:10:07Z","lastAt":"2011-04-03T09:10:07Z","messageCount":1,"participants":["Nguyễn Thái Ngọc Duy"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"164987","messageId":"1301821807-16914-1-git-send-email-pclouds@gmail.com","threadId":"26974","inReplyTo":null,"subject":"[RFC PATCH] convert: add functions to check if we can bypass conversion","fromName":"Nguyễn Thái Ngọc Duy","fromEmail":"pclouds@gmail.com","sentAt":"2011-04-03T09:10:07Z","receivedAt":"2011-04-03T09:10:07Z","isPatch":true,"sender":{"key":"pclouds@gmail.com","avatar":"https://avatars.githubusercontent.com/u/720?v=4"},"body":"Blob conversion from/to repository requires the entire blob in memory.\nThe conversion is rarely used most of the time and that requirement\ncould put pressure on memory for large blobs.\n\nAdd two functions to determine early whether we can bypass conversion\nwithout looking at the content.\n\nSigned-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>\n---\n I have patches to checkout loose objects directly to working tree but\n they are ugly and do not support packs. We probably should flag deltified\n objects as \"conversion required\" too.\n\n Anyway I think the intention of this patch is good. Whatever we are\n going to do with large blobs wrt memory usage, we need to cut this\n part out, or support streaming conversion interface. I doubt the\n latter would come.\n\n cache.h   |    3 ++\n convert.c |   79 +++++++++++++++++++++++++++++++++++++++++++++++-------------\n 2 files changed, 65 insertions(+), 17 deletions(-)\n\ndiff --git a/cache.h b/cache.h\nindex 08a9022..be3845d 100644\n--- a/cache.h\n+++ b/cache.h\n@@ -1091,7 +1091,10 @@ extern void trace_repo_setup(const char *prefix);\n /* returns 1 if *dst was used */\n extern int convert_to_git(const char *path, const char *src, size_t len,\n                           struct strbuf *dst, enum safe_crlf checksafe);\n+extern int convert_to_git_needed(const char *path, size_t len,\n+\t\t\t\t enum safe_crlf checksafe);\n extern int convert_to_working_tree(const char *path, const char *src, size_t len, struct strbuf *dst);\n+extern int convert_to_working_tree_needed(const char *path, size_t len);\n extern int renormalize_buffer(const char *path, const char *src, size_t len, struct strbuf *dst);\n \n /* add */\ndiff --git a/convert.c b/convert.c\nindex d5aebed..39545ed 100644\n--- a/convert.c\n+++ b/convert.c\n@@ -188,7 +188,8 @@ static int has_cr_in_index(const char *path)\n }\n \n static int crlf_to_git(const char *path, const char *src, size_t len,\n-\t\t       struct strbuf *buf, enum action action, enum safe_crlf checksafe)\n+\t\t       struct strbuf *buf, enum action action,\n+\t\t       enum safe_crlf checksafe, int dry_run)\n {\n \tstruct text_stat stats;\n \tchar *dst;\n@@ -197,6 +198,9 @@ static int crlf_to_git(const char *path, const char *src, size_t len,\n \t    (action == CRLF_GUESS && auto_crlf == AUTO_CRLF_FALSE) || !len)\n \t\treturn 0;\n \n+\tif (dry_run)\n+\t\treturn 1;\n+\n \tgather_stats(src, len, &stats);\n \n \tif (action == CRLF_AUTO || action == CRLF_GUESS) {\n@@ -257,7 +261,7 @@ static int crlf_to_git(const char *path, const char *src, size_t len,\n }\n \n static int crlf_to_worktree(const char *path, const char *src, size_t len,\n-\t\t\t    struct strbuf *buf, enum action action)\n+\t\t\t    struct strbuf *buf, enum action action, int dry_run)\n {\n \tchar *to_free = NULL;\n \tstruct text_stat stats;\n@@ -265,6 +269,9 @@ static int crlf_to_worktree(const char *path, const char *src, size_t len,\n \tif (!len || determine_output_conversion(action) != EOL_CRLF)\n \t\treturn 0;\n \n+\tif (dry_run)\n+\t\treturn 1;\n+\n \tgather_stats(src, len, &stats);\n \n \t/* No LF? Nothing to convert, regardless. */\n@@ -374,7 +381,7 @@ static int filter_buffer(int in, int out, void *data)\n }\n \n static int apply_filter(const char *path, const char *src, size_t len,\n-                        struct strbuf *dst, const char *cmd)\n+\t\t\tstruct strbuf *dst, const char *cmd, int dry_run)\n {\n \t/*\n \t * Create a pipeline to have the command filter the buffer's\n@@ -390,6 +397,9 @@ static int apply_filter(const char *path, const char *src, size_t len,\n \tif (!cmd)\n \t\treturn 0;\n \n+\tif (dry_run)\n+\t\treturn 1;\n+\n \tmemset(&async, 0, sizeof(async));\n \tasync.proc = filter_buffer;\n \tasync.data = &params;\n@@ -541,11 +551,17 @@ static int count_ident(const char *cp, unsigned long size)\n }\n \n static int ident_to_git(const char *path, const char *src, size_t len,\n-                        struct strbuf *buf, int ident)\n+\t\t\tstruct strbuf *buf, int ident, int dry_run)\n {\n \tchar *dst, *dollar;\n \n-\tif (!ident || !count_ident(src, len))\n+\tif (!ident)\n+\t\treturn 0;\n+\n+\tif (dry_run)\n+\t\treturn 1;\n+\n+\tif(!count_ident(src, len))\n \t\treturn 0;\n \n \t/* only grow if not in place */\n@@ -582,7 +598,7 @@ static int ident_to_git(const char *path, const char *src, size_t len,\n }\n \n static int ident_to_worktree(const char *path, const char *src, size_t len,\n-                             struct strbuf *buf, int ident)\n+\t\t\t     struct strbuf *buf, int ident, int dry_run)\n {\n \tunsigned char sha1[20];\n \tchar *to_free = NULL, *dollar, *spc;\n@@ -591,6 +607,9 @@ static int ident_to_worktree(const char *path, const char *src, size_t len,\n \tif (!ident)\n \t\treturn 0;\n \n+\tif (dry_run)\n+\t\treturn 1;\n+\n \tcnt = count_ident(src, len);\n \tif (!cnt)\n \t\treturn 0;\n@@ -726,8 +745,9 @@ static enum action determine_action(enum action text_attr, enum eol eol_attr)\n \treturn text_attr;\n }\n \n-int convert_to_git(const char *path, const char *src, size_t len,\n-                   struct strbuf *dst, enum safe_crlf checksafe)\n+static int convert_to_git_1(const char *path, const char *src, size_t len,\n+\t\t\t    struct strbuf *dst, enum safe_crlf checksafe,\n+\t\t\t    int dry_run)\n {\n \tstruct git_attr_check check[5];\n \tenum action action = CRLF_GUESS;\n@@ -748,23 +768,39 @@ int convert_to_git(const char *path, const char *src, size_t len,\n \t\t\tfilter = drv->clean;\n \t}\n \n-\tret |= apply_filter(path, src, len, dst, filter);\n+\tret |= apply_filter(path, src, len, dst, filter, dry_run);\n \tif (ret) {\n+\t\tif (dry_run)\n+\t\t\treturn 1;\n \t\tsrc = dst->buf;\n \t\tlen = dst->len;\n \t}\n \taction = determine_action(action, eol_attr);\n-\tret |= crlf_to_git(path, src, len, dst, action, checksafe);\n+\tret |= crlf_to_git(path, src, len, dst, action, checksafe, dry_run);\n \tif (ret) {\n+\t\tif (dry_run)\n+\t\t\treturn 1;\n \t\tsrc = dst->buf;\n \t\tlen = dst->len;\n \t}\n-\treturn ret | ident_to_git(path, src, len, dst, ident);\n+\treturn ret | ident_to_git(path, src, len, dst, ident, dry_run);\n+}\n+\n+int convert_to_git(const char *path, const char *src, size_t len,\n+\t\t   struct strbuf *dst, enum safe_crlf checksafe)\n+{\n+\treturn convert_to_git_1(path, src, len, dst, checksafe, 0);\n+}\n+\n+int convert_to_git_needed(const char *path, size_t len,\n+\t\t\t  enum safe_crlf checksafe)\n+{\n+\treturn convert_to_git_1(path, NULL, len, NULL, checksafe, 1);\n }\n \n static int convert_to_working_tree_internal(const char *path, const char *src,\n \t\t\t\t\t    size_t len, struct strbuf *dst,\n-\t\t\t\t\t    int normalizing)\n+\t\t\t\t\t    int normalizing, int dry_run)\n {\n \tstruct git_attr_check check[5];\n \tenum action action = CRLF_GUESS;\n@@ -785,8 +821,10 @@ static int convert_to_working_tree_internal(const char *path, const char *src,\n \t\t\tfilter = drv->smudge;\n \t}\n \n-\tret |= ident_to_worktree(path, src, len, dst, ident);\n+\tret |= ident_to_worktree(path, src, len, dst, ident, dry_run);\n \tif (ret) {\n+\t\tif (dry_run)\n+\t\t\treturn 1;\n \t\tsrc = dst->buf;\n \t\tlen = dst->len;\n \t}\n@@ -796,23 +834,30 @@ static int convert_to_working_tree_internal(const char *path, const char *src,\n \t */\n \tif (filter || !normalizing) {\n \t\taction = determine_action(action, eol_attr);\n-\t\tret |= crlf_to_worktree(path, src, len, dst, action);\n+\t\tret |= crlf_to_worktree(path, src, len, dst, action, dry_run);\n \t\tif (ret) {\n+\t\t\tif (dry_run)\n+\t\t\t\treturn 1;\n \t\t\tsrc = dst->buf;\n \t\t\tlen = dst->len;\n \t\t}\n \t}\n-\treturn ret | apply_filter(path, src, len, dst, filter);\n+\treturn ret | apply_filter(path, src, len, dst, filter, dry_run);\n }\n \n int convert_to_working_tree(const char *path, const char *src, size_t len, struct strbuf *dst)\n {\n-\treturn convert_to_working_tree_internal(path, src, len, dst, 0);\n+\treturn convert_to_working_tree_internal(path, src, len, dst, 0, 0);\n+}\n+\n+int convert_to_working_tree_needed(const char *path, size_t len)\n+{\n+\treturn convert_to_working_tree_internal(path, NULL, len, NULL, 0, 1);\n }\n \n int renormalize_buffer(const char *path, const char *src, size_t len, struct strbuf *dst)\n {\n-\tint ret = convert_to_working_tree_internal(path, src, len, dst, 1);\n+\tint ret = convert_to_working_tree_internal(path, src, len, dst, 1, 0);\n \tif (ret) {\n \t\tsrc = dst->buf;\n \t\tlen = dst->len;\n-- \n1.7.4.74.g639db\n"}]}