Show changes to 5 files +434 −2
Documentation/gitattributes.txt, convert.c, convert.h, sha1_file.c, t/t0028-working-tree-encoding.sh
diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt
index 30687de81a..5ec179d631 100644
--- a/Documentation/gitattributes.txt
+++ b/Documentation/gitattributes.txt
@@ -272,6 +272,72 @@ few exceptions. Even though...
catch potential problems early, safety triggers.
+`working-tree-encoding`
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Git recognizes files encoded with ASCII or one of its supersets (e.g.
+UTF-8 or ISO-8859-1) as text files. All other encodings are usually
+interpreted as binary and consequently built-in Git text processing
+tools (e.g. 'git diff') as well as most Git web front ends do not
+visualize the content.
+
+In these cases you can tell Git the encoding of a file in the working
+directory with the `working-tree-encoding` attribute. If a file with this
+attribute is added to Git, then Git reencodes the content from the
+specified encoding to UTF-8. Finally, Git stores the UTF-8 encoded
+content in its internal data structure (called "the index"). On checkout
+the content is reencoded back to the specified encoding.
+
+Please note that using the `working-tree-encoding` attribute may have a
+number of pitfalls:
+
+- Third party Git implementations that do not support the
+ `working-tree-encoding` attribute will checkout the respective files
+ UTF-8 encoded and not in the expected encoding. Consequently, these
+ files will appear different which typically causes trouble. This is
+ in particular the case for older Git versions and alternative Git
+ implementations such as JGit or libgit2 (as of February 2018).
+
+- Reencoding content requires resources that might slow down certain
+ Git operations (e.g 'git checkout' or 'git add').
+
+Use the `working-tree-encoding` attribute only if you cannot store a file
+in UTF-8 encoding and if you want Git to be able to process the content
+as text.
+
+As an example, use the following attributes if your '*.proj' files are
+UTF-16 encoded with byte order mark (BOM) and you want Git to perform
+automatic line ending conversion based on your platform.
+
+------------------------
+*.proj text working-tree-encoding=UTF-16
+------------------------
+
+Use the following attributes if your '*.proj' files are UTF-16 little
+endian encoded without BOM and you want Git to use Windows line endings
+in the working directory. Please note, it is highly recommended to
+explicitly define the line endings with `eol` if the `working-tree-encoding`
+attribute is used to avoid ambiguity.
+
+------------------------
+*.proj text working-tree-encoding=UTF-16LE eol=CRLF
+------------------------
+
+You can get a list of all available encodings on your platform with the
+following command:
+
+------------------------
+iconv --list
+------------------------
+
+If you do not know the encoding of a file, then you can use the `file`
+command to guess the encoding:
+
+------------------------
+file foo.proj
+------------------------
+
+
`ident`
^^^^^^^
diff --git a/convert.c b/convert.c
index b976eb968c..d20c341b6d 100644
--- a/convert.c
+++ b/convert.c
@@ -7,6 +7,7 @@
#include "sigchain.h"
#include "pkt-line.h"
#include "sub-process.h"
+#include "utf8.h"
/*
* convert.c - convert a file when checking it out and checking it in.
@@ -265,6 +266,110 @@ static int will_convert_lf_to_crlf(size_t len, struct text_stat *stats,
}
+static struct encoding {
+ const char *name;
+ struct encoding *next;
+} *encoding, **encoding_tail;
+static const char *default_encoding = "UTF-8";
+
+static int encode_to_git(const char *path, const char *src, size_t src_len,
+ struct strbuf *buf, struct encoding *enc, int conv_flags)
+{
+ char *dst;
+ int dst_len;
+
+ /*
+ * No encoding is specified or there is nothing to encode.
+ * Tell the caller that the content was not modified.
+ */
+ if (!enc || (src && !src_len))
+ return 0;
+
+ /*
+ * Looks like we got called from "would_convert_to_git()".
+ * This means Git wants to know if it would encode (= modify!)
+ * the content. Let's answer with "yes", since an encoding was
+ * specified.
+ */
+ if (!buf && !src)
+ return 1;
+
+ if (has_prohibited_utf_bom(enc->name, src, src_len)) {
+ const char *error_msg = _(
+ "BOM is prohibited in '%s' if encoded as %s");
+ /*
+ * This advise is shown for UTF-??BE and UTF-??LE encodings.
+ * We truncate the encoding name to 6 chars with %.6s to cut
+ * off the last two "byte order" characters.
+ */
+ const char *advise_msg = _(
+ "The file '%s' contains a byte order mark (BOM). "
+ "Please use %.6s as working-tree-encoding.");
+ advise(advise_msg, path, enc->name);
+ if (conv_flags & CONV_WRITE_OBJECT)
+ die(error_msg, path, enc->name);
+ else
+ error(error_msg, path, enc->name);
+
+ } else if (is_missing_required_utf_bom(enc->name, src, src_len)) {
+ const char *error_msg = _(
+ "BOM is required in '%s' if encoded as %s");
+ const char *advise_msg = _(
+ "The file '%s' is missing a byte order mark (BOM). "
+ "Please use %sBE or %sLE (depending on the byte order) "
+ "as working-tree-encoding.");
+ advise(advise_msg, path, enc->name, enc->name);
+ if (conv_flags & CONV_WRITE_OBJECT)
+ die(error_msg, path, enc->name);
+ else
+ error(error_msg, path, enc->name);
+ }
+
+ dst = reencode_string_len(src, src_len, default_encoding, enc->name,
+ &dst_len);
+ if (!dst) {
+ /*
+ * We could add the blob "as-is" to Git. However, on checkout
+ * we would try to reencode to the original encoding. This
+ * would fail and we would leave the user with a messed-up
+ * working tree. Let's try to avoid this by screaming loud.
+ */
+ const char* msg = _("failed to encode '%s' from %s to %s");
+ if (conv_flags & CONV_WRITE_OBJECT)
+ die(msg, path, enc->name, default_encoding);
+ else
+ error(msg, path, enc->name, default_encoding);
+ }
+
+ strbuf_attach(buf, dst, dst_len, dst_len + 1);
+ return 1;
+}
+
+static int encode_to_worktree(const char *path, const char *src, size_t src_len,
+ struct strbuf *buf, struct encoding *enc)
+{
+ char *dst;
+ int dst_len;
+
+ /*
+ * No encoding is specified or there is nothing to encode.
+ * Tell the caller that the content was not modified.
+ */
+ if (!enc || (src && !src_len))
+ return 0;
+
+ dst = reencode_string_len(src, src_len, enc->name, default_encoding,
+ &dst_len);
+ if (!dst) {
+ error("failed to encode '%s' from %s to %s",
+ path, enc->name, default_encoding);
+ return 0;
+ }
+
+ strbuf_attach(buf, dst, dst_len, dst_len + 1);
+ return 1;
+}
+
static int crlf_to_git(const struct index_state *istate,
const char *path, const char *src, size_t len,
struct strbuf *buf,
@@ -978,6 +1083,35 @@ static int ident_to_worktree(const char *path, const char *src, size_t len,
return 1;
}
+static struct encoding *git_path_check_encoding(struct attr_check_item *check)
+{
+ const char *value = check->value;
+ struct encoding *enc;
+
+ if (ATTR_TRUE(value) || ATTR_FALSE(value) || ATTR_UNSET(value) ||
+ !strlen(value))
+ return NULL;
+
+ for (enc = encoding; enc; enc = enc->next)
+ if (!strcasecmp(value, enc->name))
+ return enc;
+
+ /* Don't encode to the default encoding */
+ if (!strcasecmp(value, default_encoding))
+ return NULL;
+
+ enc = xcalloc(1, sizeof(*enc));
+ /*
+ * Ensure encoding names are always upper case (e.g. UTF-8) to
+ * simplify subsequent string comparisons.
+ */
+ enc->name = xstrdup_toupper(value);
+ *encoding_tail = enc;
+ encoding_tail = &(enc->next);
+
+ return enc;
+}
+
static enum crlf_action git_path_check_crlf(struct attr_check_item *check)
{
const char *value = check->value;
@@ -1033,6 +1167,7 @@ struct conv_attrs {
enum crlf_action attr_action; /* What attr says */
enum crlf_action crlf_action; /* When no attr is set, use core.autocrlf */
int ident;
+ struct encoding *working_tree_encoding; /* Supported encoding or default encoding if NULL */
};
static void convert_attrs(struct conv_attrs *ca, const char *path)
@@ -1041,8 +1176,10 @@ static void convert_attrs(struct conv_attrs *ca, const char *path)
if (!check) {
check = attr_check_initl("crlf", "ident", "filter",
- "eol", "text", NULL);
+ "eol", "text", "working-tree-encoding",
+ NULL);
user_convert_tail = &user_convert;
+ encoding_tail = &encoding;
git_config(read_convert_config, NULL);
}
@@ -1064,6 +1201,7 @@ static void convert_attrs(struct conv_attrs *ca, const char *path)
else if (eol_attr == EOL_CRLF)
ca->crlf_action = CRLF_TEXT_CRLF;
}
+ ca->working_tree_encoding = git_path_check_encoding(ccheck + 5);
} else {
ca->drv = NULL;
ca->crlf_action = CRLF_UNDEFINED;
@@ -1144,6 +1282,13 @@ int convert_to_git(const struct index_state *istate,
src = dst->buf;
len = dst->len;
}
+
+ ret |= encode_to_git(path, src, len, dst, ca.working_tree_encoding, conv_flags);
+ if (ret && dst) {
+ src = dst->buf;
+ len = dst->len;
+ }
+
if (!(conv_flags & CONV_EOL_KEEP_CRLF)) {
ret |= crlf_to_git(istate, path, src, len, dst, ca.crlf_action, conv_flags);
if (ret && dst) {
@@ -1167,6 +1312,7 @@ void convert_to_git_filter_fd(const struct index_state *istate,
if (!apply_filter(path, NULL, 0, fd, dst, ca.drv, CAP_CLEAN, NULL))
die("%s: clean filter '%s' failed", path, ca.drv->name);
+ encode_to_git(path, dst->buf, dst->len, dst, ca.working_tree_encoding, conv_flags);
crlf_to_git(istate, path, dst->buf, dst->len, dst, ca.crlf_action, conv_flags);
ident_to_git(path, dst->buf, dst->len, dst, ca.ident);
}
@@ -1198,6 +1344,12 @@ static int convert_to_working_tree_internal(const char *path, const char *src,
}
}
+ ret |= encode_to_worktree(path, src, len, dst, ca.working_tree_encoding);
+ if (ret) {
+ src = dst->buf;
+ len = dst->len;
+ }
+
ret_filter = apply_filter(
path, src, len, -1, dst, ca.drv, CAP_SMUDGE, dco);
if (!ret_filter && ca.drv && ca.drv->required)
@@ -1664,6 +1816,9 @@ struct stream_filter *get_stream_filter(const char *path, const unsigned char *s
if (ca.drv && (ca.drv->process || ca.drv->smudge || ca.drv->clean))
return NULL;
+ if (ca.working_tree_encoding)
+ return NULL;
+
if (ca.crlf_action == CRLF_AUTO || ca.crlf_action == CRLF_AUTO_CRLF)
return NULL;
diff --git a/convert.h b/convert.h
index 65ab3e5167..1d9539ed0b 100644
--- a/convert.h
+++ b/convert.h
@@ -12,6 +12,7 @@ struct index_state;
#define CONV_EOL_RNDTRP_WARN (1<<1) /* Warn if CRLF to LF to CRLF is different */
#define CONV_EOL_RENORMALIZE (1<<2) /* Convert CRLF to LF */
#define CONV_EOL_KEEP_CRLF (1<<3) /* Keep CRLF line endings as is */
+#define CONV_WRITE_OBJECT (1<<4) /* Content is written to the index */
extern int global_conv_flags_eol;
diff --git a/sha1_file.c b/sha1_file.c
index 6bc7c6ada9..e2f319d677 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -138,7 +138,7 @@ static int get_conv_flags(unsigned flags)
if (flags & HASH_RENORMALIZE)
return CONV_EOL_RENORMALIZE;
else if (flags & HASH_WRITE_OBJECT)
- return global_conv_flags_eol;
+ return global_conv_flags_eol | CONV_WRITE_OBJECT;
else
return 0;
}
diff --git a/t/t0028-working-tree-encoding.sh b/t/t0028-working-tree-encoding.sh
new file mode 100755
index 0000000000..f9ce3e5ef5
--- /dev/null
+++ b/t/t0028-working-tree-encoding.sh
@@ -0,0 +1,210 @@
+#!/bin/sh
+
+test_description='working-tree-encoding conversion via gitattributes'
+
+. ./test-lib.sh
+
+test_expect_success 'setup test repo' '
+ git config core.eol lf &&
+
+ text="hallo there!\ncan you read me?" &&
+ echo "*.utf16 text working-tree-encoding=utf-16" >.gitattributes &&
+ printf "$text" >test.utf8.raw &&
+ printf "$text" | iconv -f UTF-8 -t UTF-16 >test.utf16.raw &&
+ cp test.utf16.raw test.utf16 &&
+
+ git add .gitattributes test.utf16 &&
+ git commit -m initial
+'
+
+test_expect_success 'ensure UTF-8 is stored in Git' '
+ git cat-file -p :test.utf16 >test.utf16.git &&
+ test_cmp_bin test.utf8.raw test.utf16.git &&
+
+ # cleanup
+ rm test.utf8.raw test.utf16.git
+'
+
+test_expect_success 're-encode to UTF-16 on checkout' '
+ rm test.utf16 &&
+ git checkout test.utf16 &&
+ test_cmp_bin test.utf16.raw test.utf16 &&
+
+ # cleanup
+ rm test.utf16.raw
+'
+
+test_expect_success 'check prohibited UTF BOM' '
+ printf "\0a\0b\0c" >nobom.utf16be.raw &&
+ printf "a\0b\0c\0" >nobom.utf16le.raw &&
+ printf "\376\777\0a\0b\0c" >bebom.utf16be.raw &&
+ printf "\777\376a\0b\0c\0" >lebom.utf16le.raw &&
+
+ printf "\0\0\0a\0\0\0b\0\0\0c" >nobom.utf32be.raw &&
+ printf "a\0\0\0b\0\0\0c\0\0\0" >nobom.utf32le.raw &&
+ printf "\0\0\376\777\0\0\0a\0\0\0b\0\0\0c" >bebom.utf32be.raw &&
+ printf "\777\376\0\0a\0\0\0b\0\0\0c\0\0\0" >lebom.utf32le.raw &&
+
+ echo "*.utf16be text working-tree-encoding=utf-16be" >>.gitattributes &&
+ echo "*.utf16le text working-tree-encoding=utf-16le" >>.gitattributes &&
+ echo "*.utf32be text working-tree-encoding=utf-32be" >>.gitattributes &&
+ echo "*.utf32le text working-tree-encoding=utf-32le" >>.gitattributes &&
+
+ # Here we add a UTF-16 files with BOM (big-endian and little-endian)
+ # but we tell Git to treat it as UTF-16BE/UTF-16LE. In these cases
+ # the BOM is prohibited.
+ cp bebom.utf16be.raw bebom.utf16be &&
+ test_must_fail git add bebom.utf16be 2>err.out &&
+ test_i18ngrep "fatal: BOM is prohibited .* UTF-16BE" err.out &&
+
+ cp lebom.utf16le.raw lebom.utf16be &&
+ test_must_fail git add lebom.utf16be 2>err.out &&
+ test_i18ngrep "fatal: BOM is prohibited .* UTF-16BE" err.out &&
+
+ cp bebom.utf16be.raw bebom.utf16le &&
+ test_must_fail git add bebom.utf16le 2>err.out &&
+ test_i18ngrep "fatal: BOM is prohibited .* UTF-16LE" err.out &&
+
+ cp lebom.utf16le.raw lebom.utf16le &&
+ test_must_fail git add lebom.utf16le 2>err.out &&
+ test_i18ngrep "fatal: BOM is prohibited .* UTF-16LE" err.out &&
+
+ # ... and the same for UTF-32
+ cp bebom.utf32be.raw bebom.utf32be &&
+ test_must_fail git add bebom.utf32be 2>err.out &&
+ test_i18ngrep "fatal: BOM is prohibited .* UTF-32BE" err.out &&
+
+ cp lebom.utf32le.raw lebom.utf32be &&
+ test_must_fail git add lebom.utf32be 2>err.out &&
+ test_i18ngrep "fatal: BOM is prohibited .* UTF-32BE" err.out &&
+
+ cp bebom.utf32be.raw bebom.utf32le &&
+ test_must_fail git add bebom.utf32le 2>err.out &&
+ test_i18ngrep "fatal: BOM is prohibited .* UTF-32LE" err.out &&
+
+ cp lebom.utf32le.raw lebom.utf32le &&
+ test_must_fail git add lebom.utf32le 2>err.out &&
+ test_i18ngrep "fatal: BOM is prohibited .* UTF-32LE" err.out &&
+
+ # cleanup
+ git reset --hard HEAD
+'
+
+test_expect_success 'check required UTF BOM' '
+ echo "*.utf32 text working-tree-encoding=utf-32" >>.gitattributes &&
+
+ cp nobom.utf16be.raw nobom.utf16 &&
+ test_must_fail git add nobom.utf16 2>err.out &&
+ test_i18ngrep "fatal: BOM is required .* UTF-16" err.out &&
+
+ cp nobom.utf16le.raw nobom.utf16 &&
+ test_must_fail git add nobom.utf16 2>err.out &&
+ test_i18ngrep "fatal: BOM is required .* UTF-16" err.out &&
+
+ cp nobom.utf32be.raw nobom.utf32 &&
+ test_must_fail git add nobom.utf32 2>err.out &&
+ test_i18ngrep "fatal: BOM is required .* UTF-32" err.out &&
+
+ cp nobom.utf32le.raw nobom.utf32 &&
+ test_must_fail git add nobom.utf32 2>err.out &&
+ test_i18ngrep "fatal: BOM is required .* UTF-32" err.out &&
+
+ # cleanup
+ rm nobom.utf16 nobom.utf32 &&
+ git reset --hard HEAD
+'
+
+test_expect_success 'eol conversion for UTF-16 encoded files on checkout' '
+ printf "one\ntwo\nthree\n" >lf.utf8.raw &&
+ printf "one\r\ntwo\r\nthree\r\n" >crlf.utf8.raw &&
+
+ cat lf.utf8.raw | iconv -f UTF-8 -t UTF-16 >lf.utf16.raw &&
+ cat crlf.utf8.raw | iconv -f UTF-8 -t UTF-16 >crlf.utf16.raw &&
+ cp crlf.utf16.raw eol.utf16 &&
+
+ cat >expectIndexLF <<-\EOF &&
+ i/lf w/-text attr/text eol.utf16
+ EOF
+
+ git add eol.utf16 &&
+ git commit -m eol &&
+
+ # UTF-16 with CRLF (Windows line endings)
+ rm eol.utf16 &&
+ git -c core.eol=crlf checkout eol.utf16 &&
+ test_cmp_bin crlf.utf16.raw eol.utf16 &&
+
+ # Although the file has CRLF in the working tree, ensure LF in the index
+ git ls-files --eol eol.utf16 >actual &&
+ test_cmp expectIndexLF actual &&
+
+ # UTF-16 with LF (Unix line endings)
+ rm eol.utf16 &&
+ git -c core.eol=lf checkout eol.utf16 &&
+ test_cmp_bin lf.utf16.raw eol.utf16 &&
+
+ # The file LF in the working tree, ensure LF in the index
+ git ls-files --eol eol.utf16 >actual &&
+ test_cmp expectIndexLF actual&&
+
+ rm crlf.utf16.raw crlf.utf8.raw lf.utf16.raw lf.utf8.raw &&
+
+ # cleanup
+ git reset --hard HEAD^
+'
+
+test_expect_success 'check unsupported encodings' '
+
+ echo "*.nothing text working-tree-encoding=" >>.gitattributes &&
+ printf "nothing" >t.nothing &&
+ git add t.nothing &&
+
+ echo "*.garbage text working-tree-encoding=garbage" >>.gitattributes &&
+ printf "garbage" >t.garbage &&
+ test_must_fail git add t.garbage 2>err.out &&
+ test_i18ngrep "fatal: failed to encode" err.out &&
+
+ # cleanup
+ rm err.out &&
+ git reset --hard HEAD
+'
+
+test_expect_success 'error if encoding round trip is not the same during refresh' '
+ BEFORE_STATE=$(git rev-parse HEAD) &&
+
+ # Skip the UTF-16 filter for the added file
+ # This simulates a Git version that has no working tree encoding support
+ echo "hallo" >nonsense.utf16 &&
+ TEST_HASH=$(git hash-object --no-filters -w nonsense.utf16) &&
+ git update-index --add --cacheinfo 100644 $TEST_HASH nonsense.utf16 &&
+ COMMIT=$(git commit-tree -p $(git rev-parse HEAD) -m "plain commit" $(git write-tree)) &&
+ git update-ref refs/heads/master $COMMIT &&
+
+ test_must_fail git checkout HEAD^ 2>err.out &&
+ test_i18ngrep "error: .* overwritten by checkout:" err.out &&
+
+ # cleanup
+ rm err.out &&
+ git reset --hard $BEFORE_STATE
+'
+
+test_expect_success 'error if encoding garbage is already in Git' '
+ BEFORE_STATE=$(git rev-parse HEAD) &&
+
+ # Skip the UTF-16 filter for the added file
+ # This simulates a Git version that has no checkoutEncoding support
+ cp nobom.utf16be.raw nonsense.utf16 &&
+ TEST_HASH=$(git hash-object --no-filters -w nonsense.utf16) &&
+ git update-index --add --cacheinfo 100644 $TEST_HASH nonsense.utf16 &&
+ COMMIT=$(git commit-tree -p $(git rev-parse HEAD) -m "plain commit" $(git write-tree)) &&
+ git update-ref refs/heads/master $COMMIT &&
+
+ git diff 2>err.out &&
+ test_i18ngrep "error: BOM is required" err.out &&
+
+ # cleanup
+ rm err.out &&
+ git reset --hard $BEFORE_STATE
+'
+
+test_done
lars.schneider@autodesk.com
