git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v11 07/10] convert: check for detectable errors in UTF encodings

From
lars.schneider@autodesk.com <lars.schneider@autodesk.com>
Date
Mar 9, 2018, 17:35 UTC
Message-ID
<20180309173536.62012-8-lars.schneider@autodesk.com>
In-Reply-To
<20180309173536.62012-1-lars.schneider@autodesk.com>
From: Lars Schneider <larsxschneider@gmail.com>

Check that new content is valid with respect to the user defined 'working-tree-encoding' attribute.

Signed-off-by: Lars Schneider <larsxschneider@gmail.com>
---
 convert.c                        | 48 ++++++++++++++++++++++++++++++++++
 t/t0028-working-tree-encoding.sh | 56 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
diff --git a/convert.c b/convert.c
index aa59ecfe49..b80d666a6b 100644
--- a/convert.c
+++ b/convert.c
@@ -266,6 +266,51 @@ static int will_convert_lf_to_crlf(size_t len, struct text_stat *stats,
 
 }
 
+static int validate_encoding(const char *path, const char *enc,
+		      const char *data, size_t len, int die_on_error)
+{
+	/* We only check for UTF here as UTF?? can be an alias for UTF-?? */
+	if (istarts_with(enc, "UTF")) {
+		/*
+		 * Check for detectable errors in UTF encodings
+		 */
+		if (has_prohibited_utf_bom(enc, data, len)) {
+			const char *error_msg = _(
+				"BOM is prohibited in '%s' if encoded as %s");
+			/*
+			 * This advice is shown for UTF-??BE and UTF-??LE encodings.
+			 */
+			const char *advise_msg = _(
+				"The file '%s' contains a byte order "
+				"mark (BOM). Please use %.6s as "
+				"working-tree-encoding.");
+			advise(advise_msg, path, enc);
+			if (die_on_error)
+				die(error_msg, path, enc);
+			else {
+				return error(error_msg, path, enc);
+			}
+
+		} else if (is_missing_required_utf_bom(enc, data, len)) {
+			const char *error_msg = _(
+				"BOM is required in '%s' if encoded as %s");
+			const char *advise_msg = _(
+				"The file '%s' is missing a byte order "
+				"mark (BOM). Please use %sBE or %sLE "
+				"(depending on the byte order) as "
+				"working-tree-encoding.");
+			advise(advise_msg, path, enc, enc);
+			if (die_on_error)
+				die(error_msg, path, enc);
+			else {
+				return error(error_msg, path, enc);
+			}
+		}
+
+	}
+	return 0;
+}
+
 static const char *default_encoding = "UTF-8";
 
 static int encode_to_git(const char *path, const char *src, size_t src_len,
@@ -291,6 +336,9 @@ static int encode_to_git(const char *path, const char *src, size_t src_len,
 	if (!buf && !src)
 		return 1;
 
+	if (validate_encoding(path, enc, src, src_len, die_on_error))
+		return 0;
+
 	dst = reencode_string_len(src, src_len, default_encoding, enc,
 				  &dst_len);
 	if (!dst) {
diff --git a/t/t0028-working-tree-encoding.sh b/t/t0028-working-tree-encoding.sh
index e492945a01..e8408dfe5c 100755
--- a/t/t0028-working-tree-encoding.sh
+++ b/t/t0028-working-tree-encoding.sh
@@ -62,6 +62,46 @@ test_expect_success 'check $GIT_DIR/info/attributes support' '
 
 for i in 16 32
 do
+	test_expect_success "check prohibited UTF-${i} BOM" '
+		test_when_finished "git reset --hard HEAD" &&
+
+		echo "*.utf${i}be text working-tree-encoding=utf-${i}be" >>.gitattributes &&
+		echo "*.utf${i}le text working-tree-encoding=utf-${i}LE" >>.gitattributes &&
+
+		# Here we add a UTF-16 (resp. UTF-32) files with BOM (big/little-endian)
+		# but we tell Git to treat it as UTF-16BE/UTF-16LE (resp. UTF-32).
+		# In these cases the BOM is prohibited.
+		cp bebom.utf${i}be.raw bebom.utf${i}be &&
+		test_must_fail git add bebom.utf${i}be 2>err.out &&
+		test_i18ngrep "fatal: BOM is prohibited .* utf-${i}be" err.out &&
+
+		cp lebom.utf${i}le.raw lebom.utf${i}be &&
+		test_must_fail git add lebom.utf${i}be 2>err.out &&
+		test_i18ngrep "fatal: BOM is prohibited .* utf-${i}be" err.out &&
+
+		cp bebom.utf${i}be.raw bebom.utf${i}le &&
+		test_must_fail git add bebom.utf${i}le 2>err.out &&
+		test_i18ngrep "fatal: BOM is prohibited .* utf-${i}LE" err.out &&
+
+		cp lebom.utf${i}le.raw lebom.utf${i}le &&
+		test_must_fail git add lebom.utf${i}le 2>err.out &&
+		test_i18ngrep "fatal: BOM is prohibited .* utf-${i}LE" err.out
+	'
+
+	test_expect_success "check required UTF-${i} BOM" '
+		test_when_finished "git reset --hard HEAD" &&
+
+		echo "*.utf${i} text working-tree-encoding=utf-${i}" >>.gitattributes &&
+
+		cp nobom.utf${i}be.raw nobom.utf${i} &&
+		test_must_fail git add nobom.utf${i} 2>err.out &&
+		test_i18ngrep "fatal: BOM is required .* utf-${i}" err.out &&
+
+		cp nobom.utf${i}le.raw nobom.utf${i} &&
+		test_must_fail git add nobom.utf${i} 2>err.out &&
+		test_i18ngrep "fatal: BOM is required .* utf-${i}" err.out
+	'
+
 	test_expect_success "eol conversion for UTF-${i} encoded files on checkout" '
 		test_when_finished "rm -f crlf.utf${i}.raw lf.utf${i}.raw" &&
 		test_when_finished "git reset --hard HEAD^" &&
@@ -141,4 +181,20 @@ test_expect_success 'error if encoding round trip is not the same during refresh
 	test_i18ngrep "error: .* overwritten by checkout:" err.out
 '
 
+test_expect_success 'error if encoding garbage is already in Git' '
+	BEFORE_STATE=$(git rev-parse HEAD) &&
+	test_when_finished "git reset --hard $BEFORE_STATE" &&
+
+	# Skip the UTF-16 filter for the added file
+	# This simulates a Git version that has no checkoutEncoding support
+	cp nobom.utf16be.raw nonsense.utf16 &&
+	TEST_HASH=$(git hash-object --no-filters -w nonsense.utf16) &&
+	git update-index --add --cacheinfo 100644 $TEST_HASH nonsense.utf16 &&
+	COMMIT=$(git commit-tree -p $(git rev-parse HEAD) -m "plain commit" $(git write-tree)) &&
+	git update-ref refs/heads/master $COMMIT &&
+
+	git diff 2>err.out &&
+	test_i18ngrep "error: BOM is required" err.out
+'
+
 test_done

-- 
2.16.2
Previous: lars.schneider@autodesk.comNext: Junio C Hamano
Message 8 of 25 in “convert: add support for different encodings”
  1. 00/10 convert: add support for different encodingslars.schneider@autodesk.com, Mar 9, 2018
  2. 02/10 strbuf: add xstrdup_toupper()lars.schneider@autodesk.com, Mar 9, 2018
  3. 08/10 convert: advise canonical UTF encoding nameslars.schneider@autodesk.com, Mar 9, 2018
  4. Junio C HamanoMar 9, 2018
  5. Lars SchneiderMar 15, 2018
  6. 03/10 strbuf: add a case insensitive starts_with()lars.schneider@autodesk.com, Mar 9, 2018
  7. 09/10 convert: add tracing for 'working-tree-encoding' attributelars.schneider@autodesk.com, Mar 9, 2018
  8. 07/10 convert: check for detectable errors in UTF encodingslars.schneider@autodesk.com, Mar 9, 2018
  9. Junio C HamanoMar 9, 2018
  10. Lars SchneiderMar 9, 2018
  11. Junio C HamanoMar 9, 2018
  12. 06/10 convert: add 'working-tree-encoding' attributelars.schneider@autodesk.com, Mar 9, 2018
  13. Junio C HamanoMar 9, 2018
  14. Lars SchneiderMar 15, 2018
  15. Torsten BögershausenMar 18, 2018
  16. Lars SchneiderApr 1, 2018
  17. Torsten BögershausenApr 5, 2018
  18. Lars SchneiderApr 15, 2018
  19. 10/10 convert: add round trip check based on 'core.checkRoundtripEncoding'lars.schneider@autodesk.com, Mar 9, 2018
  20. Eric SunshineMar 9, 2018
  21. Junio C HamanoMar 9, 2018
  22. Eric SunshineMar 9, 2018
  23. 01/10 strbuf: remove unnecessary NUL assignment in xstrdup_tolower()lars.schneider@autodesk.com, Mar 9, 2018
  24. 05/10 utf8: add function to detect a missing UTF-16/32 BOMlars.schneider@autodesk.com, Mar 9, 2018
  25. 04/10 utf8: add function to detect prohibited UTF-16/32 BOMlars.schneider@autodesk.com, Mar 9, 2018

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.