git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[JGIT PATCH 4/4] Intelligent parsing of ambiguously encoded meta data.

From
Robin Rosenberg <robin.rosenberg@dewire.com>
Date
Oct 12, 2008, 22:51 UTC
Message-ID
<1223851860-13068-5-git-send-email-robin.rosenberg@dewire.com>
In-Reply-To
<1223851860-13068-4-git-send-email-robin.rosenberg@dewire.com>

We cannot trust meta data to be encoded in any particular way, so we try different encodings. First we try UTF-8, which is the only sane encoding for non-local data, even when used in regions where eight bit legacy encodings are common. The chance of mistakenly parsing non-UTF-8 data as valid UTF-8 is varies from extremely low (western encodings) to low for most other encodings. If the data does not look like UTF-8, we try the suggested encoding. If that fails we try the user locale and finally, if that fails we try ISO-8859-1, which cannot fail.

Signed-off-by: Robin Rosenberg <robin.rosenberg@dewire.com>
---
 .../spearce/jgit/revwalk/RevCommitParseTest.java   |  119 ++++++++++++++++++++
 .../src/org/spearce/jgit/util/RawParseUtils.java   |   63 ++++++++++-
 2 files changed, 179 insertions(+), 3 deletions(-)
diff --git a/org.spearce.jgit.test/tst/org/spearce/jgit/revwalk/RevCommitParseTest.java b/org.spearce.jgit.test/tst/org/spearce/jgit/revwalk/RevCommitParseTest.java
index 3d9d42d..805e29e 100644
--- a/org.spearce.jgit.test/tst/org/spearce/jgit/revwalk/RevCommitParseTest.java
+++ b/org.spearce.jgit.test/tst/org/spearce/jgit/revwalk/RevCommitParseTest.java
@@ -37,6 +37,8 @@
 
 package org.spearce.jgit.revwalk;
 
+import java.io.ByteArrayOutputStream;
+
 import org.spearce.jgit.lib.ObjectId;
 import org.spearce.jgit.lib.PersonIdent;
 import org.spearce.jgit.lib.RepositoryTestCase;
@@ -130,6 +132,123 @@ public void testParse_WeirdHeaderOnlyCommit() throws Exception {
 		assertEquals("", c.getShortMessage());
 	}
 
+	public void testParse_implicit_UTF8_encoded() throws Exception {
+		final ByteArrayOutputStream b = new ByteArrayOutputStream();
+		b.write("tree 9788669ad918b6fcce64af8882fc9a81cb6aba67\n".getBytes("UTF-8"));
+		b.write("author F\u00f6r fattare <a_u_thor@example.com> 1218123387 +0700\n".getBytes("UTF-8"));
+		b.write("committer C O. Miter <c@example.com> 1218123390 -0500\n".getBytes("UTF-8"));
+		b.write("\n".getBytes("UTF-8"));
+		b.write("Sm\u00f6rg\u00e5sbord\n".getBytes("UTF-8"));
+		b.write("\n".getBytes("UTF-8"));
+		b.write("\u304d\u308c\u3044\n".getBytes("UTF-8"));
+		final RevCommit c;
+		c = new RevCommit(id("9473095c4cb2f12aefe1db8a355fe3fafba42f67")); // bogus id
+		c.parseCanonical(new RevWalk(db), b.toByteArray());
+
+		assertEquals("F\u00f6r fattare", c.getAuthorIdent().getName());
+		assertEquals("Sm\u00f6rg\u00e5sbord", c.getShortMessage());
+		assertEquals("Sm\u00f6rg\u00e5sbord\n\n\u304d\u308c\u3044\n", c.getFullMessage());
+	}
+
+	public void testParse_implicit_mixed_encoded() throws Exception {
+		final ByteArrayOutputStream b = new ByteArrayOutputStream();
+		b.write("tree 9788669ad918b6fcce64af8882fc9a81cb6aba67\n".getBytes("UTF-8"));
+		b.write("author F\u00f6r fattare <a_u_thor@example.com> 1218123387 +0700\n".getBytes("ISO-8859-1"));
+		b.write("committer C O. Miter <c@example.com> 1218123390 -0500\n".getBytes("UTF-8"));
+		b.write("\n".getBytes("UTF-8"));
+		b.write("Sm\u00f6rg\u00e5sbord\n".getBytes("UTF-8"));
+		b.write("\n".getBytes("UTF-8"));
+		b.write("\u304d\u308c\u3044\n".getBytes("UTF-8"));
+		final RevCommit c;
+		c = new RevCommit(id("9473095c4cb2f12aefe1db8a355fe3fafba42f67")); // bogus id
+		c.parseCanonical(new RevWalk(db), b.toByteArray());
+
+		assertEquals("F\u00f6r fattare", c.getAuthorIdent().getName());
+		assertEquals("Sm\u00f6rg\u00e5sbord", c.getShortMessage());
+		assertEquals("Sm\u00f6rg\u00e5sbord\n\n\u304d\u308c\u3044\n", c.getFullMessage());
+	}
+
+	/**
+	 * Test parsing of a commit whose encoding is given and works.
+	 *
+	 * @throws Exception
+	 */
+	public void testParse_explicit_encoded() throws Exception {
+		final ByteArrayOutputStream b = new ByteArrayOutputStream();
+		b.write("tree 9788669ad918b6fcce64af8882fc9a81cb6aba67\n".getBytes("EUC-JP"));
+		b.write("author F\u00f6r fattare <a_u_thor@example.com> 1218123387 +0700\n".getBytes("EUC-JP"));
+		b.write("committer C O. Miter <c@example.com> 1218123390 -0500\n".getBytes("EUC-JP"));
+		b.write("encoding euc_JP\n".getBytes("EUC-JP"));
+		b.write("\n".getBytes("EUC-JP"));
+		b.write("\u304d\u308c\u3044\n".getBytes("EUC-JP"));
+		b.write("\n".getBytes("EUC-JP"));
+		b.write("Hi\n".getBytes("EUC-JP"));
+		final RevCommit c;
+		c = new RevCommit(id("9473095c4cb2f12aefe1db8a355fe3fafba42f67")); // bogus id
+		c.parseCanonical(new RevWalk(db), b.toByteArray());
+
+		assertEquals("F\u00f6r fattare", c.getAuthorIdent().getName());
+		assertEquals("\u304d\u308c\u3044", c.getShortMessage());
+		assertEquals("\u304d\u308c\u3044\n\nHi\n", c.getFullMessage());
+	}
+
+	/**
+	 * This is a twisted case, but show what we expect here. We can revise the
+	 * expectations provided this case is updated.
+	 *
+	 * What happens here is that an encoding us given, but data is not encoded
+	 * that way (and we can detect it), so we try other encodings.
+	 *
+	 * @throws Exception
+	 */
+	public void testParse_explicit_bad_encoded() throws Exception {
+		final ByteArrayOutputStream b = new ByteArrayOutputStream();
+		b.write("tree 9788669ad918b6fcce64af8882fc9a81cb6aba67\n".getBytes("UTF-8"));
+		b.write("author F\u00f6r fattare <a_u_thor@example.com> 1218123387 +0700\n".getBytes("ISO-8859-1"));
+		b.write("committer C O. Miter <c@example.com> 1218123390 -0500\n".getBytes("UTF-8"));
+		b.write("encoding EUC-JP\n".getBytes("UTF-8"));
+		b.write("\n".getBytes("UTF-8"));
+		b.write("\u304d\u308c\u3044\n".getBytes("UTF-8"));
+		b.write("\n".getBytes("UTF-8"));
+		b.write("Hi\n".getBytes("UTF-8"));
+		final RevCommit c;
+		c = new RevCommit(id("9473095c4cb2f12aefe1db8a355fe3fafba42f67")); // bogus id
+		c.parseCanonical(new RevWalk(db), b.toByteArray());
+
+		assertEquals("F\u00f6r fattare", c.getAuthorIdent().getName());
+		assertEquals("\u304d\u308c\u3044", c.getShortMessage());
+		assertEquals("\u304d\u308c\u3044\n\nHi\n", c.getFullMessage());
+	}
+
+	/**
+	 * This is a twisted case too, but show what we expect here. We can revise the
+	 * expectations provided this case is updated.
+	 *
+	 * What happens here is that an encoding us given, but data is not encoded
+	 * that way (and we can detect it), so we try other encodings. Here data could
+	 * actually be decoded in the stated encoding, but we overide using UTF-8.
+	 *
+	 * @throws Exception
+	 */
+	public void testParse_explicit_bad_encoded2() throws Exception {
+		final ByteArrayOutputStream b = new ByteArrayOutputStream();
+		b.write("tree 9788669ad918b6fcce64af8882fc9a81cb6aba67\n".getBytes("UTF-8"));
+		b.write("author F\u00f6r fattare <a_u_thor@example.com> 1218123387 +0700\n".getBytes("UTF-8"));
+		b.write("committer C O. Miter <c@example.com> 1218123390 -0500\n".getBytes("UTF-8"));
+		b.write("encoding ISO-8859-1\n".getBytes("UTF-8"));
+		b.write("\n".getBytes("UTF-8"));
+		b.write("\u304d\u308c\u3044\n".getBytes("UTF-8"));
+		b.write("\n".getBytes("UTF-8"));
+		b.write("Hi\n".getBytes("UTF-8"));
+		final RevCommit c;
+		c = new RevCommit(id("9473095c4cb2f12aefe1db8a355fe3fafba42f67")); // bogus id
+		c.parseCanonical(new RevWalk(db), b.toByteArray());
+
+		assertEquals("F\u00f6r fattare", c.getAuthorIdent().getName());
+		assertEquals("\u304d\u308c\u3044", c.getShortMessage());
+		assertEquals("\u304d\u308c\u3044\n\nHi\n", c.getFullMessage());
+	}
+
 	public void testParse_NoMessage() throws Exception {
 		final String msg = "";
 		final RevCommit c = create(msg);
diff --git a/org.spearce.jgit/src/org/spearce/jgit/util/RawParseUtils.java b/org.spearce.jgit/src/org/spearce/jgit/util/RawParseUtils.java
index a31734b..7c16394 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/util/RawParseUtils.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/util/RawParseUtils.java
@@ -42,7 +42,10 @@
 import static org.spearce.jgit.lib.ObjectChecker.encoding;
 
 import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
 import java.util.Arrays;
 
 import org.spearce.jgit.lib.Constants;
@@ -376,7 +379,10 @@ public static PersonIdent parsePersonIdent(final byte[] raw, final int nameB) {
 	}
 
 	/**
-	 * Decode a region of the buffer under the specified character set.
+	 * Decode a region of the buffer under the specified character set if possible.
+	 *
+	 * If the byte stream cannot be decoded that way, the platform default is tried
+	 * and if that too fails, the fail-safe ISO-8859-1 encoding is tried.
 	 * 
 	 * @param cs
 	 *            character set to use when decoding the buffer.
@@ -393,12 +399,63 @@ public static PersonIdent parsePersonIdent(final byte[] raw, final int nameB) {
 	public static String decode(final Charset cs, final byte[] buffer,
 			final int start, final int end) {
 		final ByteBuffer b = ByteBuffer.wrap(buffer, start, end - start);
-		return cs.decode(b).toString();
+		b.mark();
+		for (int i = 0;; ++i) {
+			try {
+				Charset charset;
+				switch (i) {
+				case 0:
+					/*
+					 * Try our built-in favorite. The assumption here is that
+					 * decoding will fail if the data is not actually encoded
+					 * using that encoder.
+					 */
+					charset = Constants.CHARSET;
+					break;
+				case 1:
+					/*
+					 * Try the suggested encoding, it might be right since it
+					 * was provided
+					 */
+					if (cs.equals(Constants.CHARSET))
+						continue;
+					b.reset();
+					charset = cs;
+					break;
+				case 2:
+					/*
+					 * Try the default character set. A small group of people
+					 * might actually use the same (or very similar) locale.
+					 */
+					charset = Charset.defaultCharset();
+					if (charset.equals(Constants.CHARSET))
+						continue;
+					if (charset.equals(cs))
+						continue;
+					b.reset();
+					break;
+				default:
+					/*
+					 * This one is to make sure we do no fail. Data may look
+					 * funny but there is nothing we can do here withou much
+					 * more advanced guessing.
+					 */
+					b.reset();
+					charset = Charset.forName("ISO-8859-1");
+				}
+				CharsetDecoder d = charset.newDecoder();
+				d.onMalformedInput(CodingErrorAction.REPORT);
+				d.onUnmappableCharacter(CodingErrorAction.REPORT);
+				return d.decode(b).toString();
+			} catch (CharacterCodingException e1) {
+				continue;
+			}
+		}
 	}
 
 	/**
 	 * Locate the position of the commit message body.
-	 * 
+	 *
 	 * @param b
 	 *            buffer to scan.
 	 * @param ptr

-- 
1.6.0.2.308.gef4a
Previous: Robin RosenbergNext: Shawn O. Pearce
Message 5 of 8 in “Decodings”
  1. 0/4 DecodingsRobin Rosenberg, Oct 12, 2008
  2. 1/4 log command: Use explicit US locale for datesRobin Rosenberg, Oct 12, 2008
  3. 2/4 jgit programs: Use i18n.logOutputEncoding or user's locale for outputRobin Rosenberg, Oct 12, 2008
  4. 3/4 The git config file is case insensitiveRobin Rosenberg, Oct 12, 2008
  5. 4/4 Intelligent parsing of ambiguously encoded meta data.Robin Rosenberg, Oct 12, 2008
  6. Shawn O. PearceOct 13, 2008
  7. Robin RosenbergOct 13, 2008
  8. Shawn O. PearceOct 13, 2008

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.