git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[JGIT PATCH 20/21] Add client side encryption to Amazon S3 client library

From
Shawn O. Pearce <spearce@spearce.org>
Date
Jun 29, 2008, 07:59 UTC
Message-ID
<1214726371-93520-21-git-send-email-spearce@spearce.org>
In-Reply-To
<1214726371-93520-20-git-send-email-spearce@spearce.org>

By encrypting (and decrypting) all data on the client side we are able to safely hide the content of our repository from the owners/operators the Amazon S3 service, making it a secure backup solution for Git repositories.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/transport/AmazonS3.java   |   82 ++++++++-
 .../org/spearce/jgit/transport/WalkEncryption.java |  188 ++++++++++++++++++++
 2 files changed, 266 insertions(+), 4 deletions(-)
 create mode 100644 org.spearce.jgit/src/org/spearce/jgit/transport/WalkEncryption.java
diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/AmazonS3.java b/org.spearce.jgit/src/org/spearce/jgit/transport/AmazonS3.java
index 466d9e9..4c82967 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/transport/AmazonS3.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/transport/AmazonS3.java
@@ -37,6 +37,7 @@
 
 package org.spearce.jgit.transport;
 
+import java.io.ByteArrayOutputStream;
 import java.io.EOFException;
 import java.io.File;
 import java.io.FileInputStream;
@@ -53,6 +54,7 @@ import java.security.DigestOutputStream;
 import java.security.InvalidKeyException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.security.spec.InvalidKeySpecException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -93,6 +95,11 @@ import org.xml.sax.helpers.XMLReaderFactory;
  * <p>
  * Authentication is always performed using the user's AWSAccessKeyId and their
  * private AWSSecretAccessKey.
+ * <p>
+ * Optional client-side encryption may be enabled if requested. The format is
+ * compatible with <a href="http://jets3t.s3.amazonaws.com/index.html">jets3t</a>,
+ * a popular Java based Amazon S3 client library. Enabling encryption can hide
+ * sensitive data from the operators of the S3 service.
  */
 public class AmazonS3 {
 	private static final Set<String> SIGNED_HEADERS;
@@ -103,6 +110,8 @@ public class AmazonS3 {
 
 	private static final String X_AMZ_ACL = "x-amz-acl";
 
+	private static final String X_AMZ_META = "x-amz-meta-";
+
 	static {
 		SIGNED_HEADERS = new HashSet<String>();
 		SIGNED_HEADERS.add("content-type");
@@ -161,6 +170,9 @@ public class AmazonS3 {
 	/** Maximum number of times to try an operation. */
 	private final int maxAttempts;
 
+	/** Encryption algorithm, may be a null instance that provides pass-through. */
+	private final WalkEncryption encryption;
+
 	/**
 	 * Create a new S3 client for the supplied user information.
 	 * <p>
@@ -179,6 +191,10 @@ public class AmazonS3 {
 	 * 
 	 * # Number of times to retry after internal error from S3.
 	 * httpclient.retry-max: 3
+	 * 
+	 * # End-to-end encryption (hides content from S3 owners)
+	 * password: &lt;encryption pass-phrase&gt;
+	 * crypto.algorithm: PBEWithMD5AndDES
 	 * </pre>
 	 * 
 	 * @param props
@@ -207,6 +223,22 @@ public class AmazonS3 {
 		else
 			throw new IllegalArgumentException("Invalid acl: " + pacl);
 
+		try {
+			final String cPas = props.getProperty("password");
+			if (cPas != null) {
+				String cAlg = props.getProperty("crypto.algorithm");
+				if (cAlg == null)
+					cAlg = "PBEWithMD5AndDES";
+				encryption = new WalkEncryption.ObjectEncryptionV2(cAlg, cPas);
+			} else {
+				encryption = WalkEncryption.NONE;
+			}
+		} catch (InvalidKeySpecException e) {
+			throw new IllegalArgumentException("Invalid encryption", e);
+		} catch (NoSuchAlgorithmException e) {
+			throw new IllegalArgumentException("Invalid encryption", e);
+		}
+
 		maxAttempts = Integer.parseInt(props.getProperty(
 				"httpclient.retry-max", "3"));
 		proxySelector = ProxySelector.getDefault();
@@ -232,6 +264,7 @@ public class AmazonS3 {
 			authorize(c);
 			switch (HttpSupport.response(c)) {
 			case HttpURLConnection.HTTP_OK:
+				encryption.validate(c, X_AMZ_META);
 				return c;
 			case HttpURLConnection.HTTP_NOT_FOUND:
 				throw new FileNotFoundException(key);
@@ -245,6 +278,19 @@ public class AmazonS3 {
 	}
 
 	/**
+	 * Decrypt an input stream from {@link #get(String, String)}.
+	 * 
+	 * @param u
+	 *            connection previously created by {@link #get(String, String)}}.
+	 * @return stream to read plain text from.
+	 * @throws IOException
+	 *             decryption could not be configured.
+	 */
+	public InputStream decrypt(final URLConnection u) throws IOException {
+		return encryption.decrypt(u.getInputStream());
+	}
+
+	/**
 	 * List the names of keys available within a bucket.
 	 * <p>
 	 * This method is primarily meant for obtaining a "recursive directory
@@ -326,6 +372,16 @@ public class AmazonS3 {
 	 */
 	public void put(final String bucket, final String key, final byte[] data)
 			throws IOException {
+		if (encryption != WalkEncryption.NONE) {
+			// We have to copy to produce the cipher text anyway so use
+			// the large object code path as it supports that behavior.
+			//
+			final OutputStream os = beginPut(bucket, key);
+			os.write(data);
+			os.close();
+			return;
+		}
+
 		final String md5str = Base64.encodeBytes(newMD5().digest(data));
 		final String lenstr = String.valueOf(data.length);
 		for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
@@ -375,8 +431,11 @@ public class AmazonS3 {
 	 * @param key
 	 *            key of the object within its bucket.
 	 * @return a stream which accepts the new data, and transmits once closed.
+	 * @throws IOException
+	 *             if encryption was enabled it could not be configured.
 	 */
-	public OutputStream beginPut(final String bucket, final String key) {
+	public OutputStream beginPut(final String bucket, final String key)
+			throws IOException {
 		final MessageDigest md5 = newMD5();
 		final TemporaryBuffer buffer = new TemporaryBuffer() {
 			@Override
@@ -389,7 +448,7 @@ public class AmazonS3 {
 				}
 			}
 		};
-		return new DigestOutputStream(buffer, md5);
+		return encryption.encrypt(new DigestOutputStream(buffer, md5));
 	}
 
 	private void putImpl(final String bucket, final String key,
@@ -402,6 +461,7 @@ public class AmazonS3 {
 			c.setRequestProperty("Content-Length", lenstr);
 			c.setRequestProperty("Content-MD5", md5str);
 			c.setRequestProperty(X_AMZ_ACL, acl);
+			encryption.request(c, X_AMZ_META);
 			authorize(c);
 			c.setDoOutput(true);
 			c.setFixedLengthStreamingMode((int) len);
@@ -426,8 +486,22 @@ public class AmazonS3 {
 
 	private IOException error(final String action, final String key,
 			final HttpURLConnection c) throws IOException {
-		return new IOException(action + " of '" + key + "' failed: "
-				+ HttpSupport.response(c) + " " + c.getResponseMessage());
+		final IOException err = new IOException(action + " of '" + key
+				+ "' failed: " + HttpSupport.response(c) + " "
+				+ c.getResponseMessage());
+		final ByteArrayOutputStream b = new ByteArrayOutputStream();
+		byte[] buf = new byte[2048];
+		for (;;) {
+			final int n = c.getErrorStream().read(buf);
+			if (n < 0)
+				break;
+			if (n > 0)
+				b.write(buf, 0, n);
+		}
+		buf = b.toByteArray();
+		if (buf.length > 0)
+			err.initCause(new IOException("\n" + new String(buf)));
+		return err;
 	}
 
 	private IOException maxAttempts(final String action, final String key) {
diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/WalkEncryption.java b/org.spearce.jgit/src/org/spearce/jgit/transport/WalkEncryption.java
new file mode 100644
index 0000000..cec6d75
--- /dev/null
+++ b/org.spearce.jgit/src/org/spearce/jgit/transport/WalkEncryption.java
@@ -0,0 +1,188 @@
+/*
+ * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Git Development Community nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.spearce.jgit.transport;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.security.spec.InvalidKeySpecException;
+
+import javax.crypto.Cipher;
+import javax.crypto.CipherInputStream;
+import javax.crypto.CipherOutputStream;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.PBEParameterSpec;
+
+abstract class WalkEncryption {
+	static final WalkEncryption NONE = new NoEncryption();
+
+	static final String JETS3T_CRYPTO_VER = "jets3t-crypto-ver";
+
+	static final String JETS3T_CRYPTO_ALG = "jets3t-crypto-alg";
+
+	abstract OutputStream encrypt(OutputStream os) throws IOException;
+
+	abstract InputStream decrypt(InputStream in) throws IOException;
+
+	abstract void request(HttpURLConnection u, String prefix);
+
+	abstract void validate(HttpURLConnection u, String p) throws IOException;
+
+	protected void validateImpl(final HttpURLConnection u, final String p,
+			final String version, final String name) throws IOException {
+		String v;
+
+		v = u.getHeaderField(p + JETS3T_CRYPTO_VER);
+		if (v == null)
+			v = "";
+		if (!version.equals(v))
+			throw new IOException("Unsupported encryption version: " + v);
+
+		v = u.getHeaderField(p + JETS3T_CRYPTO_ALG);
+		if (v == null)
+			v = "";
+		if (!name.equals(v))
+			throw new IOException("Unsupported encryption algorithm: " + v);
+	}
+
+	IOException error(final Throwable why) {
+		final IOException e;
+		e = new IOException("Encryption error: " + why.getMessage());
+		e.initCause(why);
+		return e;
+	}
+
+	private static class NoEncryption extends WalkEncryption {
+		@Override
+		void request(HttpURLConnection u, String prefix) {
+			// Don't store any request properties.
+		}
+
+		@Override
+		void validate(final HttpURLConnection u, final String p)
+				throws IOException {
+			validateImpl(u, p, "", "");
+		}
+
+		@Override
+		InputStream decrypt(InputStream in) {
+			return in;
+		}
+
+		@Override
+		OutputStream encrypt(OutputStream os) {
+			return os;
+		}
+	}
+
+	static class ObjectEncryptionV2 extends WalkEncryption {
+		private static int ITERATION_COUNT = 5000;
+
+		private static byte[] salt = { (byte) 0xA4, (byte) 0x0B, (byte) 0xC8,
+				(byte) 0x34, (byte) 0xD6, (byte) 0x95, (byte) 0xF3, (byte) 0x13 };
+
+		private final String algorithmName;
+
+		private final SecretKey skey;
+
+		private final PBEParameterSpec aspec;
+
+		ObjectEncryptionV2(final String algo, final String key)
+				throws InvalidKeySpecException, NoSuchAlgorithmException {
+			algorithmName = algo;
+
+			final PBEKeySpec s;
+			s = new PBEKeySpec(key.toCharArray(), salt, ITERATION_COUNT, 32);
+			skey = SecretKeyFactory.getInstance(algo).generateSecret(s);
+			aspec = new PBEParameterSpec(salt, ITERATION_COUNT);
+		}
+
+		@Override
+		void request(final HttpURLConnection u, final String prefix) {
+			u.setRequestProperty(prefix + JETS3T_CRYPTO_VER, "2");
+			u.setRequestProperty(prefix + JETS3T_CRYPTO_ALG, algorithmName);
+		}
+
+		@Override
+		void validate(final HttpURLConnection u, final String p)
+				throws IOException {
+			validateImpl(u, p, "2", algorithmName);
+		}
+
+		@Override
+		OutputStream encrypt(final OutputStream os) throws IOException {
+			try {
+				final Cipher c = Cipher.getInstance(algorithmName);
+				c.init(Cipher.ENCRYPT_MODE, skey, aspec);
+				return new CipherOutputStream(os, c);
+			} catch (NoSuchAlgorithmException e) {
+				throw error(e);
+			} catch (NoSuchPaddingException e) {
+				throw error(e);
+			} catch (InvalidKeyException e) {
+				throw error(e);
+			} catch (InvalidAlgorithmParameterException e) {
+				throw error(e);
+			}
+		}
+
+		@Override
+		InputStream decrypt(final InputStream in) throws IOException {
+			try {
+				final Cipher c = Cipher.getInstance(algorithmName);
+				c.init(Cipher.DECRYPT_MODE, skey, aspec);
+				return new CipherInputStream(in, c);
+			} catch (NoSuchAlgorithmException e) {
+				throw error(e);
+			} catch (NoSuchPaddingException e) {
+				throw error(e);
+			} catch (InvalidKeyException e) {
+				throw error(e);
+			} catch (InvalidAlgorithmParameterException e) {
+				throw error(e);
+			}
+		}
+	}
+}

-- 
1.5.6.74.g8a5e
Previous: Shawn O. PearceNext: Shawn O. Pearce
Message 21 of 27 in “Push support over SFTP and (encrypted) Amazon S3”
  1. 00/21 Push support over SFTP and (encrypted) Amazon S3Shawn O. Pearce, Jun 29, 2008
  2. 01/21 Remove unused index files when WalkFetchConnection closesShawn O. Pearce, Jun 29, 2008
  3. 02/21 Do not show URIish passwords in TransportExceptionsShawn O. Pearce, Jun 29, 2008
  4. 03/21 Use PackedObjectInfo as a base class for PackWriter's ObjectToPackShawn O. Pearce, Jun 29, 2008
  5. 04/21 Refactor PackWriter to hold onto the sorted object listShawn O. Pearce, Jun 29, 2008
  6. 05/21 Save the pack checksum after computing it in PackWriterShawn O. Pearce, Jun 29, 2008
  7. 06/21 Allow PackIndexWriter to use any subclass of PackedObjectInfoShawn O. Pearce, Jun 29, 2008
  8. 07/21 Allow PackWriter to create a corresponding index fileShawn O. Pearce, Jun 29, 2008
  9. 08/21 Allow PackWriter to prepare object list and compute name before writingShawn O. Pearce, Jun 29, 2008
  10. 09/21 Remember how a Ref was read in from disk and createdShawn O. Pearce, Jun 29, 2008
  11. 10/21 Simplify walker transport ref advertisement setupShawn O. Pearce, Jun 29, 2008
  12. 11/21 Indicate the protocol jgit doesn't support push overShawn O. Pearce, Jun 29, 2008
  13. 12/21 WalkTransport must allow subclasses to implement openPushShawn O. Pearce, Jun 29, 2008
  14. 13/21 Support push over the sftp:// dumb transportShawn O. Pearce, Jun 29, 2008
  15. 14/21 Extract readPackedRefs from TransportSftp for reuseShawn O. Pearce, Jun 29, 2008
  16. 15/21 Specialized byte array output stream for large filesShawn O. Pearce, Jun 29, 2008
  17. 16/21 Add Robert Harder's public domain Base64 encoding utilityShawn O. Pearce, Jun 29, 2008
  18. 17/21 Misc. documentation fixes to Base64 utilityShawn O. Pearce, Jun 29, 2008
  19. 18/21 Extract the basic HTTP proxy support to its own classShawn O. Pearce, Jun 29, 2008
  20. 19/21 Create a really simple Amazon S3 REST clientShawn O. Pearce, Jun 29, 2008
  21. 20/21 Add client side encryption to Amazon S3 client libraryShawn O. Pearce, Jun 29, 2008
  22. 21/21 Bidirectional protocol support for Amazon S3Shawn O. Pearce, Jun 29, 2008
  23. Robin RosenbergJun 29, 2008
  24. Shawn O. PearceJun 29, 2008
  25. Robin RosenbergJun 29, 2008
  26. Johannes SchindelinJun 29, 2008
  27. Shawn O. PearceJun 29, 2008

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.