[JGIT PATCH 21/21] Bidirectional protocol support for Amazon S3

["Shawn O. Pearce" <spearce@spearce.org>]

The new "amazon-s3://" transport provides bi-directional communication
for Git repositories to the S3 service.  This may be useful for backup
of private data which users do not want published to the world.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/transport/Transport.java  |    3 +
 .../spearce/jgit/transport/TransportAmazonS3.java  |  319 ++++++++++++++++++++
 .../src/org/spearce/jgit/transport/URIish.java     |    2 +-
 3 files changed, 323 insertions(+), 1 deletions(-)
 create mode 100644 org.spearce.jgit/src/org/spearce/jgit/transport/TransportAmazonS3.java

diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/Transport.java b/org.spearce.jgit/src/org/spearce/jgit/transport/Transport.java
index 5376a9e..b962162 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/transport/Transport.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/transport/Transport.java
@@ -140,6 +140,9 @@ public abstract class Transport {
 		else if (TransportGitAnon.canHandle(remote))
 			return new TransportGitAnon(local, remote);
 
+		else if (TransportAmazonS3.canHandle(remote))
+			return new TransportAmazonS3(local, remote);
+
 		else if (TransportBundle.canHandle(remote))
 			return new TransportBundle(local, remote);
 
diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/TransportAmazonS3.java b/org.spearce.jgit/src/org/spearce/jgit/transport/TransportAmazonS3.java
new file mode 100644
index 0000000..ceb6848
--- /dev/null
+++ b/org.spearce.jgit/src/org/spearce/jgit/transport/TransportAmazonS3.java
@@ -0,0 +1,319 @@
+/*
+ * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Git Development Community nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.spearce.jgit.transport;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URLConnection;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+import org.spearce.jgit.errors.NotSupportedException;
+import org.spearce.jgit.errors.TransportException;
+import org.spearce.jgit.lib.ObjectId;
+import org.spearce.jgit.lib.Ref;
+import org.spearce.jgit.lib.Repository;
+import org.spearce.jgit.lib.Ref.Storage;
+import org.spearce.jgit.util.FS;
+
+/**
+ * Transport over the non-Git aware Amazon S3 protocol.
+ * <p>
+ * This transport communicates with the Amazon S3 servers (a non-free commercial
+ * hosting service that users must subscribe to). Some users may find transport
+ * to and from S3 to be a useful backup service.
+ * <p>
+ * The transport does not require any specialized Git support on the remote
+ * (server side) repository, as Amazon does not provide any such support.
+ * Repository files are retrieved directly through the S3 API, which uses
+ * extended HTTP/1.1 semantics. This make it possible to read or write Git data
+ * from a remote repository that is stored on S3.
+ * <p>
+ * Unlike the HTTP variant (see {@link TransportHttp}) we rely upon being able
+ * to list objects in a bucket, as the S3 API supports this function. By listing
+ * the bucket contents we can avoid relying on <code>objects/info/packs</code>
+ * or <code>info/refs</code> in the remote repository.
+ * <p>
+ * Concurrent pushing over this transport is not supported. Multiple concurrent
+ * push operations may cause confusion in the repository state.
+ * 
+ * @see WalkFetchConnection
+ * @see WalkPushConnection
+ */
+class TransportAmazonS3 extends WalkTransport {
+	static final String S3_SCHEME = "amazon-s3";
+
+	static boolean canHandle(final URIish uri) {
+		if (!uri.isRemote())
+			return false;
+		return S3_SCHEME.equals(uri.getScheme());
+	}
+
+	/** User information necessary to connect to S3. */
+	private final AmazonS3 s3;
+
+	/** Bucket the remote repository is stored in. */
+	private final String bucket;
+
+	/**
+	 * Key prefix which all objects related to the repository start with.
+	 * <p>
+	 * The prefix does not start with "/".
+	 * <p>
+	 * The prefix does not end with "/". The trailing slash is stripped during
+	 * the constructor if a trailing slash was supplied in the URIish.
+	 * <p>
+	 * All files within the remote repository start with
+	 * <code>keyPrefix + "/"</code>.
+	 */
+	private final String keyPrefix;
+
+	TransportAmazonS3(final Repository local, final URIish uri)
+			throws NotSupportedException {
+		super(local, uri);
+
+		Properties props = null;
+		File propsFile = new File(local.getDirectory(), uri.getUser());
+		if (!propsFile.isFile())
+			propsFile = new File(FS.userHome(), uri.getUser());
+		if (propsFile.isFile()) {
+			try {
+				props = AmazonS3.properties(propsFile);
+			} catch (IOException e) {
+				throw new NotSupportedException("cannot read " + propsFile, e);
+			}
+		} else {
+			props = new Properties();
+			props.setProperty("accesskey", uri.getUser());
+			props.setProperty("secretkey", uri.getPass());
+		}
+
+		s3 = new AmazonS3(props);
+		bucket = uri.getHost();
+
+		String p = uri.getPath();
+		if (p.startsWith("/"))
+			p = p.substring(1);
+		if (p.endsWith("/"))
+			p = p.substring(0, p.length() - 1);
+		keyPrefix = p;
+	}
+
+	@Override
+	public FetchConnection openFetch() throws TransportException {
+		final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects");
+		final WalkFetchConnection r = new WalkFetchConnection(this, c);
+		r.available(c.readAdvertisedRefs());
+		return r;
+	}
+
+	@Override
+	public PushConnection openPush() throws TransportException {
+		final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects");
+		final WalkPushConnection r = new WalkPushConnection(this, c);
+		r.available(c.readAdvertisedRefs());
+		return r;
+	}
+
+	class DatabaseS3 extends WalkRemoteObjectDatabase {
+		private final String bucketName;
+
+		private final String objectsKey;
+
+		DatabaseS3(final String b, final String o) {
+			bucketName = b;
+			objectsKey = o;
+		}
+
+		private String resolveKey(String subpath) {
+			if (subpath.endsWith("/"))
+				subpath = subpath.substring(0, subpath.length() - 1);
+			String k = objectsKey;
+			while (subpath.startsWith("../")) {
+				k = k.substring(0, k.lastIndexOf('/'));
+				subpath = subpath.substring(3);
+			}
+			return k + "/" + subpath;
+		}
+
+		@Override
+		URIish getURI() {
+			URIish u = new URIish();
+			u = u.setScheme(S3_SCHEME);
+			u = u.setHost(bucketName);
+			u = u.setPath("/" + objectsKey);
+			return u;
+		}
+
+		@Override
+		Collection<WalkRemoteObjectDatabase> getAlternates() throws IOException {
+			try {
+				return readAlternates(INFO_ALTERNATES);
+			} catch (FileNotFoundException err) {
+				// Fall through.
+			}
+			return null;
+		}
+
+		@Override
+		WalkRemoteObjectDatabase openAlternate(final String location)
+				throws IOException {
+			return new DatabaseS3(bucketName, resolveKey(location));
+		}
+
+		@Override
+		Collection<String> getPackNames() throws IOException {
+			final HashSet<String> have = new HashSet<String>();
+			have.addAll(s3.list(bucket, resolveKey("pack")));
+
+			final Collection<String> packs = new ArrayList<String>();
+			for (final String n : have) {
+				if (!n.startsWith("pack-") || !n.endsWith(".pack"))
+					continue;
+
+				final String in = n.substring(0, n.length() - 5) + ".idx";
+				if (have.contains(in))
+					packs.add(n);
+			}
+			return packs;
+		}
+
+		@Override
+		FileStream open(final String path) throws IOException {
+			final URLConnection c = s3.get(bucket, resolveKey(path));
+			final InputStream raw = c.getInputStream();
+			final InputStream in = s3.decrypt(c);
+			final int len = c.getContentLength();
+			return new FileStream(in, raw == in ? len : -1);
+		}
+
+		@Override
+		void deleteFile(final String path) throws IOException {
+			s3.delete(bucket, resolveKey(path));
+		}
+
+		@Override
+		OutputStream writeFile(final String path) throws IOException {
+			return s3.beginPut(bucket, resolveKey(path));
+		}
+
+		@Override
+		void writeFile(final String path, final byte[] data) throws IOException {
+			s3.put(bucket, resolveKey(path), data);
+		}
+
+		Map<String, Ref> readAdvertisedRefs() throws TransportException {
+			final TreeMap<String, Ref> avail = new TreeMap<String, Ref>();
+			readPackedRefs(avail);
+			readLooseRefs(avail);
+			readRef(avail, "HEAD");
+			return avail;
+		}
+
+		private void readLooseRefs(final TreeMap<String, Ref> avail)
+				throws TransportException {
+			try {
+				for (final String n : s3.list(bucket, resolveKey("../refs")))
+					readRef(avail, "refs/" + n);
+			} catch (IOException e) {
+				throw new TransportException(getURI(), "cannot list refs", e);
+			}
+		}
+
+		private Ref readRef(final TreeMap<String, Ref> avail, final String rn)
+				throws TransportException {
+			final String s;
+			try {
+				final BufferedReader br = openReader("../" + rn);
+				try {
+					s = br.readLine();
+				} finally {
+					br.close();
+				}
+			} catch (FileNotFoundException noRef) {
+				return null;
+			} catch (IOException err) {
+				throw new TransportException(getURI(), "read ../" + rn, err);
+			}
+
+			if (s == null)
+				throw new TransportException(getURI(), "Empty ref: " + rn);
+
+			if (s.startsWith("ref: ")) {
+				final String target = s.substring("ref: ".length());
+				Ref r = avail.get(target);
+				if (r == null)
+					r = readRef(avail, target);
+				if (r == null)
+					return null;
+				r = new Ref(r.getStorage(), rn, r.getObjectId(), r
+						.getPeeledObjectId());
+				avail.put(r.getName(), r);
+				return r;
+			}
+
+			if (ObjectId.isId(s)) {
+				final Ref r = new Ref(loose(avail.get(rn)), rn, ObjectId
+						.fromString(s));
+				avail.put(r.getName(), r);
+				return r;
+			}
+
+			throw new TransportException(getURI(), "Bad ref: " + rn + ": " + s);
+		}
+
+		private Storage loose(final Ref r) {
+			if (r != null && r.getStorage() == Storage.PACKED)
+				return Storage.LOOSE_PACKED;
+			return Storage.LOOSE;
+		}
+
+		@Override
+		void close() {
+			// We do not maintain persistent connections.
+		}
+	}
+}
diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/URIish.java b/org.spearce.jgit/src/org/spearce/jgit/transport/URIish.java
index 9e7ca83..8aa5d35 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/transport/URIish.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/transport/URIish.java
@@ -51,7 +51,7 @@ import java.util.regex.Pattern;
  */
 public class URIish {
 	private static final Pattern FULL_URI = Pattern
-			.compile("^(?:([a-z+]+)://(?:([^/]+?)(?::([^/]+?))?@)?(?:([^/]+?))?(?::(\\d+))?)?((?:[A-Za-z]:)?/.+)$");
+			.compile("^(?:([a-z0-9+-]+)://(?:([^/]+?)(?::([^/]+?))?@)?(?:([^/]+?))?(?::(\\d+))?)?((?:[A-Za-z]:)?/.+)$");
 
 	private static final Pattern SCP_URI = Pattern
 			.compile("^(?:([^@]+?)@)?([^:]+?):(.+)$");
-- 
1.5.6.74.g8a5e