[JGIT PATCH] 1/2: Externalizable items

[JGIT PATCH] 1/2: Externalizable items

[Nigel Magnay]

Make parts of jgit externalizable, so that they can be marshalled over
the wire or onto disk,
using formats from git mailing list.

Signed-off-by: Nigel Magnay <nigel.magnay@gmail.com>
---
 .../src/org/spearce/jgit/lib/ObjectId.java         |   31 ++++-
 .../src/org/spearce/jgit/transport/RefSpec.java    |   77 +++++++----
 .../org/spearce/jgit/transport/RemoteConfig.java   |  142 +++++++++++++++++++-
 .../src/org/spearce/jgit/transport/URIish.java     |   31 ++++-
 4 files changed, 252 insertions(+), 29 deletions(-)

diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/ObjectId.java
b/org.spearce.jgit/src/org/spearce/jgit/lib/ObjectId.java
index 52ce0d4..1385325 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/ObjectId.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/ObjectId.java
@@ -38,6 +38,10 @@

 package org.spearce.jgit.lib;

+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
 import java.io.UnsupportedEncodingException;

 import org.spearce.jgit.util.NB;
@@ -45,7 +49,7 @@
 /**
  * A SHA-1 abstraction.
  */
-public class ObjectId extends AnyObjectId {
+public class ObjectId extends AnyObjectId implements Externalizable {
 	private static final ObjectId ZEROID;

 	private static final String ZEROID_STR;
@@ -56,6 +60,13 @@
 	}

 	/**
+	 * Empty constructor, for Externalizable.
+	 */
+	public ObjectId() {
+		// For Externalizable
+	}
+	
+	/**
 	 * Get the special all-null ObjectId.
 	 *
 	 * @return the all-null ObjectId, often used to stand-in for no object.
@@ -269,4 +280,22 @@ protected ObjectId(final AnyObjectId src) {
 	public ObjectId toObjectId() {
 		return this;
 	}
+
+	public void readExternal(ObjectInput in) throws IOException,
+			ClassNotFoundException {
+		byte[] sha1 = new byte[20];
+		in.read(sha1);
+		
+		w1 = NB.decodeInt32(sha1, 0);
+		w2 = NB.decodeInt32(sha1, 4);
+		w3 = NB.decodeInt32(sha1, 8);
+		w4 = NB.decodeInt32(sha1, 12);
+		w5 = NB.decodeInt32(sha1, 16);
+	}
+
+	public void writeExternal(ObjectOutput out) throws IOException {
+		byte[] sha1 = new byte[20];
+		copyRawTo(sha1, 0);
+		out.write(sha1);
+	}
 }
diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/RefSpec.java
b/org.spearce.jgit/src/org/spearce/jgit/transport/RefSpec.java
index 521110b..0ee89b0 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/transport/RefSpec.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/transport/RefSpec.java
@@ -37,6 +37,11 @@

 package org.spearce.jgit.transport;

+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+
 import org.spearce.jgit.lib.Constants;
 import org.spearce.jgit.lib.Ref;

@@ -46,7 +51,7 @@
  * A ref specification provides matching support and limited rules to rewrite a
  * reference in one repository to another reference in another repository.
  */
-public class RefSpec {
+public class RefSpec implements Externalizable {
 	/**
 	 * Suffix for wildcard ref spec component, that indicate matching all refs
 	 * with specified prefix.
@@ -109,30 +114,7 @@ public RefSpec() {
 	 *             the specification is invalid.
 	 */
 	public RefSpec(final String spec) {
-		String s = spec;
-		if (s.startsWith("+")) {
-			force = true;
-			s = s.substring(1);
-		}
-
-		final int c = s.indexOf(':');
-		if (c == 0) {
-			s = s.substring(1);
-			if (isWildcard(s))
-				throw new IllegalArgumentException("Invalid wildcards " + spec);
-			dstName = s;
-		} else if (c > 0) {
-			srcName = s.substring(0, c);
-			dstName = s.substring(c + 1);
-			if (isWildcard(srcName) && isWildcard(dstName))
-				wildcard = true;
-			else if (isWildcard(srcName) || isWildcard(dstName))
-				throw new IllegalArgumentException("Invalid wildcards " + spec);
-		} else {
-			if (isWildcard(s))
-				throw new IllegalArgumentException("Invalid wildcards " + spec);
-			srcName = s;
-		}
+		initializeFromString(spec);
 	}

 	/**
@@ -161,6 +143,42 @@ private RefSpec(final RefSpec p) {
 	}

 	/**
+	 * Initialize the ref specification from a string.
+	 *
+	 * @param spec
+	 *            string describing the specification.
+	 * @throws IllegalArgumentException
+	 *             the specification is invalid.
+	 */
+	private void initializeFromString(final String spec) {
+		srcName = null;
+		String s = spec;
+		if (s.startsWith("+")) {
+			force = true;
+			s = s.substring(1);
+		}
+
+		final int c = s.indexOf(':');
+		if (c == 0) {
+			s = s.substring(1);
+			if (isWildcard(s))
+				throw new IllegalArgumentException("Invalid wildcards " + spec);
+			dstName = s;
+		} else if (c > 0) {
+			srcName = s.substring(0, c);
+			dstName = s.substring(c + 1);
+			if (isWildcard(srcName) && isWildcard(dstName))
+				wildcard = true;
+			else if (isWildcard(srcName) || isWildcard(dstName))
+				throw new IllegalArgumentException("Invalid wildcards " + spec);
+		} else {
+			if (isWildcard(s))
+				throw new IllegalArgumentException("Invalid wildcards " + spec);
+			srcName = s;
+		}
+	}
+	
+	/**
 	 * Check if this specification wants to forcefully update the destination.
 	 *
 	 * @return true if this specification asks for updates without merge tests.
@@ -421,4 +439,13 @@ public String toString() {
 		}
 		return r.toString();
 	}
+
+	public void readExternal(ObjectInput in) throws IOException,
+			ClassNotFoundException {
+		initializeFromString(in.readUTF());	
+	}
+
+	public void writeExternal(ObjectOutput out) throws IOException {
+		out.writeUTF(toString());
+	}
 }
diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/RemoteConfig.java
b/org.spearce.jgit/src/org/spearce/jgit/transport/RemoteConfig.java
index 5bbf664..22443b4 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/transport/RemoteConfig.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/transport/RemoteConfig.java
@@ -38,10 +38,17 @@

 package org.spearce.jgit.transport;

+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;

 import org.spearce.jgit.lib.RepositoryConfig;

@@ -53,7 +60,7 @@
  * describing how refs should be transferred between this repository and the
  * remote repository.
  */
-public class RemoteConfig {
+public class RemoteConfig implements Externalizable {
 	private static final String SECTION = "remote";

 	private static final String KEY_URL = "url";
@@ -166,6 +173,17 @@ public RemoteConfig(final RepositoryConfig rc,
final String remoteName)
 	}

 	/**
+	 * Construct an empty remote config.
+	 */
+	public RemoteConfig() {
+		uris = new ArrayList<URIish>();
+		fetch = new ArrayList<RefSpec>();
+		push = new ArrayList<RefSpec>();
+		uploadpack = DEFAULT_UPLOAD_PACK;
+		receivepack = DEFAULT_RECEIVE_PACK;
+	}
+	
+	/**
 	 * Update this remote's definition within the configuration.
 	 *
 	 * @param rc
@@ -382,4 +400,126 @@ public TagOpt getTagOpt() {
 	public void setTagOpt(final TagOpt option) {
 		tagopt = option != null ? option : TagOpt.AUTO_FOLLOW;
 	}
+
+	private Map<String, Collection<String>> toMap() {
+		Map<String, Collection<String>> map = new HashMap<String,
Collection<String>>();
+		
+		if (uris.size() > 0) {
+			Collection<String> values = new ArrayList<String>();
+			for (URIish uri : uris) {
+				values.add(uri.toPrivateString());
+			}
+			map.put(KEY_URL, values);
+		}
+		
+		if (fetch.size() > 0) {
+			Collection<String> values = new ArrayList<String>();
+			for (RefSpec refspec : fetch) {
+				values.add(refspec.toString());
+			}
+			map.put(KEY_FETCH, values);
+		}
+		
+		if (push.size() > 0) {
+			Collection<String> values = new ArrayList<String>();
+			for (RefSpec refspec : push) {
+				values.add(refspec.toString());
+			}
+			map.put(KEY_PUSH, values);
+		}
+		
+		Collection<String> uploads = new ArrayList<String>(1);
+		uploads.add(uploadpack);
+		map.put(KEY_UPLOADPACK, uploads);
+		
+		Collection<String> receives = new ArrayList<String>(1);
+		receives.add(uploadpack);
+		map.put(KEY_RECEIVEPACK, receives);
+		
+		Collection<String> tag = new ArrayList<String>(1);
+		tag.add(tagopt.option());
+		map.put(KEY_TAGOPT, tag);
+		
+		
+		return map;
+	}
+
+	private void fromMap(Map<String, Collection<String>> map)
+			throws URISyntaxException {
+		for (Map.Entry<String, Collection<String>> entry : map.entrySet()) {
+			String key = entry.getKey();
+
+			if (key.equals(KEY_URL)) {
+				for (String value : entry.getValue()) {
+					uris.add(new URIish(value));
+				}
+			} else if (key.equals(KEY_FETCH)) {
+				for (String value : entry.getValue()) {
+					fetch.add(new RefSpec(value));
+				}
+			} else if (key.equals(KEY_PUSH)) {
+				for (String value : entry.getValue()) {
+					push.add(new RefSpec(value));
+				}
+			} else if (key.equals(KEY_UPLOADPACK)) {
+				for (String value : entry.getValue()) {
+					uploadpack = value;
+				}
+			} else if (key.equals(KEY_RECEIVEPACK)) {
+				for (String value : entry.getValue()) {
+					receivepack = value;
+				}
+			} else if (key.equals(KEY_TAGOPT)) {
+				for (String value : entry.getValue()) {
+					tagopt = TagOpt.fromOption(value);
+				}
+			}
+		}
+	}
+
+	public void readExternal(ObjectInput in) throws IOException,
+			ClassNotFoundException {
+		name = in.readUTF();
+		int items = in.readInt();
+
+		Map<String, Collection<String>> map = new HashMap<String,
Collection<String>>();
+		for (int i = 0; i < items; i++) {
+			String key = in.readUTF();
+			String value = in.readUTF();
+
+			Collection<String> values = map.get(key);
+			if (values == null) {
+				values = new ArrayList<String>();
+				map.put(key, values);
+			}
+
+			values.add(value);
+		}
+
+		try {
+			fromMap(map);
+		} catch (URISyntaxException ex) {
+			throw new IOException("Problem reading RemoteConfig map");
+		}
+	}
+
+	public void writeExternal(ObjectOutput out) throws IOException {
+		Map<String, Collection<String>> map = toMap();
+		
+		int size = 0;
+		
+		for (Map.Entry<String, Collection<String>> entry : map.entrySet()) {
+			size += entry.getValue().size();
+		}
+		
+		out.writeUTF(name);
+		out.writeInt(size);
+
+		for (Map.Entry<String, Collection<String>> entry : map.entrySet()) {
+			for (String value : entry.getValue()) {
+				out.writeUTF(entry.getKey());
+				out.writeUTF(value);
+			}
+		}
+	}
 }
diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/URIish.java
b/org.spearce.jgit/src/org/spearce/jgit/transport/URIish.java
index b86e00c..05e24b2 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/transport/URIish.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/transport/URIish.java
@@ -38,6 +38,10 @@

 package org.spearce.jgit.transport;

+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.regex.Matcher;
@@ -49,7 +53,7 @@
  * RFC 2396 URI's is that no URI encoding/decoding ever takes place. A space or
  * any special character is written as-is.
  */
-public class URIish {
+public class URIish implements Externalizable {
 	private static final Pattern FULL_URI = Pattern
 			.compile("^(?:([a-z][a-z0-9+-]+)://(?:([^/]+?)(?::([^/]+?))?@)?(?:([^/]+?))?(?::(\\d+))?)?((?:[A-Za-z]:)?/.+)$");

@@ -75,7 +79,17 @@
 	 * @throws URISyntaxException
 	 */
 	public URIish(String s) throws URISyntaxException {
-		s = s.replace('\\', '/');
+		initializeFromString(s);
+	}
+	
+	/**
+	 * Set fields from string based URI.
+	 *
+	 * @param s
+	 * @throws URISyntaxException
+	 */
+	private void initializeFromString(String s)  throws URISyntaxException {
+	    s = s.replace('\\', '/');
 		Matcher matcher = FULL_URI.matcher(s);
 		if (matcher.matches()) {
 			scheme = matcher.group(1);
@@ -357,4 +371,17 @@ private String format(final boolean includePassword) {

 		return r.toString();
 	}
+
+	public void readExternal(ObjectInput in) throws IOException,
+			ClassNotFoundException {
+	    try {
+			initializeFromString(in.readUTF());
+		} catch (URISyntaxException e) {
+			throw new IOException("Incorrect format URI");
+		}
+	}
+
+	public void writeExternal(ObjectOutput out) throws IOException {
+		out.writeUTF(format(true));
+	}
 }
-- 
1.6.0.2

Re: [JGIT PATCH] 1/2: Externalizable items

[Johannes Schindelin]

Hi,

On Mon, 16 Feb 2009, Nigel Magnay wrote:

> Make parts of jgit externalizable, so that they can be marshalled over
> the wire or onto disk,
> using formats from git mailing list.

Hmm.  I have to be honest and admit that I have no idea what you mean by 
externalizable.  From the "marshalling" comment, I'd have assumed that you 
mean Serializable, but then I still do not understand what "formats from 
git mailing list" are.  And that being a regular of said list for quite 
some time...

Care to enlighten me?

Ciao,
Dscho

Re: [JGIT PATCH] 1/2: Externalizable items

[Nigel Magnay]

Externalizable is the java standard for Serializable, but controlling
the exact format (rather than using introspection).

http://n2.nabble.com/-PATCH-JGIT--Minor-%3A-Make-ObjectId%2C-RemoteConfig-Serializable-td2286559.html#none


On Mon, Feb 16, 2009 at 4:59 PM, Johannes Schindelin
<Johannes.Schindelin@gmx.de> wrote:
> Hi,
>
> On Mon, 16 Feb 2009, Nigel Magnay wrote:
>
>> Make parts of jgit externalizable, so that they can be marshalled over
>> the wire or onto disk,
>> using formats from git mailing list.
>
> Hmm.  I have to be honest and admit that I have no idea what you mean by
> externalizable.  From the "marshalling" comment, I'd have assumed that you
> mean Serializable, but then I still do not understand what "formats from
> git mailing list" are.  And that being a regular of said list for quite
> some time...
>
> Care to enlighten me?
>
> Ciao,
> Dscho
>

Re: [JGIT PATCH] 1/2: Externalizable items

[Shawn O. Pearce]

Nigel Magnay <nigel.magnay@gmail.com> wrote:
> Make parts of jgit externalizable, so that they can be marshalled over
> the wire or onto disk,
> using formats from git mailing list.

As Dscho pointed out, a bit more detail here would be appreciated.
 
> diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/ObjectId.java
> b/org.spearce.jgit/src/org/spearce/jgit/lib/ObjectId.java
> index 52ce0d4..1385325 100644
> --- a/org.spearce.jgit/src/org/spearce/jgit/lib/ObjectId.java
> +++ b/org.spearce.jgit/src/org/spearce/jgit/lib/ObjectId.java
> @@ -56,6 +60,13 @@
>  	}
> 
>  	/**
> +	 * Empty constructor, for Externalizable.
> +	 */
> +	public ObjectId() {
> +		// For Externalizable
> +	}

Yikes.  Do we really need a public no-arg constructor for
Externalizable?  If we do, maybe we should use Serializable instead
so we can hide this constructor.  I don't like the idea of people
creating ObjectId.zeroId() by new ObjectId().  That's not a pattern
we should encourage.

> @@ -269,4 +280,22 @@ protected ObjectId(final AnyObjectId src) {
>  	public ObjectId toObjectId() {
>  		return this;
>  	}
> +
> +	public void readExternal(ObjectInput in) throws IOException,
> +			ClassNotFoundException {
> +		byte[] sha1 = new byte[20];
> +		in.read(sha1);
> +		
> +		w1 = NB.decodeInt32(sha1, 0);
> +		w2 = NB.decodeInt32(sha1, 4);
> +		w3 = NB.decodeInt32(sha1, 8);
> +		w4 = NB.decodeInt32(sha1, 12);
> +		w5 = NB.decodeInt32(sha1, 16);
> +	}
> +
> +	public void writeExternal(ObjectOutput out) throws IOException {
> +		byte[] sha1 = new byte[20];
> +		copyRawTo(sha1, 0);
> +		out.write(sha1);
> +	}

Hmm.  I was thinking of just writing the 5 ints out, and reading
the 5 ints back in.  We're always talking to another Java process.
The ints are written in network byte order anyway on a serialization
stream.  Doing this conversion to a byte[] thrases the caller's
per-thread new generation rather hard.  I think applications using
this type in a serialization stream would expect it to be quick.

> diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/RemoteConfig.java
> b/org.spearce.jgit/src/org/spearce/jgit/transport/RemoteConfig.java
> index 5bbf664..22443b4 100644
> --- a/org.spearce.jgit/src/org/spearce/jgit/transport/RemoteConfig.java
> +++ b/org.spearce.jgit/src/org/spearce/jgit/transport/RemoteConfig.java
> +
> +	public void readExternal(ObjectInput in) throws IOException,
> +			ClassNotFoundException {
> +		name = in.readUTF();
> +		int items = in.readInt();
> +
> +		Map<String, Collection<String>> map = new HashMap<String,
> Collection<String>>();
> +		for (int i = 0; i < items; i++) {
> +			String key = in.readUTF();
> +			String value = in.readUTF();

Why not just serialize the Map in the stream?

-- 
Shawn.

Re: [JGIT PATCH] 1/2: Externalizable items

[Nigel Magnay]

> Yikes.  Do we really need a public no-arg constructor for
> Externalizable?  If we do, maybe we should use Serializable instead
> so we can hide this constructor.  I don't like the idea of people
> creating ObjectId.zeroId() by new ObjectId().  That's not a pattern
> we should encourage.
>

Yes, you have to have a public no-args constructor for Externalizable.

 I agree, it's hideous. But I thought that was known as you explicitly
asked for Externalizable rather than Serializable with readObject /
writeObject... :-/

More than happy to re-roll with Serializable instead - do you want
this for all 4? (RemoteConfig also gained a no-args constructor
because of Externalizable..)

>> +     public void writeExternal(ObjectOutput out) throws IOException {
>> +             byte[] sha1 = new byte[20];
>> +             copyRawTo(sha1, 0);
>> +             out.write(sha1);
>> +     }
>
> Hmm.  I was thinking of just writing the 5 ints out, and reading
> the 5 ints back in.  We're always talking to another Java process.
> The ints are written in network byte order anyway on a serialization
> stream.  Doing this conversion to a byte[] thrases the caller's
> per-thread new generation rather hard.  I think applications using
> this type in a serialization stream would expect it to be quick.

I've taken the request for "the 20 byte SHA-1" too literally :-)

> +             Map<String, Collection<String>> map = new HashMap<String,
> Collection<String>>();
> +             for (int i = 0; i < items; i++) {
> +                     String key = in.readUTF();
> +                     String value = in.readUTF();
>Why not just serialize the Map in the stream?

Sure - if you're happy with that representation - it's not " a map of
keys/values as it appears in the config " though as it's a map to a
list because of the multi-values that are available for things like
URL and Fetch.

Re: [JGIT PATCH] 1/2: Externalizable items

[Shawn O. Pearce]

Nigel Magnay <nigel.magnay@gmail.com> wrote:
> > Yikes.  Do we really need a public no-arg constructor for
> > Externalizable?  If we do, maybe we should use Serializable instead
> > so we can hide this constructor.  I don't like the idea of people
> > creating ObjectId.zeroId() by new ObjectId().  That's not a pattern
> > we should encourage.
> >
> 
> Yes, you have to have a public no-args constructor for Externalizable.
> 
>  I agree, it's hideous. But I thought that was known as you explicitly
> asked for Externalizable rather than Serializable with readObject /
> writeObject... :-/

I forgot/didn't remember that Externalizable has this hideous
requirement.  I'd rather not introduce a public no-arg constructor
just for Java serialization.  So yea, if you needed to add a
constructor than we should instead use Serializable and define an
explicit serialVersionUID.  Sorry.
 
> More than happy to re-roll with Serializable instead - do you want
> this for all 4? (RemoteConfig also gained a no-args constructor
> because of Externalizable..)

Yea, at least for ObjectId and RemoteConfig.
 
> > +             Map<String, Collection<String>> map = new HashMap<String,
> > Collection<String>>();
> > +             for (int i = 0; i < items; i++) {
> > +                     String key = in.readUTF();
> > +                     String value = in.readUTF();
> >Why not just serialize the Map in the stream?
> 
> Sure - if you're happy with that representation - it's not " a map of
> keys/values as it appears in the config " though as it's a map to a
> list because of the multi-values that are available for things like
> URL and Fetch.

Right.

The format of HashMap and ArrayList is pretty stable on the wire,
so we can depend on them not changing on us.  We might as well
just serialize with those and save ourselves some code in the JGit
library.  I don't see a compelling reason here to use our own map
format on the wire.  Especially not if we are doing this conversion
to a java.util.Map<String,java.util.List> just to dump to the wire.

On the other hand, you could modify the read and write code to do
direct writing of string pairs, and direct reading of string pairs,
that would save allocs in each direction and make RemoteConfig
that much easier to write out.  Of course to do that you can't use
a count header, but instead would want a sential marker to break
the reader out of its loop, like an empty key string.

-- 
Shawn.