[JGIT PATCH 0/9] Misc. pack code cleanups

[JGIT PATCH 0/9] Misc. pack code cleanups

[Shawn O. Pearce]

This is a short batch of misc. cleanups related to pack handling.
I'm trying to work towards making it safer to run "git gc" while
JGit has a repository open.

Currently:

  - when a new pack is added we don't notice it;
  - when a previously listed pack is removed, we crash;
  - if a pack stays but its offsets change, we crash;
  - objects can go *poof* if they were loose and get packed;

This series doesn't fix any of these issues, but it cleans
up the code enough that I can start to consider this more.

The last patch is perhaps a bit more controversial.  It sets
core.packindex to 2 by default, which was done in C Git back
when 1.6.0 shipped.


Shawn O. Pearce (9):
  Remove the Repository parameter from PackFile's constructor
  Remove dead stats code from WindowCache
  Document why WindowFile's hash is *31
  Allow PackFile to lazily load its PackIndex on first demand
  Arrange pack files in recency order to improve quick hits
  Rename readPackHeader() to be onOpenPack()
  Validate the pack's footer checksum matches that in the index
  Remove yet another legacy StGit utility function
  Make pack.indexversion config option default to version 2

 .../src/org/spearce/jgit/pgm/IndexPack.java        |    4 +-
 .../tst/org/spearce/jgit/lib/PackWriterTest.java   |    2 +-
 .../tst/org/spearce/jgit/lib/T0004_PackReader.java |    2 +-
 .../org/spearce/jgit/transport/IndexPackTest.java  |    4 +-
 .../src/org/spearce/jgit/lib/CoreConfig.java       |    3 +-
 .../src/org/spearce/jgit/lib/PackFile.java         |   95 +++++++++++++------
 .../src/org/spearce/jgit/lib/PackIndex.java        |    3 +
 .../src/org/spearce/jgit/lib/PackIndexV1.java      |    3 +
 .../src/org/spearce/jgit/lib/PackIndexV2.java      |    3 +
 .../src/org/spearce/jgit/lib/PackWriter.java       |    1 +
 .../org/spearce/jgit/lib/PackedObjectLoader.java   |    4 +-
 .../src/org/spearce/jgit/lib/Repository.java       |   41 +++------
 .../src/org/spearce/jgit/lib/WindowCache.java      |   31 +------
 .../src/org/spearce/jgit/lib/WindowedFile.java     |    4 +
 14 files changed, 106 insertions(+), 94 deletions(-)

[JGIT PATCH 1/9] Remove the Repository parameter from PackFile's constructor

[Shawn O. Pearce]

This is a legacy parameter, from back when the WindowCache was
per-repository and not per-process.  We no longer need to pass
it down, so I'm ripping it out.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../tst/org/spearce/jgit/lib/PackWriterTest.java   |    2 +-
 .../tst/org/spearce/jgit/lib/T0004_PackReader.java |    2 +-
 .../org/spearce/jgit/transport/IndexPackTest.java  |    4 ++--
 .../src/org/spearce/jgit/lib/PackFile.java         |    5 +----
 .../src/org/spearce/jgit/lib/Repository.java       |    4 ++--
 5 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java b/org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java
index e828ccc..f7139fc 100644
--- a/org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java
+++ b/org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java
@@ -471,7 +471,7 @@ private void verifyOpenPack(final boolean thin) throws IOException {
 		indexer.setKeepEmpty(true);
 		indexer.setFixThin(thin);
 		indexer.index(new TextProgressMonitor());
-		pack = new PackFile(db, indexFile, packFile);
+		pack = new PackFile(indexFile, packFile);
 	}
 
 	private void verifyObjectsOrder(final ObjectId objectsOrder[]) {
diff --git a/org.spearce.jgit.test/tst/org/spearce/jgit/lib/T0004_PackReader.java b/org.spearce.jgit.test/tst/org/spearce/jgit/lib/T0004_PackReader.java
index 6ffb904..b9e5b49 100644
--- a/org.spearce.jgit.test/tst/org/spearce/jgit/lib/T0004_PackReader.java
+++ b/org.spearce.jgit.test/tst/org/spearce/jgit/lib/T0004_PackReader.java
@@ -54,7 +54,7 @@ public void test003_lookupCompressedObject() throws IOException {
 		final PackedObjectLoader or;
 
 		id = ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327");
-		pr = new PackFile(db, TEST_IDX, TEST_PACK);
+		pr = new PackFile(TEST_IDX, TEST_PACK);
 		or = pr.get(new WindowCursor(), id);
 		assertNotNull(or);
 		assertEquals(Constants.OBJ_TREE, or.getType());
diff --git a/org.spearce.jgit.test/tst/org/spearce/jgit/transport/IndexPackTest.java b/org.spearce.jgit.test/tst/org/spearce/jgit/transport/IndexPackTest.java
index 46bd969..0c6e678 100644
--- a/org.spearce.jgit.test/tst/org/spearce/jgit/transport/IndexPackTest.java
+++ b/org.spearce.jgit.test/tst/org/spearce/jgit/transport/IndexPackTest.java
@@ -68,7 +68,7 @@ public void test1() throws  IOException {
 		try {
 			IndexPack pack = new IndexPack(db, is, new File(trash, "tmp_pack1"));
 			pack.index(new TextProgressMonitor());
-			PackFile file = new PackFile(db, new File(trash, "tmp_pack1.idx"), new File(trash, "tmp_pack1.pack"));
+			PackFile file = new PackFile(new File(trash, "tmp_pack1.idx"), new File(trash, "tmp_pack1.pack"));
 			assertTrue(file.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904")));
 			assertTrue(file.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab")));
 			assertTrue(file.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259")));
@@ -94,7 +94,7 @@ public void test2() throws  IOException {
 		try {
 			IndexPack pack = new IndexPack(db, is, new File(trash, "tmp_pack2"));
 			pack.index(new TextProgressMonitor());
-			PackFile file = new PackFile(db, new File(trash, "tmp_pack2.idx"), new File(trash, "tmp_pack2.pack"));
+			PackFile file = new PackFile(new File(trash, "tmp_pack2.idx"), new File(trash, "tmp_pack2.pack"));
 			assertTrue(file.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9")));
 			assertTrue(file.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6")));
 			assertTrue(file.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680")));
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
index 0de4c55..8368827 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
@@ -65,8 +65,6 @@
 	/**
 	 * Construct a reader for an existing, pre-indexed packfile.
 	 * 
-	 * @param parentRepo
-	 *            Git repository holding this pack file
 	 * @param idxFile
 	 *            path of the <code>.idx</code> file listing the contents.
 	 * @param packFile
@@ -74,8 +72,7 @@
 	 * @throws IOException
 	 *             the index file cannot be accessed at this time.
 	 */
-	public PackFile(final Repository parentRepo, final File idxFile,
-			final File packFile) throws IOException {
+	public PackFile(final File idxFile, final File packFile) throws IOException {
 		pack = new WindowedFile(packFile) {
 			@Override
 			protected void onOpen() throws IOException {
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java b/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
index 8d35f82..963b6e0 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
@@ -800,7 +800,7 @@ synchronized (this) {
 			final PackFile[] cur = packFileList;
 			final PackFile[] arr = new PackFile[cur.length + 1];
 			System.arraycopy(cur, 0, arr, 1, cur.length);
-			arr[0] = new PackFile(this, idx, pack);
+			arr[0] = new PackFile(idx, pack);
 			packFileList = arr;
 		}
 	}
@@ -849,7 +849,7 @@ public boolean accept(final File baseDir, final String n) {
 				}
 
 				try {
-					packList.add(new PackFile(this, idxFile, packFile));
+					packList.add(new PackFile(idxFile, packFile));
 				} catch (IOException ioe) {
 					// Whoops. That's not a pack!
 					//
-- 
1.6.2.rc0.204.gf6b427

[JGIT PATCH 2/9] Remove dead stats code from WindowCache

[Shawn O. Pearce]

I used this code to try and tune the WindowCache a bit, but we're
about as good as we can do for now.  There's no point in keeping
it all around in somewhat critical paths if we never use it.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/lib/WindowCache.java      |   28 --------------------
 1 files changed, 0 insertions(+), 28 deletions(-)

diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/WindowCache.java b/org.spearce.jgit/src/org/spearce/jgit/lib/WindowCache.java
index 0077f52..600ebdf 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/WindowCache.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/WindowCache.java
@@ -76,14 +76,6 @@ private static final int bits(int newSize) {
 
 	private static int openByteCount;
 
-	private static int hits;
-
-	private static int reqs;
-
-	private static int opens;
-
-	private static int closes;
-
 	static {
 		maxByteCount = 10 * MB;
 		windowSizeShift = bits(8 * KB);
@@ -93,22 +85,6 @@ private static final int bits(int newSize) {
 		clearedWindowQueue = new ReferenceQueue<Object>();
 	}
 
-	static synchronized String statString() {
-		int maxChain = 0, tot = 0;
-		for (ByteWindow<?> e : cache) {
-			int n = 0;
-			for (; e != null; e = e.chainNext) {
-				n++;
-				tot++;
-			}
-			maxChain = Math.max(maxChain, n);
-		}
-		return "WindowCache[ hits: " + (hits * 100 / reqs) + "%"
-				+ "; windows: " + tot + " maxChain: " + maxChain + "; kb:"
-				+ (openByteCount / KB) + "; cache: " + cache.length + " fds: "
-				+ (opens - closes) + "," + opens + "," + closes + " ]";
-	}
-
 	private static int cacheTableSize() {
 		return 5 * (maxByteCount / windowSize) / 2;
 	}
@@ -215,7 +191,6 @@ private static synchronized void reconfigureImpl(final int packedGitLimit,
 	 */
 	public static synchronized final void get(final WindowCursor curs,
 			final WindowedFile wp, final long position) throws IOException {
-		reqs++;
 		final int id = (int) (position >> windowSizeShift);
 		final int idx = hash(wp, id);
 		for (ByteWindow<?> e = cache[idx]; e != null; e = e.chainNext) {
@@ -223,7 +198,6 @@ public static synchronized final void get(final WindowCursor curs,
 				if ((curs.handle = e.get()) != null) {
 					curs.window = e;
 					makeMostRecent(e);
-					hits++;
 					return;
 				}
 
@@ -234,7 +208,6 @@ public static synchronized final void get(final WindowCursor curs,
 
 		if (wp.openCount == 0) {
 			try {
-				opens++;
 				wp.openCount = 1;
 				wp.cacheOpen();
 			} catch (IOException ioe) {
@@ -340,7 +313,6 @@ private static void clear(final ByteWindow<?> e) {
 	private static void unlinkSize(final ByteWindow<?> e) {
 		if (e.sizeActive) {
 			if (--e.provider.openCount == 0) {
-				closes++;
 				e.provider.cacheClose();
 			}
 			openByteCount -= e.size;
-- 
1.6.2.rc0.204.gf6b427

[JGIT PATCH 3/9] Document why WindowFile's hash is *31

[Shawn O. Pearce]

It doesn't really make sense on its own, but if you know about
the assumption in WindowCache.hash() it all becomes a bit more
clear to the reader.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/lib/WindowCache.java      |    3 +++
 .../src/org/spearce/jgit/lib/WindowedFile.java     |    4 ++++
 2 files changed, 7 insertions(+), 0 deletions(-)

diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/WindowCache.java b/org.spearce.jgit/src/org/spearce/jgit/lib/WindowCache.java
index 600ebdf..0b9d20c 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/WindowCache.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/WindowCache.java
@@ -362,6 +362,9 @@ private static void insertLRU(final ByteWindow<?> e) {
 	}
 
 	private static int hash(final WindowedFile wp, final int id) {
+		// wp.hash was already "stirred up" a bit by * 31 when
+		// it was created. Its reasonable to just add here.
+		//
 		return ((wp.hash + id) >>> 1) % cache.length;
 	}
 
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/WindowedFile.java b/org.spearce.jgit/src/org/spearce/jgit/lib/WindowedFile.java
index 5eb8465..db8ea88 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/WindowedFile.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/WindowedFile.java
@@ -88,6 +88,10 @@
 	 */
 	public WindowedFile(final File file) {
 		fPath = file;
+
+		// Multiply by 31 here so we can more directly combine with another
+		// value in WindowCache.hash(), without doing the multiply there.
+		//
 		hash = System.identityHashCode(this) * 31;
 		length = Long.MAX_VALUE;
 	}
-- 
1.6.2.rc0.204.gf6b427

[JGIT PATCH 4/9] Allow PackFile to lazily load its PackIndex on first demand

[Shawn O. Pearce]

Repositories which contain several pack files may have a very large
historical pack holding the bulk of the project data, and a much
smaller more recent pack holding current data.  In such cases we
may be able to answer queries entirely by reading the recent pack
index, and never reading the larger historical index.  Since JGit
reads the entire index into memory, avoiding reading the larger
index can save some time on short uses of a Repository.

This change only sets up the lazy loading of the index.  Because
the packs aren't sorted by newest-to-oldest in Repository we are
still very likely to demand load packs we didn't need to examine.

This change also opens the door for potentially releasing one or
more PackIndex objects when memory gets low, but doesn't actually
implement any sort of soft reference semantics.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/lib/PackFile.java         |   67 +++++++++++++-------
 .../org/spearce/jgit/lib/PackedObjectLoader.java   |    4 +-
 .../src/org/spearce/jgit/lib/Repository.java       |   19 +++---
 3 files changed, 56 insertions(+), 34 deletions(-)

diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
index 8368827..3542560 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
@@ -42,6 +42,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.zip.CRC32;
 import java.util.zip.CheckedOutputStream;
@@ -56,9 +57,11 @@
  * objects are similar.
  */
 public class PackFile implements Iterable<PackIndex.MutableEntry> {
+	private final File idxFile;
+
 	private final WindowedFile pack;
 
-	private final PackIndex idx;
+	private PackIndex loadedIdx;
 
 	private PackReverseIndex reverseIdx;
 
@@ -69,21 +72,22 @@
 	 *            path of the <code>.idx</code> file listing the contents.
 	 * @param packFile
 	 *            path of the <code>.pack</code> file holding the data.
-	 * @throws IOException
-	 *             the index file cannot be accessed at this time.
 	 */
-	public PackFile(final File idxFile, final File packFile) throws IOException {
+	public PackFile(final File idxFile, final File packFile) {
+		this.idxFile = idxFile;
 		pack = new WindowedFile(packFile) {
 			@Override
 			protected void onOpen() throws IOException {
 				readPackHeader();
 			}
 		};
-		try {
-			idx = PackIndex.open(idxFile);
-		} catch (IOException ioe) {
-			throw ioe;
+	}
+
+	private synchronized PackIndex idx() throws IOException {
+		if (loadedIdx == null) {
+			loadedIdx = PackIndex.open(idxFile);
 		}
+		return loadedIdx;
 	}
 
 	final PackedObjectLoader resolveBase(final WindowCursor curs, final long ofs)
@@ -106,9 +110,11 @@ public File getPackFile() {
 	 * @param id
 	 *            the object to look for. Must not be null.
 	 * @return true if the object is in this pack; false otherwise.
+	 * @throws IOException
+	 *             the index file cannot be loaded into memory.
 	 */
-	public boolean hasObject(final AnyObjectId id) {
-		return idx.hasObject(id);
+	public boolean hasObject(final AnyObjectId id) throws IOException {
+		return idx().hasObject(id);
 	}
 
 	/**
@@ -125,7 +131,7 @@ public boolean hasObject(final AnyObjectId id) {
 	 */
 	public PackedObjectLoader get(final WindowCursor curs, final AnyObjectId id)
 			throws IOException {
-		final long offset = idx.findOffset(id);
+		final long offset = idx().findOffset(id);
 		return 0 < offset ? reader(curs, offset) : null;
 	}
 
@@ -135,6 +141,9 @@ public PackedObjectLoader get(final WindowCursor curs, final AnyObjectId id)
 	public void close() {
 		UnpackedObjectCache.purge(pack);
 		pack.close();
+		synchronized (this) {
+			loadedIdx = null;
+		}
 	}
 
 	/**
@@ -150,7 +159,11 @@ public void close() {
 	 * @see PackIndex#iterator()
 	 */
 	public Iterator<PackIndex.MutableEntry> iterator() {
-		return idx.iterator();
+		try {
+			return idx().iterator();
+		} catch (IOException e) {
+			return Collections.<PackIndex.MutableEntry> emptyList().iterator();
+		}
 	}
 
 	/**
@@ -158,9 +171,11 @@ public void close() {
 	 * relies on pack index, giving number of effectively available objects.
 	 * 
 	 * @return number of objects in index of this pack, likewise in this pack
+	 * @throws IOException
+	 *             the index file cannot be loaded into memory.
 	 */
-	long getObjectCount() {
-		return idx.getObjectCount();
+	long getObjectCount() throws IOException {
+		return idx().getObjectCount();
 	}
 
 	/**
@@ -170,8 +185,10 @@ long getObjectCount() {
 	 * @param offset
 	 *            start offset of object to find
 	 * @return object id for this offset, or null if no object was found
+	 * @throws IOException
+	 *             the index file cannot be loaded into memory.
 	 */
-	ObjectId findObjectForOffset(final long offset) {
+	ObjectId findObjectForOffset(final long offset) throws IOException {
 		return getReverseIdx().findObject(offset);
 	}
 
@@ -196,6 +213,7 @@ final void copyRawData(final PackedObjectLoader loader,
 		final long dataOffset = loader.dataOffset;
 		final int cnt = (int) (findEndOffset(objectOffset) - dataOffset);
 		final WindowCursor curs = loader.curs;
+		final PackIndex idx = idx();
 
 		if (idx.hasCRC32Support()) {
 			final CRC32 crc = new CRC32();
@@ -231,8 +249,8 @@ final void copyRawData(final PackedObjectLoader loader,
 		}
 	}
 
-	boolean supportsFastCopyRawData() {
-		return idx.hasCRC32Support();
+	boolean supportsFastCopyRawData() throws IOException {
+		return idx().hasCRC32Support();
 	}
 
 
@@ -258,11 +276,12 @@ private void readPackHeader() throws IOException {
 		position += 4;
 
 		pack.readFully(position, intbuf, curs);
-		final long objectCnt = NB.decodeUInt32(intbuf, 0);
-		if (idx.getObjectCount() != objectCnt)
+		final long packCnt = NB.decodeUInt32(intbuf, 0);
+		final long idxCnt = idx().getObjectCount();
+		if (idxCnt != packCnt)
 			throw new IOException("Pack index"
-					+ " object count mismatch; expected " + objectCnt
-					+ " found " + idx.getObjectCount() + ": " + pack.getName());
+					+ " object count mismatch; expected " + packCnt
+					+ " found " + idxCnt + ": " + pack.getName());
 	}
 
 	private PackedObjectLoader reader(final WindowCursor curs,
@@ -315,14 +334,14 @@ private PackedObjectLoader reader(final WindowCursor curs,
 	}
 
 	private long findEndOffset(final long startOffset)
-			throws CorruptObjectException {
+			throws IOException, CorruptObjectException {
 		final long maxOffset = pack.length() - Constants.OBJECT_ID_LENGTH;
 		return getReverseIdx().findNextOffset(startOffset, maxOffset);
 	}
 
-	private synchronized PackReverseIndex getReverseIdx() {
+	private synchronized PackReverseIndex getReverseIdx() throws IOException {
 		if (reverseIdx == null)
-			reverseIdx = new PackReverseIndex(idx);
+			reverseIdx = new PackReverseIndex(idx());
 		return reverseIdx;
 	}
 }
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackedObjectLoader.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackedObjectLoader.java
index 35983fe..56985c1 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackedObjectLoader.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackedObjectLoader.java
@@ -103,8 +103,10 @@ public void copyRawData(OutputStream out, byte buf[]) throws IOException {
 	 * @return true if this loader is capable of fast raw-data copying basing on
 	 *         compressed data checksum; false if raw-data copying needs
 	 *         uncompressing and compressing data
+	 * @throws IOException
+	 *             the index file format cannot be determined.
 	 */
-	public boolean supportsFastCopyRawData() {
+	public boolean supportsFastCopyRawData() throws IOException {
 		return pack.supportsFastCopyRawData();
 	}
 
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java b/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
index 963b6e0..b9c7b2e 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
@@ -248,8 +248,15 @@ public boolean hasObject(final AnyObjectId objectId) {
 		final PackFile[] packs = packs();
 		int k = packs.length;
 		while (k > 0) {
-			if (packs[--k].hasObject(objectId))
-				return true;
+			try {
+				if (packs[--k].hasObject(objectId))
+					return true;
+			} catch (IOException e) {
+				// Assume that means the pack is invalid, and such
+				// packs are treated as though they are empty.
+				//
+				continue;
+			}
 		}
 		return toFile(objectId).isFile();
 	}
@@ -848,13 +855,7 @@ public boolean accept(final File baseDir, final String n) {
 						continue SCAN;
 				}
 
-				try {
-					packList.add(new PackFile(idxFile, packFile));
-				} catch (IOException ioe) {
-					// Whoops. That's not a pack!
-					//
-					ioe.printStackTrace();
-				}
+				packList.add(new PackFile(idxFile, packFile));
 			}
 		}
 	}
-- 
1.6.2.rc0.204.gf6b427

[JGIT PATCH 5/9] Arrange pack files in recency order to improve quick hits

[Shawn O. Pearce]

Packs with newer modification dates are more likely to contain data
related to recent commits, and thus will have a better probability
of containing an object we care about when we are looking only at
recent history.

In some cases, this may permit JGit to avoid reading a very large
historical pack's index file into memory if the only data we ever
need is available in newer pack files.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/lib/PackFile.java         |   11 +++++++++++
 .../src/org/spearce/jgit/lib/Repository.java       |    1 +
 2 files changed, 12 insertions(+), 0 deletions(-)

diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
index 3542560..7a16bf7 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
@@ -43,6 +43,7 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Iterator;
 import java.util.zip.CRC32;
 import java.util.zip.CheckedOutputStream;
@@ -57,10 +58,19 @@
  * objects are similar.
  */
 public class PackFile implements Iterable<PackIndex.MutableEntry> {
+	/** Sorts PackFiles to be most recently created to least recently created. */
+	public static Comparator<PackFile> SORT = new Comparator<PackFile>() {
+		public int compare(final PackFile a, final PackFile b) {
+			return b.packLastModified - a.packLastModified;
+		}
+	};
+
 	private final File idxFile;
 
 	private final WindowedFile pack;
 
+	private int packLastModified;
+
 	private PackIndex loadedIdx;
 
 	private PackReverseIndex reverseIdx;
@@ -75,6 +85,7 @@
 	 */
 	public PackFile(final File idxFile, final File packFile) {
 		this.idxFile = idxFile;
+		this.packLastModified = (int) (packFile.lastModified() >> 10);
 		pack = new WindowedFile(packFile) {
 			@Override
 			protected void onOpen() throws IOException {
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java b/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
index b9c7b2e..96c62df 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
@@ -823,6 +823,7 @@ public void scanForPacks() {
 			scanForPacks(new File(d, "pack"), p);
 		final PackFile[] arr = new PackFile[p.size()];
 		p.toArray(arr);
+		Arrays.sort(arr, PackFile.SORT);
 		synchronized (this) {
 			packFileList = arr;
 		}
-- 
1.6.2.rc0.204.gf6b427

[JGIT PATCH 6/9] Rename readPackHeader() to be onOpenPack()

[Shawn O. Pearce]

Its just a better name of the function.  Any logic we might try
to perform to validate the pack file as we open it up to read
should be here, and that logic may eventually include validation
of the footer checksum against the index's footer data.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/lib/PackFile.java         |    5 ++---
 1 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
index 7a16bf7..d34df03 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
@@ -89,7 +89,7 @@ public PackFile(final File idxFile, final File packFile) {
 		pack = new WindowedFile(packFile) {
 			@Override
 			protected void onOpen() throws IOException {
-				readPackHeader();
+				onOpenPack();
 			}
 		};
 	}
@@ -264,8 +264,7 @@ boolean supportsFastCopyRawData() throws IOException {
 		return idx().hasCRC32Support();
 	}
 
-
-	private void readPackHeader() throws IOException {
+	private void onOpenPack() throws IOException {
 		final WindowCursor curs = new WindowCursor();
 		long position = 0;
 		final byte[] sig = new byte[Constants.PACK_SIGNATURE.length];
-- 
1.6.2.rc0.204.gf6b427

[JGIT PATCH 7/9] Validate the pack's footer checksum matches that in the index

[Shawn O. Pearce]

If the SHA-1 checksum of the entire pack content does not match the
checksum stored in the corresponding *.idx file we cannot use the
index, as the offsets recorded with it are most likely incorrect.

This situation can arise if another application repacks the pack
with different compression settings, but puts the same set of
objects into it.  The offsets are going to be different, but the
resulting pack file and index file names will be the same.  If we
had previously read the old index into memory, but the pack isn't
a match anymore, we can't safely access this pack.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/lib/PackFile.java         |   11 ++++++++++-
 .../src/org/spearce/jgit/lib/PackIndex.java        |    3 +++
 .../src/org/spearce/jgit/lib/PackIndexV1.java      |    3 +++
 .../src/org/spearce/jgit/lib/PackIndexV2.java      |    3 +++
 4 files changed, 19 insertions(+), 1 deletions(-)

diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
index d34df03..1ee0ad7 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java
@@ -42,6 +42,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
@@ -265,6 +266,7 @@ boolean supportsFastCopyRawData() throws IOException {
 	}
 
 	private void onOpenPack() throws IOException {
+		final PackIndex idx = idx();
 		final WindowCursor curs = new WindowCursor();
 		long position = 0;
 		final byte[] sig = new byte[Constants.PACK_SIGNATURE.length];
@@ -287,11 +289,18 @@ private void onOpenPack() throws IOException {
 
 		pack.readFully(position, intbuf, curs);
 		final long packCnt = NB.decodeUInt32(intbuf, 0);
-		final long idxCnt = idx().getObjectCount();
+		final long idxCnt = idx.getObjectCount();
 		if (idxCnt != packCnt)
 			throw new IOException("Pack index"
 					+ " object count mismatch; expected " + packCnt
 					+ " found " + idxCnt + ": " + pack.getName());
+		
+		final byte[] csumbuf = new byte[20];
+		pack.readFully(pack.length() - 20, csumbuf, curs);
+		if (!Arrays.equals(csumbuf, idx.packChecksum))
+			throw new IOException("Pack index mismatch; pack SHA-1 is "
+					+ ObjectId.fromRaw(csumbuf).name() + ", index expects "
+					+ ObjectId.fromRaw(idx.packChecksum).name());
 	}
 
 	private PackedObjectLoader reader(final WindowCursor curs,
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndex.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndex.java
index 13acbd1..473ce1e 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndex.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndex.java
@@ -112,6 +112,9 @@ private static boolean isTOC(final byte[] h) {
 		return true;
 	}
 
+	/** Footer checksum applied on the bottom of the pack file. */
+	protected byte[] packChecksum;
+
 	/**
 	 * Determine if an object is contained within the pack file.
 	 * 
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexV1.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexV1.java
index 90b5f6f..fdaa094 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexV1.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexV1.java
@@ -79,6 +79,9 @@ PackIndexV1(final InputStream fd, final byte[] hdr)
 			}
 		}
 		objectCnt = idxHeader[255];
+
+		packChecksum = new byte[20];
+		NB.readFully(fd, packChecksum, 0, packChecksum.length);
 	}
 
 	long getObjectCount() {
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexV2.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexV2.java
index 48a5206..eb56ed9 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexV2.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexV2.java
@@ -142,6 +142,9 @@ PackIndexV2(final InputStream fd) throws IOException {
 		} else {
 			offset64 = NO_BYTES;
 		}
+
+		packChecksum = new byte[20];
+		NB.readFully(fd, packChecksum, 0, packChecksum.length);
 	}
 
 	@Override
-- 
1.6.2.rc0.204.gf6b427

[JGIT PATCH 8/9] Remove yet another legacy StGit utility function

[Shawn O. Pearce]

We have long since ripped out any StGit support from JGit or EGit.
There is no current caller for this function, and it should have
been removed much sooner.  Take it out now anyway.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/lib/Repository.java       |   19 -------------------
 1 files changed, 0 insertions(+), 19 deletions(-)

diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java b/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
index 96c62df..30bd4a3 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/Repository.java
@@ -878,25 +878,6 @@ public String toString() {
 	}
 
 	/**
-	 * @return name of topmost Stacked Git patch.
-	 * @throws IOException
-	 */
-	public String getPatch() throws IOException {
-		final File ptr = new File(getDirectory(),"patches/"+getBranch()+"/applied");
-		final BufferedReader br = new BufferedReader(new FileReader(ptr));
-		String last=null;
-		try {
-			String line;
-			while ((line=br.readLine())!=null) {
-				last = line;
-			}
-		} finally {
-			br.close();
-		}
-		return last;
-	}
-
-	/**
 	 * @return name of current branch
 	 * @throws IOException
 	 */
-- 
1.6.2.rc0.204.gf6b427

[JGIT PATCH 9/9] Make pack.indexversion config option default to version 2

[Shawn O. Pearce]

Git 1.6.0 (released Sun Aug 17 11:42:22 2008 -0700) defaults
to creating the much safer pack index version 2 format when
writing a pack to disk.  Most clients trying to use Git will
be running a recent version of C Git alongside JGit so it is
reasonably safe to assume they have index version 2 reading
support, and are thus prepared to accept this change in the
default output format.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/pgm/IndexPack.java        |    4 +++-
 .../src/org/spearce/jgit/lib/CoreConfig.java       |    3 ++-
 .../src/org/spearce/jgit/lib/PackWriter.java       |    1 +
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/org.spearce.jgit.pgm/src/org/spearce/jgit/pgm/IndexPack.java b/org.spearce.jgit.pgm/src/org/spearce/jgit/pgm/IndexPack.java
index 5eacaa4..22803a4 100644
--- a/org.spearce.jgit.pgm/src/org/spearce/jgit/pgm/IndexPack.java
+++ b/org.spearce.jgit.pgm/src/org/spearce/jgit/pgm/IndexPack.java
@@ -49,13 +49,15 @@
 	private boolean fixThin;
 
 	@Option(name = "--index-version", usage = "index file format to create")
-	private int indexVersion;
+	private int indexVersion = -1;
 
 	@Argument(index = 0, required = true, metaVar = "base")
 	private File base;
 
 	@Override
 	protected void run() throws Exception {
+		if (indexVersion == -1)
+			indexVersion = db.getConfig().getCore().getPackIndexVersion();
 		final BufferedInputStream in;
 		final org.spearce.jgit.transport.IndexPack ip;
 		in = new BufferedInputStream(System.in);
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/CoreConfig.java b/org.spearce.jgit/src/org/spearce/jgit/lib/CoreConfig.java
index 2dd8aea..e98e0bc 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/CoreConfig.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/CoreConfig.java
@@ -45,6 +45,7 @@
  */
 public class CoreConfig {
 	private static final int DEFAULT_COMPRESSION = Deflater.DEFAULT_COMPRESSION;
+	private static final int DEFAULT_INDEXVERSION = 2;
 
 	private final int compression;
 
@@ -52,7 +53,7 @@
 
 	CoreConfig(final RepositoryConfig rc) {
 		compression = rc.getInt("core", "compression", DEFAULT_COMPRESSION);
-		packIndexVersion = rc.getInt("pack", "indexversion", 0);
+		packIndexVersion = rc.getInt("pack", "indexversion", DEFAULT_INDEXVERSION);
 	}
 
 	/**
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackWriter.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackWriter.java
index cba2ee7..f9945c4 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackWriter.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackWriter.java
@@ -237,6 +237,7 @@ public PackWriter(final Repository repo, final ProgressMonitor imonitor,
 		initMonitor = imonitor;
 		writeMonitor = wmonitor;
 		this.deflater = new Deflater(db.getConfig().getCore().getCompression());
+		outputVersion = repo.getConfig().getCore().getPackIndexVersion();
 	}
 
 	/**
-- 
1.6.2.rc0.204.gf6b427