git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[EGIT PATCH 20/20] PackWriter test suite

From
MZMarek Zawirski <marek.zawirski@gmail.com>
Date
Jun 15, 2008, 21:45 UTC
Message-ID
<1213566349-25395-21-git-send-email-marek.zawirski@gmail.com>
In-Reply-To
<1213566349-25395-20-git-send-email-marek.zawirski@gmail.com>

Test suite is provided relying on IndexPack and PackIndex to verify PackWriter output for various configurations.

Signed-off-by: Marek Zawirski <marek.zawirski@gmail.com>
---
 .../tst/org/spearce/jgit/lib/PackWriterTest.java   |  454 ++++++++++++++++++++
 1 files changed, 454 insertions(+), 0 deletions(-)
 create mode 100644 org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java
diff --git a/org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java b/org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java
new file mode 100644
index 0000000..9572342
--- /dev/null
+++ b/org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java
@@ -0,0 +1,454 @@
+/*
+ * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Git Development Community nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.spearce.jgit.lib;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.spearce.jgit.errors.MissingObjectException;
+import org.spearce.jgit.lib.PackIndex.MutableEntry;
+import org.spearce.jgit.revwalk.RevObject;
+import org.spearce.jgit.revwalk.RevWalk;
+import org.spearce.jgit.transport.IndexPack;
+import org.spearce.jgit.util.CountingOutputStream;
+
+public class PackWriterTest extends RepositoryTestCase {
+
+	private static final List<ObjectId> EMPTY_LIST_OBJECT = Collections
+			.<ObjectId> emptyList();
+
+	private static final List<RevObject> EMPTY_LIST_REVS = Collections
+			.<RevObject> emptyList();
+
+	private PackWriter writer;
+
+	private ByteArrayOutputStream os;
+
+	private CountingOutputStream cos;
+
+	private File packBase;
+
+	private File packFile;
+
+	private File indexFile;
+
+	private PackFile pack;
+
+	public void setUp() throws Exception {
+		super.setUp();
+		os = new ByteArrayOutputStream();
+		cos = new CountingOutputStream(os);
+		packBase = new File(trash, "tmp_pack");
+		packFile = new File(trash, "tmp_pack.pack");
+		indexFile = new File(trash, "tmp_pack.idx");
+		writer = new PackWriter(db, cos, new TextProgressMonitor());
+	}
+
+	/**
+	 * Test constructor for exceptions, default settings, initialization.
+	 */
+	public void testContructor() {
+		assertEquals(false, writer.isDeltaBaseAsOffset());
+		assertEquals(true, writer.isReuseDeltas());
+		assertEquals(true, writer.isReuseObjects());
+		assertEquals(0, writer.getObjectsNumber());
+	}
+
+	/**
+	 * Change default settings and verify them.
+	 */
+	public void testModifySettings() {
+		writer.setDeltaBaseAsOffset(true);
+		writer.setReuseDeltas(false);
+		writer.setReuseObjects(false);
+
+		assertEquals(true, writer.isDeltaBaseAsOffset());
+		assertEquals(false, writer.isReuseDeltas());
+		assertEquals(false, writer.isReuseObjects());
+	}
+
+	/**
+	 * Write empty pack by providing empty sets of interesting/uninteresting
+	 * objects and check for correct format.
+	 * 
+	 * @throws IOException
+	 */
+	public void testWriteEmptyPack1() throws IOException {
+		createVerifyOpenPack(EMPTY_LIST_OBJECT, EMPTY_LIST_OBJECT, false);
+
+		assertEquals(0, writer.getObjectsNumber());
+		assertEquals(0, pack.getObjectCount());
+		assertEquals("da39a3ee5e6b4b0d3255bfef95601890afd80709", writer
+				.computeName().toString());
+	}
+
+	/**
+	 * Write empty pack by providing empty iterator of objects to write and
+	 * check for correct format.
+	 * 
+	 * @throws IOException
+	 */
+	public void testWriteEmptyPack2() throws IOException {
+		createVerifyOpenPack(EMPTY_LIST_REVS.iterator());
+
+		assertEquals(0, writer.getObjectsNumber());
+		assertEquals(0, pack.getObjectCount());
+	}
+
+	/**
+	 * Create pack basing on only interesting objects, then precisely verify
+	 * content. No delta reuse here.
+	 * 
+	 * @throws IOException
+	 */
+	public void testWritePack1() throws IOException {
+		writer.setReuseDeltas(false);
+		writeVerifyPack1();
+	}
+
+	/**
+	 * Test writing pack without object reuse. Pack content/preparation as in
+	 * {@link #testWritePack1()}.
+	 * 
+	 * @throws IOException
+	 */
+	public void testWritePack1NoObjectReuse() throws IOException {
+		writer.setReuseDeltas(false);
+		writer.setReuseObjects(false);
+		writeVerifyPack1();
+	}
+
+	/**
+	 * Create pack basing on both interesting and uninteresting objects, then
+	 * precisely verify content. No delta reuse here.
+	 * 
+	 * @throws IOException
+	 */
+	public void testWritePack2() throws IOException {
+		writeVerifyPack2(false);
+	}
+
+	/**
+	 * Test pack writing with deltas reuse, delta-base first rule. Pack
+	 * content/preparation as in {@link #testWritePack2()}.
+	 * 
+	 * @throws IOException
+	 */
+	public void testWritePack2DeltasReuseRefs() throws IOException {
+		writeVerifyPack2(true);
+	}
+
+	/**
+	 * Test pack writing with delta reuse. Delta bases referred as offsets. Pack
+	 * configuration as in {@link #testWritePack2DeltasReuseRefs()}.
+	 * 
+	 * @throws IOException
+	 */
+	public void testWritePack2DeltasReuseOffsets() throws IOException {
+		writer.setDeltaBaseAsOffset(true);
+		writeVerifyPack2(true);
+	}
+
+	/**
+	 * Test pack writing with delta reuse. Raw-data copy (reuse) is made on a
+	 * pack with CRC32 index. Pack configuration as in
+	 * {@link #testWritePack2DeltasReuseRefs()}.
+	 * 
+	 * @throws IOException
+	 */
+	public void testWritePack2DeltasCRC32Copy() throws IOException {
+		final File packDir = new File(db.getObjectsDirectory(), "pack");
+		final File crc32Pack = new File(packDir,
+				"pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
+		final File crc32Idx = new File(packDir,
+				"pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx");
+		copyFile(new File(new File("tst"),
+				"pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"),
+				crc32Idx);
+		db.openPack(crc32Pack, crc32Idx);
+
+		writeVerifyPack2(true);
+	}
+
+	/**
+	 * Create pack basing on fixed objects list, then precisely verify content.
+	 * No delta reuse here.
+	 * 
+	 * @throws IOException
+	 * @throws MissingObjectException
+	 * 
+	 */
+	public void testWritePack3() throws MissingObjectException, IOException {
+		writer.setReuseDeltas(false);
+		final ObjectId forcedOrder[] = new ObjectId[] {
+				ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
+				ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
+				ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
+				ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
+				ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
+				ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
+		final RevWalk parser = new RevWalk(db);
+		final RevObject forcedOrderRevs[] = new RevObject[forcedOrder.length];
+		for (int i = 0; i < forcedOrder.length; i++)
+			forcedOrderRevs[i] = parser.parseAny(forcedOrder[i]);
+
+		createVerifyOpenPack(Arrays.asList(forcedOrderRevs).iterator());
+
+		assertEquals(forcedOrder.length, writer.getObjectsNumber());
+		verifyObjectsOrder(forcedOrder);
+		assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
+				.computeName().toString());
+	}
+
+	/**
+	 * Another pack creation: basing on both interesting and uninteresting
+	 * objects. No delta reuse possible here, as this is a specific case when we
+	 * write only 1 commit, associated with 1 tree, 1 blob.
+	 * 
+	 * @throws IOException
+	 */
+	public void testWritePack4() throws IOException {
+		writeVerifyPack4(false);
+	}
+
+	/**
+	 * Test thin pack writing: 1 blob delta base is on objects edge. Pack
+	 * configuration as in {@link #testWritePack4()}.
+	 * 
+	 * @throws IOException
+	 */
+	public void testWritePack4ThinPack() throws IOException {
+		writeVerifyPack4(true);
+	}
+
+	/**
+	 * Compare sizes of packs created using {@link #testWritePack2()} and
+	 * {@link #testWritePack2DeltasReuseRefs()}. The pack using deltas should
+	 * be smaller.
+	 * 
+	 * @throws Exception
+	 */
+	public void testWritePack2SizeDeltasVsNoDeltas() throws Exception {
+		testWritePack2();
+		final int sizePack2NoDeltas = cos.getCount();
+		setUp();
+		testWritePack2DeltasReuseRefs();
+		final int sizePack2DeltasRefs = cos.getCount();
+
+		assertTrue(sizePack2NoDeltas > sizePack2DeltasRefs);
+	}
+
+	/**
+	 * Compare sizes of packs created using
+	 * {@link #testWritePack2DeltasReuseRefs()} and
+	 * {@link #testWritePack2DeltasReuseOffsets()}. The pack with delta bases
+	 * written as offsets should be smaller.
+	 * 
+	 * @throws Exception
+	 */
+	public void testWritePack2SizeOffsetsVsRefs() throws Exception {
+		testWritePack2DeltasReuseRefs();
+		final int sizePack2DeltasRefs = cos.getCount();
+		setUp();
+		testWritePack2DeltasReuseOffsets();
+		final int sizePack2DeltasOffsets = cos.getCount();
+
+		assertTrue(sizePack2DeltasRefs > sizePack2DeltasOffsets);
+	}
+
+	/**
+	 * Compare sizes of packs created using {@link #testWritePack4()} and
+	 * {@link #testWritePack4ThinPack()}. Obviously, the thin pack should be
+	 * smaller.
+	 * 
+	 * @throws Exception
+	 */
+	public void testWritePack4SizeThinVsNoThin() throws Exception {
+		testWritePack4();
+		final int sizePack4 = cos.getCount();
+		setUp();
+		testWritePack4ThinPack();
+		final int sizePack4Thin = cos.getCount();
+
+		assertTrue(sizePack4 > sizePack4Thin);
+	}
+
+	// TODO: testWritePackDeltasCycle()
+	// TODO: testWritePackDeltasDepth()
+
+	private void writeVerifyPack1() throws IOException {
+		final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
+		interestings.add(ObjectId
+				.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
+		createVerifyOpenPack(interestings, EMPTY_LIST_OBJECT, false);
+
+		final ObjectId expectedOrder[] = new ObjectId[] {
+				ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
+				ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
+				ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"),
+				ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
+				ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
+				ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"),
+				ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
+				ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
+
+		assertEquals(expectedOrder.length, writer.getObjectsNumber());
+		verifyObjectsOrder(expectedOrder);
+		assertEquals("34be9032ac282b11fa9babdc2b2a93ca996c9c2f", writer
+				.computeName().toString());
+	}
+
+	private void writeVerifyPack2(boolean deltaReuse) throws IOException {
+		writer.setReuseDeltas(deltaReuse);
+		final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
+		interestings.add(ObjectId
+				.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
+		final LinkedList<ObjectId> uninterestings = new LinkedList<ObjectId>();
+		uninterestings.add(ObjectId
+				.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"));
+		createVerifyOpenPack(interestings, uninterestings, false);
+
+		final ObjectId expectedOrder[] = new ObjectId[] {
+				ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
+				ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
+				ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
+				ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
+				ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
+				ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
+		if (deltaReuse) {
+			// objects order influenced (swapped) by delta-base first rule
+			ObjectId temp = expectedOrder[4];
+			expectedOrder[4] = expectedOrder[5];
+			expectedOrder[5] = temp;
+		}
+		assertEquals(expectedOrder.length, writer.getObjectsNumber());
+		verifyObjectsOrder(expectedOrder);
+		assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
+				.computeName().toString());
+	}
+
+	private void writeVerifyPack4(final boolean thin) throws IOException {
+		final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
+		interestings.add(ObjectId
+				.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
+		final LinkedList<ObjectId> uninterestings = new LinkedList<ObjectId>();
+		uninterestings.add(ObjectId
+				.fromString("c59759f143fb1fe21c197981df75a7ee00290799"));
+		createVerifyOpenPack(interestings, uninterestings, thin);
+
+		final ObjectId writtenObjects[] = new ObjectId[] {
+				ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
+				ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
+				ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
+		assertEquals(writtenObjects.length, writer.getObjectsNumber());
+		ObjectId expectedObjects[];
+		if (thin) {
+			expectedObjects = new ObjectId[4];
+			System.arraycopy(writtenObjects, 0, expectedObjects, 0,
+					writtenObjects.length);
+			expectedObjects[3] = ObjectId
+					.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3");
+
+		} else {
+			expectedObjects = writtenObjects;
+		}
+		verifyObjectsOrder(expectedObjects);
+		assertEquals("cded4b74176b4456afa456768b2b5aafb41c44fc", writer
+				.computeName().toString());
+	}
+
+	private void createVerifyOpenPack(final Collection<ObjectId> interestings,
+			final Collection<ObjectId> uninterestings, final boolean thin)
+			throws MissingObjectException, IOException {
+		writer.writePack(interestings, uninterestings, thin);
+		verifyOpenPack(thin);
+	}
+
+	private void createVerifyOpenPack(final Iterator<RevObject> objectSource)
+			throws MissingObjectException, IOException {
+		writer.writePack(objectSource);
+		verifyOpenPack(false);
+	}
+
+	private void verifyOpenPack(final boolean thin) throws IOException {
+		if (thin) {
+			final InputStream is = new ByteArrayInputStream(os.toByteArray());
+			final IndexPack indexer = new IndexPack(db, is, packBase);
+			try {
+				indexer.index(new TextProgressMonitor());
+				fail("indexer should grumble about missing object");
+			} catch (IOException x) {
+				// expected
+			}
+		}
+		final InputStream is = new ByteArrayInputStream(os.toByteArray());
+		final IndexPack indexer = new IndexPack(db, is, packBase);
+		indexer.setFixThin(thin);
+		indexer.index(new TextProgressMonitor());
+		pack = new PackFile(db, indexFile, packFile);
+	}
+
+	private void verifyObjectsOrder(final ObjectId objectsOrder[]) {
+		final List<PackIndex.MutableEntry> entries = new ArrayList<PackIndex.MutableEntry>();
+
+		for (MutableEntry me : pack) {
+			entries.add(me.cloneEntry());
+		}
+		Collections.sort(entries, new Comparator<PackIndex.MutableEntry>() {
+			public int compare(MutableEntry o1, MutableEntry o2) {
+				return Long.signum(o1.getOffset() - o2.getOffset());
+			}
+		});
+
+		int i = 0;
+		for (MutableEntry me : entries) {
+			assertEquals(objectsOrder[i++], me.copy());
+		}
+	}
+}

-- 
1.5.5.1
Previous: Marek ZawirskiNext: Marek Zawirski
Message 21 of 29 in “PackWriter, first usable attempt”
  1. 00/20 PackWriter, first usable attemptMarek Zawirski, Jun 15, 2008
  2. 01/20 Fix typo in PackIndexV2Marek Zawirski, Jun 15, 2008
  3. 02/20 Integer versions of copyRawTo() and fromRaw() in ObjectIdMarek Zawirski, Jun 15, 2008
  4. 03/20 Add openObjectInAllPacks() to Repository, exposing packed objects storageMarek Zawirski, Jun 15, 2008
  5. 04/20 WindowedFile fragments copying: copyToStream()Marek Zawirski, Jun 15, 2008
  6. 05/20 Reverse pack index implementation: PackReverseIndexMarek Zawirski, Jun 15, 2008
  7. 06/20 Tests for PackReverseIndexMarek Zawirski, Jun 15, 2008
  8. 07/20 Refactor PackIndexV2 - extract binarySearchLevelTwo()Marek Zawirski, Jun 15, 2008
  9. 08/20 CRC32 support for PackIndexMarek Zawirski, Jun 15, 2008
  10. 09/20 CRC32 PackIndex testsMarek Zawirski, Jun 15, 2008
  11. 10/20 Format PackedObjectLoader classMarek Zawirski, Jun 15, 2008
  12. 11/20 Format UnpackedObjectLoader classMarek Zawirski, Jun 15, 2008
  13. 12/20 Format DeltaOfsPackedObjectLoader classMarek Zawirski, Jun 15, 2008
  14. 13/20 Raw-data operations in ObjectLoaders and PackFileMarek Zawirski, Jun 15, 2008
  15. 14/20 Add hasRevSort() in RevWalk for faster sorting strategy checkingMarek Zawirski, Jun 15, 2008
  16. 15/20 Refactor getRevSort() calls to hasRevSort()Marek Zawirski, Jun 15, 2008
  17. 16/20 Support for RevSort.BOUNDARY in ObjectWalkMarek Zawirski, Jun 15, 2008
  18. 17/20 Rename confusing objects field in ObjectWalkMarek Zawirski, Jun 15, 2008
  19. 18/20 New CountingOutputStream class - stream decoratorMarek Zawirski, Jun 15, 2008
  20. 19/20 Simplified implementation of pack creation: PackWriterMarek Zawirski, Jun 15, 2008
  21. 20/20 PackWriter test suiteMarek Zawirski, Jun 15, 2008
  22. 21/20 Make isBetterDeltaReuseLoader() static in PackWriterMarek Zawirski, Jun 17, 2008
  23. Robin RosenbergJun 17, 2008
  24. Marek ZawirskiJun 19, 2008
  25. Shawn O. PearceJun 16, 2008
  26. Marek ZawirskiJun 16, 2008
  27. Shawn O. PearceJun 17, 2008
  28. Shawn O. PearceJun 16, 2008
  29. Marek ZawirskiJun 16, 2008

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.