git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[JGIT PATCH 09/10] Add support for writing pack index v2 files

From
Shawn O. Pearce <spearce@spearce.org>
Date
Jun 24, 2008, 02:10 UTC
Message-ID
<1214273408-70793-10-git-send-email-spearce@spearce.org>
In-Reply-To
<1214273408-70793-9-git-send-email-spearce@spearce.org>

The v2 format is more robust for delta reuse as it has a CRC element that covers the entire packed representation, permitting more efficient delta-reuse during packing. It also can address objects in pack files larger than 4 GB, making it a better format for the future.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 .../src/org/spearce/jgit/lib/PackIndexWriter.java  |   21 ++++
 .../org/spearce/jgit/lib/PackIndexWriterV2.java    |  101 ++++++++++++++++++++
 .../src/org/spearce/jgit/pgm/IndexPack.java        |    4 +
 .../src/org/spearce/jgit/transport/IndexPack.java  |   22 ++++-
 4 files changed, 146 insertions(+), 2 deletions(-)
 create mode 100644 org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexWriterV2.java
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexWriter.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexWriter.java
index c9b27d2..2d9d822 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexWriter.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexWriter.java
@@ -122,6 +122,8 @@ public abstract class PackIndexWriter {
 		switch (version) {
 		case 1:
 			return new PackIndexWriterV1(dst);
+		case 2:
+			return new PackIndexWriterV2(dst);
 		default:
 			throw new IllegalArgumentException(
 					"Unsupported pack index version " + version);
@@ -203,6 +205,25 @@ public abstract class PackIndexWriter {
 	protected abstract void writeImpl() throws IOException;
 
 	/**
+	 * Output the version 2 (and later) TOC header, with version number.
+	 * <p>
+	 * Post version 1 all index files start with a TOC header that makes the
+	 * file an invalid version 1 file, and then includes the version number.
+	 * This header is necessary to recognize a version 1 from a version 2
+	 * formatted index.
+	 * 
+	 * @param version
+	 *            version number of this index format being written.
+	 * @throws IOException
+	 *             an error occurred while writing to the output stream.
+	 */
+	protected void writeTOC(final int version) throws IOException {
+		out.write(TOC);
+		NB.encodeInt32(tmp, 0, version);
+		out.write(tmp, 0, 4);
+	}
+
+	/**
 	 * Output the standard 256 entry first-level fan-out table.
 	 * <p>
 	 * The fan-out table is 4 KB in size, holding 256 32-bit unsigned integer
diff --git a/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexWriterV2.java b/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexWriterV2.java
new file mode 100644
index 0000000..8fa4d1a
--- /dev/null
+++ b/org.spearce.jgit/src/org/spearce/jgit/lib/PackIndexWriterV2.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Git Development Community nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.spearce.jgit.lib;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.spearce.jgit.transport.PackedObjectInfo;
+import org.spearce.jgit.util.NB;
+
+/**
+ * Creates the version 2 pack table of contents files.
+ * 
+ * @see PackIndexWriter
+ * @see PackIndexV2
+ */
+class PackIndexWriterV2 extends PackIndexWriter {
+	PackIndexWriterV2(final OutputStream dst) {
+		super(dst);
+	}
+
+	@Override
+	protected void writeImpl() throws IOException {
+		writeTOC(2);
+		writeFanOutTable();
+		writeObjectNames();
+		writeCRCs();
+		writeOffset32();
+		writeOffset64();
+		writeChecksumFooter();
+	}
+
+	private void writeObjectNames() throws IOException {
+		for (final PackedObjectInfo oe : entries)
+			oe.copyRawTo(out);
+	}
+
+	private void writeCRCs() throws IOException {
+		for (final PackedObjectInfo oe : entries) {
+			NB.encodeInt32(tmp, 0, oe.getCRC());
+			out.write(tmp, 0, 4);
+		}
+	}
+
+	private void writeOffset32() throws IOException {
+		int o64 = 0;
+		for (final PackedObjectInfo oe : entries) {
+			final long o = oe.getOffset();
+			if (o < Integer.MAX_VALUE)
+				NB.encodeInt32(tmp, 0, (int) o);
+			else
+				NB.encodeInt32(tmp, 0, (1 << 31) | o64++);
+			out.write(tmp, 0, 4);
+		}
+	}
+
+	private void writeOffset64() throws IOException {
+		for (final PackedObjectInfo oe : entries) {
+			final long o = oe.getOffset();
+			if (o > Integer.MAX_VALUE) {
+				NB.encodeInt64(tmp, 0, o);
+				out.write(tmp, 0, 8);
+			}
+		}
+	}
+}
diff --git a/org.spearce.jgit/src/org/spearce/jgit/pgm/IndexPack.java b/org.spearce.jgit/src/org/spearce/jgit/pgm/IndexPack.java
index 5a82a35..60926c1 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/pgm/IndexPack.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/pgm/IndexPack.java
@@ -47,10 +47,13 @@ class IndexPack extends TextBuiltin {
 	void execute(final String[] args) throws Exception {
 		boolean fixThin = false;
 		int argi = 0;
+		int version = 0;
 		for (; argi < args.length; argi++) {
 			final String a = args[argi];
 			if ("--fix-thin".equals(a))
 				fixThin = true;
+			else if (a.startsWith("--index-version="))
+				version = Integer.parseInt(a.substring(a.indexOf('=') + 1));
 			else if ("--".equals(a)) {
 				argi++;
 				break;
@@ -69,6 +72,7 @@ class IndexPack extends TextBuiltin {
 		in = new BufferedInputStream(System.in);
 		ip = new org.spearce.jgit.transport.IndexPack(db, in, base);
 		ip.setFixThin(fixThin);
+		ip.setIndexVersion(version);
 		ip.index(new TextProgressMonitor());
 	}
 }
diff --git a/org.spearce.jgit/src/org/spearce/jgit/transport/IndexPack.java b/org.spearce.jgit/src/org/spearce/jgit/transport/IndexPack.java
index 047f0dc..06ef7cc 100644
--- a/org.spearce.jgit/src/org/spearce/jgit/transport/IndexPack.java
+++ b/org.spearce.jgit/src/org/spearce/jgit/transport/IndexPack.java
@@ -125,6 +125,8 @@ public class IndexPack {
 
 	private boolean fixThin;
 
+	private int outputVersion;
+
 	private final File dstPack;
 
 	private final File dstIdx;
@@ -185,6 +187,18 @@ public class IndexPack {
 	}
 
 	/**
+	 * Set the pack index file format version this instance will create.
+	 * 
+	 * @param version
+	 *            the version to write. The special version 0 designates the
+	 *            oldest (most compatible) format available for the objects.
+	 * @see PackIndexWriter
+	 */
+	public void setIndexVersion(final int version) {
+		outputVersion = version;
+	}
+
+	/**
 	 * Configure this index pack instance to make a thin pack complete.
 	 * <p>
 	 * Thin packs are sometimes used during network transfers to allow a delta
@@ -466,8 +480,12 @@ public class IndexPack {
 
 		final FileOutputStream os = new FileOutputStream(dstIdx);
 		try {
-			PackIndexWriter.createOldestPossible(os, list)
-					.write(list, packcsum);
+			final PackIndexWriter iw;
+			if (outputVersion <= 0)
+				iw = PackIndexWriter.createOldestPossible(os, list);
+			else
+				iw = PackIndexWriter.createVersion(os, outputVersion);
+			iw.write(list, packcsum);
 		} finally {
 			os.close();
 		}

-- 
1.5.6.74.g8a5e
Previous: Shawn O. PearceNext: Shawn O. Pearce
Message 10 of 14 in “Support writing pack index version 2”
  1. 00/10 Support writing pack index version 2Shawn O. Pearce, Jun 24, 2008
  2. 01/10 Extract inner ObjectEntry from IndexPack classShawn O. Pearce, Jun 24, 2008
  3. 02/10 Make ObjectEntry's position field privateShawn O. Pearce, Jun 24, 2008
  4. 03/10 Rename ObjectEntry to PackedObjectInfoShawn O. Pearce, Jun 24, 2008
  5. 04/10 Document PackedObjectInfo and make it public for reuseShawn O. Pearce, Jun 24, 2008
  6. 05/10 Refactor pack index writing to a common APIShawn O. Pearce, Jun 24, 2008
  7. 06/10 Reuse the magic tOc constant for pack index headersShawn O. Pearce, Jun 24, 2008
  8. 07/10 Add 64 bit network byte order encoding to NBShawn O. Pearce, Jun 24, 2008
  9. 08/10 Compute packed object entry CRC32 data during IndexPackShawn O. Pearce, Jun 24, 2008
  10. 09/10 Add support for writing pack index v2 filesShawn O. Pearce, Jun 24, 2008
  11. 10/10 Default IndexPack to honor pack.indexversion configurationShawn O. Pearce, Jun 24, 2008
  12. 06/10 Reuse the magic tOc constant for pack index headersShawn O. Pearce, Jun 25, 2008
  13. Robin RosenbergJun 24, 2008
  14. Shawn O. PearceJun 25, 2008

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.