git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH] Add list-all-objects command

From
Charles Bailey <charles@hashpling.org>
Date
Jun 21, 2015, 19:20 UTC
Message-ID
<1434914431-7745-2-git-send-email-charles@hashpling.org>
In-Reply-To
<1434914431-7745-1-git-send-email-charles@hashpling.org>
From: Charles Bailey <cbailey32@bloomberg.net>

list-all-objects is a command to print the ids of all objects in the object database of a repository. It is designed as a low overhead interface for scripts that want to analyse all objects but don't require the ordering implied by a revision walk.

It will list all objects, loose and packed, and will include unreachable objects.

list-all-objects is faster that "rev-list --all --objects" but there is no guarantee as to the order in which objects will be listed.

Signed-off-by: Charles Bailey <cbailey32@bloomberg.net>
---
 Documentation/git-list-all-objects.txt | 29 +++++++++++++++
 Makefile                               |  1 +
 builtin.h                              |  1 +
 builtin/list-all-objects.c             | 64 ++++++++++++++++++++++++++++++++++
 git.c                                  |  1 +
 t/t8100-list-all-objects.sh            | 45 ++++++++++++++++++++++++
 6 files changed, 141 insertions(+)
 create mode 100644 Documentation/git-list-all-objects.txt
 create mode 100644 builtin/list-all-objects.c
 create mode 100755 t/t8100-list-all-objects.sh
diff --git a/Documentation/git-list-all-objects.txt b/Documentation/git-list-all-objects.txt
new file mode 100644
index 0000000..5f28d41
--- /dev/null
+++ b/Documentation/git-list-all-objects.txt
@@ -0,0 +1,29 @@
+git-list-all-objects(1)
+=======================
+
+NAME
+----
+git-list-all-objects - List all objects in the repository.
+
+SYNOPSIS
+--------
+[verse]
+'git list-all-objects' [-v|--verbose]
+
+DESCRIPTION
+-----------
+List the ids of all objects in a repository, including any unreachable objects.
+If `--verbose` is specified then the object's type and size is printed out as
+well as its id.
+
+OPTIONS
+-------
+
+-v::
+--verbose::
+	Output in the followin format instead of just printing object ids:
+	<sha1> SP <type> SP <size>
+
+GIT
+---
+Part of the linkgit:git[1] suite
diff --git a/Makefile b/Makefile
index 149f1c7..cf4f0c3 100644
--- a/Makefile
+++ b/Makefile
@@ -853,6 +853,7 @@ BUILTIN_OBJS += builtin/help.o
 BUILTIN_OBJS += builtin/index-pack.o
 BUILTIN_OBJS += builtin/init-db.o
 BUILTIN_OBJS += builtin/interpret-trailers.o
+BUILTIN_OBJS += builtin/list-all-objects.o
 BUILTIN_OBJS += builtin/log.o
 BUILTIN_OBJS += builtin/ls-files.o
 BUILTIN_OBJS += builtin/ls-remote.o
diff --git a/builtin.h b/builtin.h
index b87df70..112bafb 100644
--- a/builtin.h
+++ b/builtin.h
@@ -74,6 +74,7 @@ extern int cmd_help(int argc, const char **argv, const char *prefix);
 extern int cmd_index_pack(int argc, const char **argv, const char *prefix);
 extern int cmd_init_db(int argc, const char **argv, const char *prefix);
 extern int cmd_interpret_trailers(int argc, const char **argv, const char *prefix);
+extern int cmd_list_all_objects(int argc, const char **argv, const char *prefix);
 extern int cmd_log(int argc, const char **argv, const char *prefix);
 extern int cmd_log_reflog(int argc, const char **argv, const char *prefix);
 extern int cmd_ls_files(int argc, const char **argv, const char *prefix);
diff --git a/builtin/list-all-objects.c b/builtin/list-all-objects.c
new file mode 100644
index 0000000..3b43b02
--- /dev/null
+++ b/builtin/list-all-objects.c
@@ -0,0 +1,64 @@
+#include "cache.h"
+#include "builtin.h"
+#include "revision.h"
+#include "parse-options.h"
+
+#include <stdio.h>
+
+static int verbose;
+
+static int print_object(const unsigned char *sha1)
+{
+	if (verbose) {
+		unsigned long size;
+		int type = sha1_object_info(sha1, &size);
+
+		if (type < 0)
+			return -1;
+
+		printf("%s %s %lu\n", sha1_to_hex(sha1), typename(type), size);
+	}
+	else
+		printf("%s\n", sha1_to_hex(sha1));
+
+	return 0;
+}
+
+static int check_loose_object(const unsigned char *sha1,
+			      const char *path,
+			      void *data)
+{
+	return print_object(sha1);
+}
+
+static int check_packed_object(const unsigned char *sha1,
+			       struct packed_git *pack,
+			       uint32_t pos,
+			       void *data)
+{
+	return print_object(sha1);
+}
+
+static struct option builtin_filter_objects_options[] = {
+	OPT__VERBOSE(&verbose, "show object type and size"),
+	OPT_END()
+};
+
+int cmd_list_all_objects(int argc, const char **argv, const char *prefix)
+{
+	struct packed_git *p;
+
+	argc = parse_options(argc, argv, prefix, builtin_filter_objects_options,
+			     NULL, 0);
+
+	for_each_loose_object(check_loose_object, NULL, 0);
+
+	prepare_packed_git();
+	for (p = packed_git; p; p = p->next) {
+		open_pack_index(p);
+	}
+
+	for_each_packed_object(check_packed_object, NULL, 0);
+
+	return 0;
+}
diff --git a/git.c b/git.c
index 44374b1..81e8ae4 100644
--- a/git.c
+++ b/git.c
@@ -417,6 +417,7 @@ static struct cmd_struct commands[] = {
 	{ "init", cmd_init_db, NO_SETUP },
 	{ "init-db", cmd_init_db, NO_SETUP },
 	{ "interpret-trailers", cmd_interpret_trailers, RUN_SETUP },
+	{ "list-all-objects", cmd_list_all_objects, RUN_SETUP },
 	{ "log", cmd_log, RUN_SETUP },
 	{ "ls-files", cmd_ls_files, RUN_SETUP },
 	{ "ls-remote", cmd_ls_remote, RUN_SETUP_GENTLY },
diff --git a/t/t8100-list-all-objects.sh b/t/t8100-list-all-objects.sh
new file mode 100755
index 0000000..a7b51ce
--- /dev/null
+++ b/t/t8100-list-all-objects.sh
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+test_description='git list-all-objects'
+. ./test-lib.sh
+
+test_expect_success 'setup' '
+	echo hello, world >file &&
+	git add file &&
+	git commit -m "initial"
+'
+
+test_basic_repo_objects () {
+	git cat-file --batch-check="%(objectname)" <<-EOF >expected.unsorted &&
+		HEAD
+		HEAD:file
+		HEAD^{tree}
+	EOF
+	git list-all-objects >all-objects.unsorted &&
+	sort expected.unsorted >expected &&
+	sort all-objects.unsorted >all-objects &&
+	test_cmp all-objects expected
+}
+
+test_expect_success 'list all objects' '
+	test_basic_repo_objects
+'
+test_expect_success 'list all objects after pack' '
+	git repack -Ad &&
+	test_basic_repo_objects
+'
+
+test_expect_success 'verbose output' '
+	git cat-file --batch-check="%(objectname) %(objecttype) %(objectsize)" \
+			<<-EOF >expected.unsorted &&
+		HEAD
+		HEAD:file
+		HEAD^{tree}
+	EOF
+	git list-all-objects -v >all-objects.unsorted &&
+	sort expected.unsorted >expected &&
+	sort all-objects.unsorted >all-objects &&
+	test_cmp all-objects expected
+'
+
+test_done
-- 
2.4.0.53.g8440f74
Previous: Charles BaileyNext: Jeff King
Message 29 of 51 in “Improvements to parse-options and a new filter-objects command”
  1. Charles BaileyJun 19, 2015
  2. 1/3 Correct test-parse-options to handle negative intsCharles Bailey, Jun 19, 2015
  3. Junio C HamanoJun 19, 2015
  4. 2/3 Move unsigned long option parsing out of pack-objects.cCharles Bailey, Jun 19, 2015
  5. Remi Galan AlfonsoJun 19, 2015
  6. Charles BaileyJun 19, 2015
  7. Junio C HamanoJun 19, 2015
  8. Junio C HamanoJun 19, 2015
  9. Jakub NarębskiJun 20, 2015
  10. Jakub NarębskiJun 19, 2015
  11. Charles BaileyJun 20, 2015
  12. Junio C HamanoJun 20, 2015
  13. 3/3 Add filter-objects commandCharles Bailey, Jun 19, 2015
  14. Jeff KingJun 19, 2015
  15. Charles BaileyJun 19, 2015
  16. Jeff KingJun 19, 2015
  17. Junio C HamanoJun 19, 2015
  18. John KeepingJun 19, 2015
  19. Charles BaileyJun 19, 2015
  20. Improvements to integer option parsingCharles Bailey, Jun 21, 2015
  21. 1/2 Correct test-parse-options to handle negative intsCharles Bailey, Jun 21, 2015
  22. 2/2 Move unsigned long option parsing out of pack-objects.cCharles Bailey, Jun 21, 2015
  23. Charles BaileyJun 21, 2015
  24. Junio C HamanoJun 22, 2015
  25. Junio C HamanoJun 22, 2015
  26. Junio C HamanoJun 22, 2015
  27. Charles BaileyJun 22, 2015
  28. Fast enumeration of objectsCharles Bailey, Jun 21, 2015
  29. Add list-all-objects commandCharles Bailey, Jun 21, 2015
  30. Jeff KingJun 22, 2015
  31. Jeff KingJun 22, 2015
  32. 1/7 for_each_packed_object: automatically open pack indexJeff King, Jun 22, 2015
  33. 2/7 cat-file: minor style fix in options listJeff King, Jun 22, 2015
  34. 3/7 cat-file: move batch_options definition to top of fileJeff King, Jun 22, 2015
  35. 4/7 cat-file: add --buffer optionJeff King, Jun 22, 2015
  36. 5/7 cat-file: stop returning value from batch_one_objectJeff King, Jun 22, 2015
  37. 6/7 cat-file: split batch_one_object into two stagesJeff King, Jun 22, 2015
  38. 7/7 cat-file: add --batch-all-objects optionJeff King, Jun 22, 2015
  39. Eric SunshineJun 26, 2015
  40. Jeff KingJun 26, 2015
  41. 8/7 cat-file: sort and de-dup output of --batch-all-objectsJeff King, Jun 22, 2015
  42. Charles BaileyJun 22, 2015
  43. Jeff KingJun 22, 2015
  44. Charles BaileyJun 22, 2015
  45. Junio C HamanoJun 22, 2015
  46. Jeff KingJun 22, 2015
  47. Charles BaileyJun 22, 2015
  48. Duy NguyenJun 22, 2015
  49. Jeff KingJun 22, 2015
  50. Jeff KingJun 22, 2015
  51. Junio C HamanoJun 22, 2015

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.