git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 08/19] rev-list: add object filtering support

From
JHJeff Hostetler <git@jeffhostetler.com>
Date
Jul 13, 2017, 17:34 UTC
Message-ID
<20170713173459.3559-9-git@jeffhostetler.com>
In-Reply-To
<20170713173459.3559-1-git@jeffhostetler.com>
From: Jeff Hostetler <jeffhost@microsoft.com>

Teach rev-list to use the filtering provided by the traverse_commit_list_filtered() interface to omit unwanted objects from the result.

This feature is only enabled when one of the "--objects*" options are used.

When the "--filter-print-manifest" option is used, the omitted objects and their sizes are printed at the end. These are marked with a "~". This can be combined with "--quiet" to get a list of just the omitted objects.

Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com>
---
 builtin/rev-list.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 56 insertions(+), 2 deletions(-)
diff --git a/builtin/rev-list.c b/builtin/rev-list.c
index bcf77f0..fd9a7e5 100644
--- a/builtin/rev-list.c
+++ b/builtin/rev-list.c
@@ -3,6 +3,8 @@
 #include "diff.h"
 #include "revision.h"
 #include "list-objects.h"
+#include "list-objects-filters.h"
+#include "object-filter.h"
 #include "pack.h"
 #include "pack-bitmap.h"
 #include "builtin.h"
@@ -52,6 +54,7 @@ static const char rev_list_usage[] =
 
 static struct progress *progress;
 static unsigned progress_counter;
+static struct object_filter_options filter_options;
 
 static void finish_commit(struct commit *commit, void *data);
 static void show_commit(struct commit *commit, void *data)
@@ -178,8 +181,20 @@ static void finish_commit(struct commit *commit, void *data)
 static void finish_object(struct object *obj, const char *name, void *cb_data)
 {
 	struct rev_list_info *info = cb_data;
-	if (obj->type == OBJ_BLOB && !has_object_file(&obj->oid))
+	if (obj->type == OBJ_BLOB && !has_object_file(&obj->oid)) {
+		if (filter_options.relax) {
+			/*
+			 * Relax consistency checks to not complain about
+			 * omitted objects (presumably caused by use of
+			 * the previous use of the 'filter-objects' feature).
+			 *
+			 * Note that this is independent of any filtering that
+			 * we are doing in this run.
+			 */
+			return;
+		}
 		die("missing blob object '%s'", oid_to_hex(&obj->oid));
+	}
 	if (info->revs->verify_objects && !obj->parsed && obj->type != OBJ_COMMIT)
 		parse_object(obj->oid.hash);
 }
@@ -199,6 +214,16 @@ static void show_edge(struct commit *commit)
 	printf("-%s\n", oid_to_hex(&commit->object.oid));
 }
 
+static void print_omitted_object(int i, int i_limit, struct oidset2_entry *e, void *cb_data)
+{
+	/* struct rev_list_info *info = cb_data; */
+
+	if (e->object_length == -1)
+		printf("~%s\n", oid_to_hex(&e->oid));
+	else
+		printf("~%s %"PRIuMAX"\n", oid_to_hex(&e->oid), e->object_length);
+}
+
 static void print_var_str(const char *var, const char *val)
 {
 	printf("%s='%s'\n", var, val);
@@ -276,6 +301,7 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
 	int bisect_find_all = 0;
 	int use_bitmap_index = 0;
 	const char *show_progress = NULL;
+	oidset2_foreach_cb fn_filter_print = NULL;
 
 	git_config(git_default_config, NULL);
 	init_revisions(&revs, prefix);
@@ -329,6 +355,14 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
 			show_progress = arg;
 			continue;
 		}
+		if (object_filter_hand_parse_arg(&filter_options, arg, 1, 1)) {
+			if (!revs.blob_objects)
+				die(_("object filtering requires --objects"));
+			if (filter_options.use_sparse &&
+			    !oidcmp(&filter_options.sparse_oid, &null_oid))
+				die(_("invalid sparse value"));
+			continue;
+		}
 		usage(rev_list_usage);
 
 	}
@@ -353,6 +387,11 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
 	if (revs.show_notes)
 		die(_("rev-list does not support display of notes"));
 
+	if (object_filter_enabled(&filter_options)) {
+		if (use_bitmap_index)
+			die(_("cannot combine --use-bitmap-index with object filtering"));
+	}
+
 	save_commit_buffer = (revs.verbose_header ||
 			      revs.grep_filter.pattern_list ||
 			      revs.grep_filter.header_list);
@@ -397,7 +436,22 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
 			return show_bisect_vars(&info, reaches, all);
 	}
 
-	traverse_commit_list(&revs, show_commit, show_object, &info);
+	if (filter_options.print_manifest)
+		fn_filter_print = print_omitted_object;
+
+	if (filter_options.omit_all_blobs)
+		traverse_commit_list_omit_all_blobs(
+			&revs, show_commit, show_object, fn_filter_print, &info);
+	else if (filter_options.omit_large_blobs)
+		traverse_commit_list_omit_large_blobs(
+			&revs, show_commit, show_object, fn_filter_print, &info,
+			(int64_t)(uint64_t)filter_options.large_byte_limit);
+	else if (filter_options.use_sparse)
+		traverse_commit_list_use_sparse(
+			&revs, show_commit, show_object, fn_filter_print, &info,
+			&filter_options.sparse_oid);
+	else
+		traverse_commit_list(&revs, show_commit, show_object, &info);
 
 	stop_progress(&progress);
 

-- 
2.9.3
Previous: Jeff HostetlerNext: Jeff Hostetler
Message 16 of 20 in “WIP object filtering for partial clone”
  1. 00/19 WIP object filtering for partial cloneJeff Hostetler, Jul 13, 2017
  2. 01/19 dir: refactor add_excludes()Jeff Hostetler, Jul 13, 2017
  3. 03/19 list-objects: filter objects in traverse_commit_listJeff Hostetler, Jul 13, 2017
  4. 06/19 list-objects-filters: add use-sparse-checkout filterJeff Hostetler, Jul 13, 2017
  5. 07/19 object-filter: common declarations for object filteringJeff Hostetler, Jul 13, 2017
  6. 11/19 pack-objects: add object filtering supportJeff Hostetler, Jul 13, 2017
  7. 10/19 t6112: rev-list object filtering testJeff Hostetler, Jul 13, 2017
  8. 13/19 upload-pack: add filter-objects to protocol documentationJeff Hostetler, Jul 13, 2017
  9. 17/19 clone: add filter argumentsJeff Hostetler, Jul 13, 2017
  10. 18/19 index-pack: relax consistency checks for omitted objectsJeff Hostetler, Jul 13, 2017
  11. 12/19 pack-objects: add filtering help textJeff Hostetler, Jul 13, 2017
  12. 14/19 upload-pack: add object filteringJeff Hostetler, Jul 13, 2017
  13. 15/19 fetch-pack: add object filtering supportJeff Hostetler, Jul 13, 2017
  14. 16/19 connected: add filter_allow_omitted option to APIJeff Hostetler, Jul 13, 2017
  15. 19/19 fetch: add object filtering to fetchJeff Hostetler, Jul 13, 2017
  16. 08/19 rev-list: add object filtering supportJeff Hostetler, Jul 13, 2017
  17. 09/19 rev-list: add filtering help textJeff Hostetler, Jul 13, 2017
  18. 02/19 oidset2: create oidset subclass with object length and pathnameJeff Hostetler, Jul 13, 2017
  19. 05/19 list-objects-filters: add omit-large-blobs filterJeff Hostetler, Jul 13, 2017
  20. 04/19 list-objects-filters: add omit-all-blobs filterJeff Hostetler, Jul 13, 2017

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.