threads / patch / 10948

patchAdd 'git fast-export', the sister of 'git fast-import'

Subject: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

## tl;dr

27 messages between Nov 21, 2007 and Nov 27, 2007. Diffs are folded; open one to read it.

replies: 26people: 9as markdown or json

Johannes Schindelin· Nov 21, 2007, 03:40 UTC · lore
[WIP: this does not handle tags yet, and it lacks a test script
 as well as documentation.]

This program dumps (parts of) a git repository in the format that fast-import understands.

For clarity's sake, it does not use the 'inline' method of specifying blobs in the commits, but builds the blobs before building the commits.B

---
	I am way too tired now to continue, but maybe someone else wants
	to pick up the ball.
	Oh, and it relies on "int" being castable to void * and vice 
	versa.  Is anybody aware of a platform where this can lead to
	problems?
	And yes, I will add a copyright when I woke up again.
 .gitignore            |    1 +
 Makefile              |    1 +
 builtin-fast-export.c |  202 +++++++++++++++++++++++++++++++++++++++++++++++++
 builtin.h             |    1 +
 git.c                 |    1 +
 5 files changed, 206 insertions(+), 0 deletions(-)
 create mode 100755 builtin-fast-export.c
Show changes to 5 files +206 −0

.gitignore, Makefile, builtin-fast-export.c, builtin.h, git.c

diff --git a/.gitignore b/.gitignore
index 507e351..ef69e2f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ git-diff-files
 git-diff-index
 git-diff-tree
 git-describe
+git-fast-export
 git-fast-import
 git-fetch
 git-fetch--tool
diff --git a/Makefile b/Makefile
index dd3ba6f..8784eff 100644
--- a/Makefile
+++ b/Makefile
@@ -336,6 +336,7 @@ BUILTIN_OBJS = \
 	builtin-diff-files.o \
 	builtin-diff-index.o \
 	builtin-diff-tree.o \
+	builtin-fast-export.o \
 	builtin-fetch.o \
 	builtin-fetch-pack.o \
 	builtin-fetch--tool.o \
diff --git a/builtin-fast-export.c b/builtin-fast-export.c
new file mode 100755
index 0000000..2e8e3dd
--- /dev/null
+++ b/builtin-fast-export.c
@@ -0,0 +1,202 @@
+#include "builtin.h"
+#include "cache.h"
+#include "commit.h"
+#include "object.h"
+#include "tag.h"
+#include "diff.h"
+#include "diffcore.h"
+#include "log-tree.h"
+#include "revision.h"
+#include "decorate.h"
+#include "path-list.h"
+
+static struct decoration idnums;
+static uint32_t last_idnum;
+
+static const char *fast_export_usage = "git-fast-export [rev-list-opts]";
+
+static int has_unshown_parent(struct commit *commit)
+{
+	struct commit_list *parent;
+
+	for (parent = commit->parents; parent; parent = parent->next)
+		if (!(parent->item->object.flags & SHOWN))
+			return 1;
+	return 0;
+}
+
+static void handle_object(const unsigned char *sha1)
+{
+	unsigned long size;
+	enum object_type type;
+	char *buf;
+	struct object *object;
+
+	if (is_null_sha1(sha1))
+		return;
+
+	object = parse_object(sha1);
+	if (!object)
+		die ("Could not read blob %s", sha1_to_hex(sha1));
+
+	if (object->flags & SHOWN)
+		return;
+
+	buf = read_sha1_file(sha1, &type, &size);
+	if (!buf)
+		die ("Could not read blob %s", sha1_to_hex(sha1));
+
+	last_idnum++;
+	add_decoration(&idnums, object, (void *)last_idnum);
+
+	printf("blob\nmark :%d\ndata %lu\n", last_idnum, size);
+	if (fwrite(buf, size, 1, stdout) != 1)
+		die ("Could not write blob %s", sha1_to_hex(sha1));
+	printf("\n");
+
+	object->flags |= SHOWN;
+	free(buf);
+}
+
+static void show_filemodify(struct diff_queue_struct *q,
+		struct diff_options *options, void *data)
+{
+	int i;
+	for (i = 0; i < q->nr; i++) {
+		struct diff_filespec *spec = q->queue[i]->two;
+		if (is_null_sha1(spec->sha1))
+			printf("D %s\n", spec->path);
+		else {
+			struct object *object = lookup_object(spec->sha1);
+			printf("M 0%06o :%d %s\n", spec->mode,
+					(int)lookup_decoration(&idnums, object),
+					spec->path);
+		}
+	}
+}
+
+static void handle_commit(struct commit *commit, struct rev_info *rev)
+{
+	const char *author, *author_end, *committer, *committer_end, *message;
+	int saved_output_format = rev->diffopt.output_format;
+	int i;
+
+	rev->diffopt.output_format = DIFF_FORMAT_CALLBACK;
+
+	parse_commit(commit);
+	author = strstr(commit->buffer, "\nauthor ");
+	if (!author)
+		die ("Could not find author in commit %s",
+				sha1_to_hex(commit->object.sha1));
+	author++;
+	author_end = strchrnul(author, '\n');
+	committer = strstr(author_end, "\ncommitter ");
+	if (!committer)
+		die ("Could not find committer in commit %s",
+				sha1_to_hex(commit->object.sha1));
+	committer++;
+	committer_end = strchrnul(committer, '\n');
+	message = strstr(committer_end, "\n\n");
+	if (message)
+		message += 2;
+
+	if (commit->parents) {
+		parse_commit(commit->parents->item);
+		diff_tree_sha1(commit->parents->item->tree->object.sha1,
+				commit->tree->object.sha1, "", &rev->diffopt);
+	}
+	else
+		diff_root_tree_sha1(commit->tree->object.sha1,
+				"", &rev->diffopt);
+
+	for (i = 0; i < diff_queued_diff.nr; i++)
+		handle_object(diff_queued_diff.queue[i]->two->sha1);
+
+	last_idnum++;
+	add_decoration(&idnums, &commit->object, (void *)last_idnum);
+	printf("commit %s\nmark :%d\n%.*s\n%.*s\ndata %u\n%s",
+		(const char *)commit->util, last_idnum,
+		author_end - author, author,
+		committer_end - committer, committer,
+		message ? strlen(message) : 0, message ? message : "");
+
+	log_tree_diff_flush(rev);
+	rev->diffopt.output_format = saved_output_format;
+
+	printf("\n");
+}
+
+static void handle_tail(struct object_array *commits, struct rev_info *revs)
+{
+	struct commit *commit;
+	while (commits->nr) {
+		commit = (struct commit *)commits->objects[commits->nr - 1].item;
+		if (has_unshown_parent(commit))
+			return;
+		handle_commit(commit, revs);
+		commits->nr--;
+	}
+}
+
+int cmd_fast_export(int argc, const char **argv, const char *prefix)
+{
+	struct rev_info revs;
+	struct object_array commits = { 0, 0, NULL };
+	struct path_list extra_refs = { NULL, 0, 0, 0 };
+	struct commit *commit;
+	int i;
+
+	init_revisions(&revs, prefix);
+	argc = setup_revisions(argc, argv, &revs, NULL);
+	if (argc > 1)
+		usage (fast_export_usage);
+
+	for (i = 0; i < revs.pending.nr; i++) {
+		struct object_array_entry *e = revs.pending.objects + i;
+		unsigned char sha1[20];
+		char *full_name;
+
+		/* TODO: handle tags */
+		commit = (struct commit *)e->item;
+		if (dwim_ref(e->name, strlen(e->name), sha1, &full_name) != 1)
+			die ("Could not get a unique ref name for '%s'",
+				e->name);
+		if (commit->util)
+			/* more than one name for the same object */
+			path_list_insert(full_name, &extra_refs)->util = commit;
+		else
+			commit->util = full_name;
+	}
+
+	prepare_revision_walk(&revs);
+	revs.diffopt.format_callback = show_filemodify;
+	revs.diffopt.recursive = 1;
+	while ((commit = get_revision(&revs))) {
+		if (has_unshown_parent(commit)) {
+			struct commit_list *parent = commit->parents;
+			add_object_array(&commit->object, NULL, &commits);
+			for (; parent; parent = parent->next)
+				if (!parent->item->util)
+					parent->item->util = commit->util;
+		}
+		else {
+			handle_commit(commit, &revs);
+			handle_tail(&commits, &revs);
+		}
+	}
+
+	if (commits.nr)
+		die ("This should not happen: there were %d commits left!",
+			commits.nr);
+
+	for (i = 0; i < extra_refs.nr; i++) {
+		/* create refs pointing to already seen commits */
+		commit = extra_refs.items[i].util;
+		printf("reset %s\nfrom :%d\n\n",
+				extra_refs.items[i].path,
+				(int)lookup_decoration(&idnums,
+					&commit->object));
+	}
+
+	return 0;
+}
diff --git a/builtin.h b/builtin.h
index e76f5da..5723f9e 100644
--- a/builtin.h
+++ b/builtin.h
@@ -31,6 +31,7 @@ extern int cmd_diff_files(int argc, const char **argv, const char *prefix);
 extern int cmd_diff_index(int argc, const char **argv, const char *prefix);
 extern int cmd_diff(int argc, const char **argv, const char *prefix);
 extern int cmd_diff_tree(int argc, const char **argv, const char *prefix);
+extern int cmd_fast_export(int argc, const char **argv, const char *prefix);
 extern int cmd_fetch(int argc, const char **argv, const char *prefix);
 extern int cmd_fetch_pack(int argc, const char **argv, const char *prefix);
 extern int cmd_fetch__tool(int argc, const char **argv, const char *prefix);
diff --git a/git.c b/git.c
index 0b6825e..deda6dd 100644
--- a/git.c
+++ b/git.c
@@ -301,6 +301,7 @@ static void handle_internal_command(int argc, const char **argv)
 		{ "diff-files", cmd_diff_files },
 		{ "diff-index", cmd_diff_index, RUN_SETUP },
 		{ "diff-tree", cmd_diff_tree, RUN_SETUP },
+		{ "fast-export", cmd_fast_export, RUN_SETUP },
 		{ "fetch", cmd_fetch, RUN_SETUP },
 		{ "fetch-pack", cmd_fetch_pack, RUN_SETUP },
 		{ "fetch--tool", cmd_fetch__tool, RUN_SETUP },
-- 
1.5.3.5.1751.gf6fb4
Johannes Sixt· Nov 21, 2007, 07:44 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Johannes Schindelin schrieb:
> 	Oh, and it relies on "int" being castable to void * and vice 
> 	versa.  Is anybody aware of a platform where this can lead to
> 	problems?
Win64?
-- Hannes
Shawn O. Pearce· Nov 21, 2007, 07:47 UTC · re: Johannes Sixt · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Johannes Sixt <j.sixt@viscovery.net> wrote:
Show 6 quoted lines
> Johannes Schindelin schrieb:
> >	Oh, and it relies on "int" being castable to void * and vice 
> >	versa.  Is anybody aware of a platform where this can lead to
> >	problems?
> 
> Win64?
Does anyone even use that?  I thought that was dead.  Like Vista.

Seriously though, we include stdint.h. If you want to cast a void* to an integer type and back use ptrint_t. That's what it exists for.

-- 
Shawn.
Johannes Schindelin· Nov 21, 2007, 14:01 UTC · re: Johannes Sixt · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
On Wed, 21 Nov 2007, Johannes Sixt wrote:
Show 5 quoted lines
> Johannes Schindelin schrieb:
> > 	Oh, and it relies on "int" being castable to void * and vice versa.
> >     Is anybody aware of a platform where this can lead to problems?
> 
> Win64?
Is this really a problem?  I mean, all I need is that i == (int)(void *)i.

My doubts came from platforms where you can access memory only aligned to 4-byte chunks, and that it _may_ be possible that some of them do not even store the least significant two bits.

Ciao, Dscho

P.S.: I'll try to read up on ptrint_t, as suggested by Shawn.
Andreas Ericsson· Nov 21, 2007, 15:09 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Johannes Schindelin wrote:
> 
> P.S.: I'll try to read up on ptrint_t, as suggested by Shawn.

I can't find a reference to ptrint_t anywhere on either my system, and the 10 first hits on Google finds it on typedef lines, most commonly like this:

	typedef unsigned long ptrint_t;

I think ptrdiff_t is more portable. Here's the relevant entry about ptrdiff_t from /usr/include/obstack.h on my system:

---%<---%<---%<---
/* We need the type of a pointer subtraction.  If __PTRDIFF_TYPE__ is
   defined, as with GNU C, use that; that way we don't pollute the
   namespace with <stddef.h>'s symbols.  Otherwise, include <stddef.h>
   and use ptrdiff_t.  */

#ifdef __PTRDIFF_TYPE__ # define PTR_INT_TYPE __PTRDIFF_TYPE__ #else # include <stddef.h> # define PTR_INT_TYPE ptrdiff_t #endif ---%<---%<---%<---

MySQL seems to have botched that one though. It has this instead:
	typedef long          my_ptrdiff_t;
-- 
Andreas Ericsson                   andreas.ericsson@op5.se
OP5 AB                             www.op5.se
Tel: +46 8-230225                  Fax: +46 8-230231
Johannes Schindelin· Nov 21, 2007, 15:47 UTC · re: Andreas Ericsson · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
On Wed, 21 Nov 2007, Andreas Ericsson wrote:
Show 9 quoted lines
> Johannes Schindelin wrote:
> > 
> > P.S.: I'll try to read up on ptrint_t, as suggested by Shawn.
> 
> I can't find a reference to ptrint_t anywhere on either my system,
> and the 10 first hits on Google finds it on typedef lines, most
> commonly like this:
> 
> 	typedef unsigned long ptrint_t;
It is really called intptr_t, and it is C99.

Ciao, Dscho

Andreas Ericsson· Nov 21, 2007, 15:53 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Johannes Schindelin wrote:
Show 14 quoted lines
> Hi,
> 
> On Wed, 21 Nov 2007, Andreas Ericsson wrote:
> 
>> Johannes Schindelin wrote:
>>> P.S.: I'll try to read up on ptrint_t, as suggested by Shawn.
>> I can't find a reference to ptrint_t anywhere on either my system,
>> and the 10 first hits on Google finds it on typedef lines, most
>> commonly like this:
>>
>> 	typedef unsigned long ptrint_t;
> 
> It is really called intptr_t, and it is C99.
> 
That'd explain it I guess. Thanks for the clarification.
-- 
Andreas Ericsson                   andreas.ericsson@op5.se
OP5 AB                             www.op5.se
Tel: +46 8-230225                  Fax: +46 8-230231
Geert Bosch· Nov 21, 2007, 12:43 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

On Nov 20, 2007, at 22:40, Johannes Schindelin wrote:
> 	Oh, and it relies on "int" being castable to void * and vice
> 	versa.  Is anybody aware of a platform where this can lead to
> 	problems?

Like any 64-bit system? int is 32 bit and void * is 64. So, casting a pointer to an int drops half the bits. When you convert it back it will point to a funny place...

   -Geert
Johannes Schindelin· Nov 21, 2007, 14:42 UTC · re: Geert Bosch · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
On Wed, 21 Nov 2007, Geert Bosch wrote:
Show 8 quoted lines
> On Nov 20, 2007, at 22:40, Johannes Schindelin wrote:
> > 	Oh, and it relies on "int" being castable to void * and vice
> > 	versa.  Is anybody aware of a platform where this can lead to
> > 	problems?
> 
> Like any 64-bit system? int is 32 bit and void * is 64. So, casting a 
> pointer to an int drops half the bits. When you convert it back it will 
> point to a funny place...

Have you even read the patch? Like I pointed out in my reply to Hannes, I only need the guarantee that

	i == (int)(void *)i
for all integers i.

Ciao, Dscho

Han-Wen Nienhuys· Nov 23, 2007, 00:27 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Johannes Schindelin escreveu:
Show 18 quoted lines
> [WIP: this does not handle tags yet, and it lacks a test script
>  as well as documentation.]
> 
> This program dumps (parts of) a git repository in the format that
> fast-import understands.
> 
> For clarity's sake, it does not use the 'inline' method of specifying
> blobs in the commits, but builds the blobs before building the commits.B
> 
> ---
> 	I am way too tired now to continue, but maybe someone else wants
> 	to pick up the ball.
> 
> 	Oh, and it relies on "int" being castable to void * and vice 
> 	versa.  Is anybody aware of a platform where this can lead to
> 	problems?
> 
> 	And yes, I will add a copyright when I woke up again.

This one seems to setup a dump of a single branch from the command line, which then follows the commit structure. Am I missing something?

The cool thing about git-fast-import is that it reads from stdin, has a very easy to use programmatic interface, and does not impose any order on how you enter the information.

This doesn't seem to be mirrored by this script? 

I am working on a script for [company] which uses git. Git is a pain to script for: for every query I need to invoke another git process, with another command (log, show-ref, cat-file, show, etc.), parse another output format and/or specify another --pretty=format:%blah format.

Besides being a nuisance, I actually run git on NFS, and every git process has to go to NFS a couple times to retrieve the same information. This has a noticeable performance impact.

It would make my life a lot easier if I could simply open a pipe to a single process for the duration of the script, and do all my queries to this one process. Of course, if the repository is changed by another process, I would have to restart it, but that's manageable. I could even write a nice Python class that runs both fast-import and fast-export. I could then have an efficient Python interface to a git-repository, without needing any library wrapping.

--
 Han-Wen Nienhuys - hanwen@xs4all.nl - http://www.xs4all.nl/~hanwen
Johannes Schindelin· Nov 23, 2007, 01:01 UTC · re: Han-Wen Nienhuys · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
[please Cc me when you reply to my message]
On Thu, 22 Nov 2007, Han-Wen Nienhuys wrote:
> This one seems to setup a dump of a single branch from the command line, 
> which then follows the commit structure.  Am I missing something?

Yes. It should work with "--all", too. In fact, with every rev-list parameters, even a..b (which would cut off the history up to -- and including -- a).

Show 5 quoted lines
> The cool thing about git-fast-import is that it reads from stdin, has a 
> very easy to use programmatic interface, and does not impose any order 
> on how you enter the information.
> 
> This doesn't seem to be mirrored by this script? 

Umm. What exactly do you want to reorder? I mean, this program is meant to dump the complete contents to stdout (whether you redirect it to a file or edit it with sed does not concern this program). It does that.

Maybe you want to specify if all blobs should be output first, and then the commits? Or files should be used? But all of these things seem to be useless to me.

So I am puzzled what you ask for.
> I am working on a script for [company] which uses git.  Git is a pain to 
> script for: for every query I need to invoke another git process, with 
> another command (log, show-ref, cat-file, show, etc.), parse another 
> output format and/or specify another --pretty=format:%blah format.

Now I am really puzzled. Git is one of the most easily scriptable programs I ever saw.

It does not even force you to use certain combinations of scripting languages, such as Python, Scheme, C++ and PostScript, separated by which part of the program you want to script.

> Besides being a nuisance, I actually run git on NFS, and every git 
> process has to go to NFS a couple times to retrieve the same 
> information. This has a noticeable performance impact.

Why don't you just work on a local clone? If it is really performance critical, and I/O is an issue, you are better off working in a tmpfs.

Once you're done, you push back to the NFS repository (which is lock-challenged AFAIR, but I guess you know that).

Show 7 quoted lines
> It would make my life a lot easier if I could simply open a pipe to a 
> single process for the duration of the script, and do all my queries to 
> this one process.  Of course, if the repository is changed by another 
> process, I would have to restart it, but that's manageable.  I could 
> even write a nice Python class that runs both fast-import and 
> fast-export. I could then have an efficient Python interface to a 
> git-repository, without needing any library wrapping.

There is a minimal python wrapper to libgit-thin, which was one of our GSoC projects. You might want to look at this if you are really that unhappy with the existing framework.

As to the niceness of Python classes; this lies definitely in the eyes of the beholder. For example, I have given up on understanding any part of your GUB framework.

Ciao, Dscho

Han-Wen Nienhuys· Nov 23, 2007, 01:23 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

2007/11/22, Johannes Schindelin <Johannes.Schindelin@gmx.de>:
Show 9 quoted lines
> > The cool thing about git-fast-import is that it reads from stdin, has a
> > very easy to use programmatic interface, and does not impose any order
> > on how you enter the information.
> >
> > This doesn't seem to be mirrored by this script?
>
> Umm.  What exactly do you want to reorder?  I mean, this program is meant
> to dump the complete contents to stdout (whether you redirect it to a file
> or edit it with sed does not concern this program).  It does that.

fast-import does not need to take a complete repository as input, but can also update lots of branches incrementally, all from one process. Likewise, it would be nice to have a program that can also dump small bits of information.

Maybe I'm misunderstanding you, but fast-export just does not seem a mirror of fast-import; perhaps you can name it 'dump-all' or something?

> Maybe you want to specify if all blobs should be output first, and then
> the commits?  Or files should be used?  But all of these things seem to be
> useless to me.

No, I want the program to wait for me to tell it what blobs/commits/trees I want. The commit I want to see secondly may depend on the output I read in the first request blob. Right now, for each data dependency I have to start a new git process.

Show 6 quoted lines
> > Besides being a nuisance, I actually run git on NFS, and every git
> > process has to go to NFS a couple times to retrieve the same
> > information. This has a noticeable performance impact.
>
> Why don't you just work on a local clone?  If it is really performance
> critical, and I/O is an issue, you are better off working in a tmpfs.

In a company setting, NFS is the easiest way to share information with colleagues without breaking access control and making our security staff nervous. It's also snapshotted and backed up automatically.

Show 11 quoted lines
> > It would make my life a lot easier if I could simply open a pipe to a
> > single process for the duration of the script, and do all my queries to
> > this one process.  Of course, if the repository is changed by another
> > process, I would have to restart it, but that's manageable.  I could
> > even write a nice Python class that runs both fast-import and
> > fast-export. I could then have an efficient Python interface to a
> > git-repository, without needing any library wrapping.
>
> There is a minimal python wrapper to libgit-thin, which was one of our
> GSoC projects.  You might want to look at this if you are really that
> unhappy with the existing framework.
What's the status of this?  I prefer not to diverge from mainline too much.
-- 
Han-Wen Nienhuys - hanwen@xs4all.nl - http://www.xs4all.nl/~hanwen
Johannes Schindelin· Nov 23, 2007, 02:11 UTC · re: Han-Wen Nienhuys · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
On Thu, 22 Nov 2007, Han-Wen Nienhuys wrote:
> Maybe I'm misunderstanding you, but fast-export just does not seem a 
> mirror of fast-import; perhaps you can name it 'dump-all' or something?
It is a mirror.  It does not necessarily dump all.
Show 8 quoted lines
> > Maybe you want to specify if all blobs should be output first, and 
> > then the commits?  Or files should be used?  But all of these things 
> > seem to be useless to me.
> 
> No, I want the program to wait for me to tell it what 
> blobs/commits/trees I want. The commit I want to see secondly may depend 
> on the output I read in the first request blob. Right now, for each data 
> dependency I have to start a new git process.

It does not seem like you want a mirror of fast-import, but rather a driver. You might be happy to hear that you can do that already. Today. However, you probably want to query different programs about certain states of the repository. This will not change.

Show 10 quoted lines
> > > Besides being a nuisance, I actually run git on NFS, and every git 
> > > process has to go to NFS a couple times to retrieve the same 
> > > information. This has a noticeable performance impact.
> >
> > Why don't you just work on a local clone?  If it is really performance 
> > critical, and I/O is an issue, you are better off working in a tmpfs.
> 
> In a company setting, NFS is the easiest way to share information with 
> colleagues without breaking access control and making our security staff 
> nervous.  It's also snapshotted and backed up automatically.
So?

How does that prevent you from following my suggestion to do the intensive tasks locally, and push when you finished?

Ciao, Dscho

Shawn O. Pearce· Nov 23, 2007, 20:59 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
Show 19 quoted lines
> On Thu, 22 Nov 2007, Han-Wen Nienhuys wrote:
> 
> > > Maybe you want to specify if all blobs should be output first, and 
> > > then the commits?  Or files should be used?  But all of these things 
> > > seem to be useless to me.
> > 
> > No, I want the program to wait for me to tell it what 
> > blobs/commits/trees I want. The commit I want to see secondly may depend 
> > on the output I read in the first request blob. Right now, for each data 
> > dependency I have to start a new git process.
> 
> It does not seem like you want a mirror of fast-import, but rather a 
> driver.  You might be happy to hear that you can do that already.  Today.
> However, you probably want to query different programs about certain 
> states of the repository.  This will not change.
> 
> > > > Besides being a nuisance, I actually run git on NFS, and every git 
> > > > process has to go to NFS a couple times to retrieve the same 
> > > > information. This has a noticeable performance impact.

I have been considering creating a "git-gui daemon" process that links to libgit.a and can be driven bidirectionally through its stdin/stdout. Based on git-fast-export, sorta. But I haven't even started it...

But the idea is sort of what Han-Wen wants. Why should I fork rev-parse to get a ref value? Or update-ref to change one?

-- 
Shawn.
Karl Hasselström· Nov 25, 2007, 17:00 UTC · re: Shawn O. Pearce · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

On 2007-11-23 15:59:58 -0500, Shawn O. Pearce wrote:
Show 7 quoted lines
> I have been considering creating a "git-gui daemon" process that
> links to libgit.a and can be driven bidirectionally through its
> stdin/stdout. Based on git-fast-export, sorta. But I haven't even
> started it...
>
> But the idea is sort of what Han-Wen wants. Why should I fork
> rev-parse to get a ref value? Or update-ref to change one?

Obviously, something like this would be very valuable for StGit as well.

-- 
Karl Hasselström, kha@treskal.com
      www.treskal.com/kalle
Johannes Schindelin· Nov 26, 2007, 16:48 UTC · re: Karl Hasselström · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
On Sun, 25 Nov 2007, Karl Hasselstr?m wrote:
Show 11 quoted lines
> On 2007-11-23 15:59:58 -0500, Shawn O. Pearce wrote:
> 
> > I have been considering creating a "git-gui daemon" process that links 
> > to libgit.a and can be driven bidirectionally through its 
> > stdin/stdout. Based on git-fast-export, sorta. But I haven't even 
> > started it...
> >
> > But the idea is sort of what Han-Wen wants. Why should I fork 
> > rev-parse to get a ref value? Or update-ref to change one?
> 
> Obviously, something like this would be very valuable for StGit as well.

Could you be a little more specific _what_ you want to do, and _how_ this could be done with fast-export | fast-import?

Ciao, Dscho

Karl Hasselström· Nov 27, 2007, 10:16 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

On 2007-11-26 16:48:14 +0000, Johannes Schindelin wrote:
Show 17 quoted lines
> On Sun, 25 Nov 2007, Karl Hasselström wrote:
>
> > On 2007-11-23 15:59:58 -0500, Shawn O. Pearce wrote:
> >
> > > I have been considering creating a "git-gui daemon" process that
> > > links to libgit.a and can be driven bidirectionally through its
> > > stdin/stdout. Based on git-fast-export, sorta. But I haven't
> > > even started it...
> > >
> > > But the idea is sort of what Han-Wen wants. Why should I fork
> > > rev-parse to get a ref value? Or update-ref to change one?
> >
> > Obviously, something like this would be very valuable for StGit as
> > well.
>
> Could you be a little more specific _what_ you want to do, and _how_
> this could be done with fast-export | fast-import?

Currently, a single StGit command can result in quite a few invocations of git-cat-file, for example, each of which forks off a new process. If it could start just one daemon such as Shawn proposed, and feed it simple questions and commands about blobs, trees, commits, and refs, that would probably be quite a lot faster.

From what I understand, this is not something that would fit a fast-export | fast-import pipeline. Which is why I didn't take the time to elaborate on (or indeed find out) exactly which commands StGit would like such a daemon to support.

By the way: one command one _would_ likely want in the daemon is "list modified files". Being long-lived (not when driven by StGit, perhaps, but definitely when driven by git-gui or qgit), the daemon would be able to use inotify for that.

-- 
Karl Hasselström, kha@treskal.com
      www.treskal.com/kalle
Johannes Schindelin· Nov 27, 2007, 11:25 UTC · re: Karl Hasselström · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
On Tue, 27 Nov 2007, Karl Hasselstr?m wrote:
Show 35 quoted lines
> On 2007-11-26 16:48:14 +0000, Johannes Schindelin wrote:
> 
> > On Sun, 25 Nov 2007, Karl Hasselstr?m wrote:
> >
> > > On 2007-11-23 15:59:58 -0500, Shawn O. Pearce wrote:
> > >
> > > > I have been considering creating a "git-gui daemon" process that 
> > > > links to libgit.a and can be driven bidirectionally through its 
> > > > stdin/stdout. Based on git-fast-export, sorta. But I haven't even 
> > > > started it...
> > > >
> > > > But the idea is sort of what Han-Wen wants. Why should I fork 
> > > > rev-parse to get a ref value? Or update-ref to change one?
> > >
> > > Obviously, something like this would be very valuable for StGit as 
> > > well.
> >
> > Could you be a little more specific _what_ you want to do, and _how_ 
> > this could be done with fast-export | fast-import?
> 
> Currently, a single StGit command can result in quite a few invocations 
> of git-cat-file, for example, each of which forks off a new process. If 
> it could start just one daemon such as Shawn proposed, and feed it 
> simple questions and commands about blobs, trees, commits, and refs, 
> that would probably be quite a lot faster.
> 
> From what I understand, this is not something that would fit a 
> fast-export | fast-import pipeline. Which is why I didn't take the time 
> to elaborate on (or indeed find out) exactly which commands StGit would 
> like such a daemon to support.
> 
> By the way: one command one _would_ likely want in the daemon is "list 
> modified files". Being long-lived (not when driven by StGit, perhaps, 
> but definitely when driven by git-gui or qgit), the daemon would be able 
> to use inotify for that.

Ah, so you would like something like "git --interactive"? This is indeed a completely different scope than the fast-export thingie, which is meant as kind of a mysqldump tool. Indeed, you could use it even as a (kind of a) stash of the repo instead of the working tree.

Ciao, Dscho

Karl Hasselström· Nov 27, 2007, 14:51 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

On 2007-11-27 11:25:47 +0000, Johannes Schindelin wrote:
> Ah, so you would like something like "git --interactive"? This is
> indeed a completely different scope than the fast-export thingie,

Yes. Or rather, I _think_ that's what I want. The only numbers I have is that StGit makes a number of trivial git calls that right now take on the order of 10 ms apiece, so the first step in this direction would be to build a simple prototype just to see what kind of speed-up one could expect (both in the git calls, and in StGit overall).

-- 
Karl Hasselström, kha@treskal.com
      www.treskal.com/kalle
Johannes Schindelin· Nov 27, 2007, 15:10 UTC · re: Karl Hasselström · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
On Tue, 27 Nov 2007, Karl Hasselstr?m wrote:
Show 10 quoted lines
> On 2007-11-27 11:25:47 +0000, Johannes Schindelin wrote:
> 
> > Ah, so you would like something like "git --interactive"? This is 
> > indeed a completely different scope than the fast-export thingie,
> 
> Yes. Or rather, I _think_ that's what I want. The only numbers I have is 
> that StGit makes a number of trivial git calls that right now take on 
> the order of 10 ms apiece, so the first step in this direction would be 
> to build a simple prototype just to see what kind of speed-up one could 
> expect (both in the git calls, and in StGit overall).
I'd rather see you using libgit-thin's Python binding.

Ciao, Dscho

Johannes Schindelin· Nov 26, 2007, 16:47 UTC · re: Shawn O. Pearce · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
On Fri, 23 Nov 2007, Shawn O. Pearce wrote:
Show 27 quoted lines
> Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> > On Thu, 22 Nov 2007, Han-Wen Nienhuys wrote:
> > 
> > > > Maybe you want to specify if all blobs should be output first, and 
> > > > then the commits?  Or files should be used?  But all of these things 
> > > > seem to be useless to me.
> > > 
> > > No, I want the program to wait for me to tell it what 
> > > blobs/commits/trees I want. The commit I want to see secondly may depend 
> > > on the output I read in the first request blob. Right now, for each data 
> > > dependency I have to start a new git process.
> > 
> > It does not seem like you want a mirror of fast-import, but rather a 
> > driver.  You might be happy to hear that you can do that already.  Today.
> > However, you probably want to query different programs about certain 
> > states of the repository.  This will not change.
> > 
> > > > > Besides being a nuisance, I actually run git on NFS, and every git 
> > > > > process has to go to NFS a couple times to retrieve the same 
> > > > > information. This has a noticeable performance impact.
> 
> I have been considering creating a "git-gui daemon" process that links 
> to libgit.a and can be driven bidirectionally through its stdin/stdout.  
> Based on git-fast-export, sorta.  But I haven't even started it...
> 
> But the idea is sort of what Han-Wen wants.  Why should I fork rev-parse 
> to get a ref value?  Or update-ref to change one?

I was thinking about this a little bit more. But I cannot think of a really versatile way of enhancing fast-export enough to be of use there. Well, if not doing something with SWIG, that is ;-)

Ciao, Dscho

Nguyen Thai Ngoc Duy· Nov 23, 2007, 12:31 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

On Nov 21, 2007 10:40 AM, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:

Show 13 quoted lines
>
> [WIP: this does not handle tags yet, and it lacks a test script
>  as well as documentation.]
>
> This program dumps (parts of) a git repository in the format that
> fast-import understands.
>
> For clarity's sake, it does not use the 'inline' method of specifying
> blobs in the commits, but builds the blobs before building the commits.B
>
> ---
>         I am way too tired now to continue, but maybe someone else wants
>         to pick up the ball.

Well, I would better be back on setup_git_directory() than picking up the ball. I have a suggestion though. git-fast-export and git-fast-import should support bundle. Bundle is very handy for transferring a bunch of commits, but it does not (cannot?) hold tags and branches. With git-fast-{im,ex}port, we will have a perfect transportation for disconnected development.

Show 5 quoted lines
>         Oh, and it relies on "int" being castable to void * and vice
>         versa.  Is anybody aware of a platform where this can lead to
>         problems?
>
>         And yes, I will add a copyright when I woke up again.
-- 
Duy
Johannes Schindelin· Nov 23, 2007, 14:31 UTC · re: Nguyen Thai Ngoc Duy · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
On Fri, 23 Nov 2007, Nguyen Thai Ngoc Duy wrote:
Show 18 quoted lines
> On Nov 21, 2007 10:40 AM, Johannes Schindelin
> <Johannes.Schindelin@gmx.de> wrote:
> >
> > [WIP: this does not handle tags yet, and it lacks a test script
> >  as well as documentation.]
> >
> > This program dumps (parts of) a git repository in the format that
> > fast-import understands.
> >
> > For clarity's sake, it does not use the 'inline' method of specifying
> > blobs in the commits, but builds the blobs before building the commits.B
> >
> > ---
> >         I am way too tired now to continue, but maybe someone else wants
> >         to pick up the ball.
> 
> Well, I would better be back on setup_git_directory() than picking up
> the ball.
Concur.
> I have a suggestion though. git-fast-export and git-fast-import should 
> support bundle.

I think this is not what fast-export and fast-import are about. They use an easy to generate, and easy to edit, format.

Bundles are optimised transport mechanisms for sneaker net. They are not to be meant to be easy to edit, but as small as possible.

> Bundle is very handy for transferring a bunch of commits, but it does 
> not (cannot?) hold tags and branches.
But they can!  Nothing prevents you from calling
	git bundle create a1.bundle refs/tags/v1.0.0 refs/heads/next
(At least this is the idea, haven't tested yet).

Ciao, Dscho

Shawn O. Pearce· Nov 23, 2007, 20:56 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
Show 9 quoted lines
> On Fri, 23 Nov 2007, Nguyen Thai Ngoc Duy wrote:
> > I have a suggestion though. git-fast-export and git-fast-import should 
> > support bundle.
> 
> I think this is not what fast-export and fast-import are about.  They use 
> an easy to generate, and easy to edit, format.
> 
> Bundles are optimised transport mechanisms for sneaker net.  They are not 
> to be meant to be easy to edit, but as small as possible.

Actually I wonder, what about bundles that are formatted as a git-fast-import data stream? Then you can have a human readable bundle format that can be (reasonably) easily turned back into a packfile and loaded into the local ODB.

Only we'd probably want to express blobs as diffs if we can, to save disk space, which means we'd need to have git-apply organized in a way that we can call it from within fast-import. :-)

But I agree with Dscho's basic comment; fast-import should not be reading a bundle. It doesn't want to. That's what git-bundle is for.

-- 
Shawn.
Nguyen Thai Ngoc Duy· Nov 24, 2007, 14:08 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

On Nov 23, 2007 9:31 PM, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
Show 42 quoted lines
> Hi,
>
> On Fri, 23 Nov 2007, Nguyen Thai Ngoc Duy wrote:
>
> > On Nov 21, 2007 10:40 AM, Johannes Schindelin
> > <Johannes.Schindelin@gmx.de> wrote:
> > >
> > > [WIP: this does not handle tags yet, and it lacks a test script
> > >  as well as documentation.]
> > >
> > > This program dumps (parts of) a git repository in the format that
> > > fast-import understands.
> > >
> > > For clarity's sake, it does not use the 'inline' method of specifying
> > > blobs in the commits, but builds the blobs before building the commits.B
> > >
> > > ---
> > >         I am way too tired now to continue, but maybe someone else wants
> > >         to pick up the ball.
> >
> > Well, I would better be back on setup_git_directory() than picking up
> > the ball.
>
> Concur.
>
> > I have a suggestion though. git-fast-export and git-fast-import should
> > support bundle.
>
> I think this is not what fast-export and fast-import are about.  They use
> an easy to generate, and easy to edit, format.
>
> Bundles are optimised transport mechanisms for sneaker net.  They are not
> to be meant to be easy to edit, but as small as possible.
>
> > Bundle is very handy for transferring a bunch of commits, but it does
> > not (cannot?) hold tags and branches.
>
> But they can!  Nothing prevents you from calling
>
>         git bundle create a1.bundle refs/tags/v1.0.0 refs/heads/next
>
> (At least this is the idea, haven't tested yet).

It can store commits and heavy tags, but it won't restore tags to refs/tags or advance branches.

-- 
Duy
Johannes Schindelin· Nov 27, 2007, 12:16 UTC · re: Nguyen Thai Ngoc Duy · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

Hi,
On Sat, 24 Nov 2007, Nguyen Thai Ngoc Duy wrote:
Show 15 quoted lines
> On Nov 23, 2007 9:31 PM, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
>
> > On Fri, 23 Nov 2007, Nguyen Thai Ngoc Duy wrote:
> >
> > > Bundle is very handy for transferring a bunch of commits, but it 
> > > does not (cannot?) hold tags and branches.
> >
> > But they can!  Nothing prevents you from calling
> >
> >         git bundle create a1.bundle refs/tags/v1.0.0 refs/heads/next
> >
> > (At least this is the idea, haven't tested yet).
> 
> It can store commits and heavy tags, but it won't restore tags to 
> refs/tags or advance branches.

The idea is that you fetch them from the bundle. So something like this should do what you want:

	git fetch a1.bundle v1.0.0:refs/tags/v1.0.0

Note that the automatic tag handling of git fetch kicks in with bundles just like with other fetch URLs (Unassuming Repository Locators).

Ciao, Dscho

Nguyen Thai Ngoc Duy· Nov 27, 2007, 14:17 UTC · re: Johannes Schindelin · lore

Re: [WIP PATCH] Add 'git fast-export', the sister of 'git fast-import'

On Nov 27, 2007 7:16 PM, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
Show 27 quoted lines
> Hi,
>
> On Sat, 24 Nov 2007, Nguyen Thai Ngoc Duy wrote:
>
> > On Nov 23, 2007 9:31 PM, Johannes Schindelin <Johannes.Schindelin@gmx.de> wrote:
> >
> > > On Fri, 23 Nov 2007, Nguyen Thai Ngoc Duy wrote:
> > >
> > > > Bundle is very handy for transferring a bunch of commits, but it
> > > > does not (cannot?) hold tags and branches.
> > >
> > > But they can!  Nothing prevents you from calling
> > >
> > >         git bundle create a1.bundle refs/tags/v1.0.0 refs/heads/next
> > >
> > > (At least this is the idea, haven't tested yet).
> >
> > It can store commits and heavy tags, but it won't restore tags to
> > refs/tags or advance branches.
>
> The idea is that you fetch them from the bundle.  So something like this
> should do what you want:
>
>         git fetch a1.bundle v1.0.0:refs/tags/v1.0.0
>
> Note that the automatic tag handling of git fetch kicks in with bundles
> just like with other fetch URLs (Unassuming Repository Locators).

Yes, you are right. Man, if I knew I could pass a bundle to git-fetch, my life would have been much easier :(

-- 
Duy

← back to recent threads