git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 11/11] git-svn: Speed up fetch

From
Adam Roben <aroben@apple.com>
Date
Apr 23, 2008, 19:17 UTC
Message-ID
<1208978273-98146-12-git-send-email-aroben@apple.com>
In-Reply-To
<1208978273-98146-11-git-send-email-aroben@apple.com>

We were spending a lot of time forking/execing git-cat-file and git-hash-object. We now maintain a global Git repository object in order to use Git.pm's more efficient hash_and_insert_object and cat_blob methods.

Signed-off-by: Adam Roben <aroben@apple.com>
---
 git-svn.perl |   42 ++++++++++++++++++++----------------------
 1 files changed, 20 insertions(+), 22 deletions(-)
diff --git a/git-svn.perl b/git-svn.perl
index b864b54..5ef9d23 100755
--- a/git-svn.perl
+++ b/git-svn.perl
@@ -4,7 +4,7 @@
 use warnings;
 use strict;
 use vars qw/	$AUTHOR $VERSION
-		$sha1 $sha1_short $_revision
+		$sha1 $sha1_short $_revision $_repository
 		$_q $_authors %users/;
 $AUTHOR = 'Eric Wong <normalperson@yhbt.net>';
 $VERSION = '@@GIT_VERSION@@';
@@ -220,6 +220,7 @@ unless ($cmd && $cmd =~ /(?:clone|init|multi-init)$/) {
 		}
 		$ENV{GIT_DIR} = $git_dir;
 	}
+	$_repository = Git->repository(Repository => $ENV{GIT_DIR});
 }
 
 my %opts = %{$cmd{$cmd}->[2]} if (defined $cmd);
@@ -301,6 +302,7 @@ sub do_git_init_db {
 			}
 		}
 		command_noisy(@init_db);
+		$_repository = Git->repository(Repository => ".git");
 	}
 	my $set;
 	my $pfx = "svn-remote.$Git::SVN::default_repo_id";
@@ -317,6 +319,7 @@ sub init_subdir {
 	mkpath([$repo_path]) unless -d $repo_path;
 	chdir $repo_path or die "Couldn't chdir to $repo_path: $!\n";
 	$ENV{GIT_DIR} = '.git';
+	$_repository = Git->repository(Repository => $ENV{GIT_DIR});
 }
 
 sub cmd_clone {
@@ -3010,6 +3013,7 @@ use vars qw/@ISA/;
 use strict;
 use warnings;
 use Carp qw/croak/;
+use File::Temp qw/tempfile/;
 use IO::File qw//;
 
 # file baton members: path, mode_a, mode_b, pool, fh, blob, base
@@ -3165,14 +3169,9 @@ sub apply_textdelta {
 	my $base = IO::File->new_tmpfile;
 	$base->autoflush(1);
 	if ($fb->{blob}) {
-		defined (my $pid = fork) or croak $!;
-		if (!$pid) {
-			open STDOUT, '>&', $base or croak $!;
-			print STDOUT 'link ' if ($fb->{mode_a} == 120000);
-			exec qw/git-cat-file blob/, $fb->{blob} or croak $!;
-		}
-		waitpid $pid, 0;
-		croak $? if $?;
+		print $base 'link ' if ($fb->{mode_a} == 120000);
+		my $size = $::_repository->cat_blob($fb->{blob}, $base);
+		die "Failed to read object $fb->{blob}" unless $size;
 
 		if (defined $exp) {
 			seek $base, 0, 0 or croak $!;
@@ -3213,14 +3212,18 @@ sub close_file {
 				sysseek($fh, 0, 0) or croak $!;
 			}
 		}
-		defined(my $pid = open my $out,'-|') or die "Can't fork: $!\n";
-		if (!$pid) {
-			open STDIN, '<&', $fh or croak $!;
-			exec qw/git-hash-object -w --stdin/ or croak $!;
+
+		my ($tmp_fh, $tmp_filename) = File::Temp::tempfile(UNLINK => 1);
+		my $result;
+		while ($result = sysread($fh, my $string, 1024)) {
+			syswrite($tmp_fh, $string, $result);
 		}
-		chomp($hash = do { local $/; <$out> });
-		close $out or croak $!;
+		defined $result or croak $!;
+		close $tmp_fh or croak $!;
+
 		close $fh or croak $!;
+
+		$hash = $::_repository->hash_and_insert_object($tmp_filename);
 		$hash =~ /^[a-f\d]{40}$/ or die "not a sha1: $hash\n";
 		close $fb->{base} or croak $!;
 	} else {
@@ -3546,13 +3549,8 @@ sub chg_file {
 	} elsif ($m->{mode_a} =~ /^120/ && $m->{mode_b} !~ /^120/) {
 		$self->change_file_prop($fbat,'svn:special',undef);
 	}
-	defined(my $pid = fork) or croak $!;
-	if (!$pid) {
-		open STDOUT, '>&', $fh or croak $!;
-		exec qw/git-cat-file blob/, $m->{sha1_b} or croak $!;
-	}
-	waitpid $pid, 0;
-	croak $? if $?;
+	my $size = $::_repository->cat_blob($m->{sha1_b}, $fh);
+	croak "Failed to read object $m->{sha1_b}" unless $size;
 	$fh->flush == 0 or croak $!;
 	seek $fh, 0, 0 or croak $!;
 
-- 
1.5.5.1.152.g9aeb7
Previous: Adam RobenNext: Junio C Hamano
Message 12 of 19 in “Speed up git-svn fetch”
  1. Adam RobenApr 23, 2008
  2. 01/11 Add tests for git cat-fileAdam Roben, Apr 23, 2008
  3. 02/11 git-cat-file: Small refactor of cmd_cat_fileAdam Roben, Apr 23, 2008
  4. 03/11 git-cat-file: Make option parsing a little more flexibleAdam Roben, Apr 23, 2008
  5. 04/11 git-cat-file: Add --batch-check optionAdam Roben, Apr 23, 2008
  6. 05/11 git-cat-file: Add --batch optionAdam Roben, Apr 23, 2008
  7. 06/11 Move git-hash-object tests from t5303 to t1007Adam Roben, Apr 23, 2008
  8. 07/11 Add more tests for git hash-objectAdam Roben, Apr 23, 2008
  9. 08/11 git-hash-object: Add --stdin-paths optionAdam Roben, Apr 23, 2008
  10. 09/11 Git.pm: Add command_bidi_pipe and command_close_bidi_pipeAdam Roben, Apr 23, 2008
  11. 10/11 Git.pm: Add hash_and_insert_object and cat_blobAdam Roben, Apr 23, 2008
  12. 11/11 git-svn: Speed up fetchAdam Roben, Apr 23, 2008
  13. Junio C HamanoApr 25, 2008
  14. Eric WongApr 25, 2008
  15. Junio C HamanoApr 25, 2008
  16. Junio C HamanoApr 25, 2008
  17. Junio C HamanoMay 6, 2008
  18. Adam RobenApr 23, 2008
  19. Eric WongApr 25, 2008

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.