# [PATCH] improve git svn performance 2 messages from 2014-01-20 to 2014-01-22. Participants: manjian2006@gmail.com, Eric Wong. Thread: https://gitlist.dev/t/35692 ## manjian2006@gmail.com, 2014-01-20 04:37 Subject: [PATCH] improve git svn performance Message-ID: <1390192672-3827-1-git-send-email-manjian2006@gmail.com> URL: https://gitlist.dev/e/1390192672-3827-1-git-send-email-manjian2006%40gmail.com ``` From: linzj Hi, I am trying to improve git svn's performance according to some profiling data.As the data showed,_rev_list subroutine and rebuild subroutine are consuming a large proportion of time.So I improve _rev_list's performance by memoize its results,and avoid subprocess invocation by memoize rebuild subroutine's key data.Here's my patch: --- perl/Git/SVN.pm | 63 ++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/perl/Git/SVN.pm b/perl/Git/SVN.pm index 5273ee8..3cd1c8f 100644 --- a/perl/Git/SVN.pm +++ b/perl/Git/SVN.pm @@ -1599,27 +1599,36 @@ sub tie_for_persistent_memoization { my %lookup_svn_merge_cache; my %check_cherry_pick_cache; my %has_no_changes_cache; + my %_rev_list_cache; tie_for_persistent_memoization(\%lookup_svn_merge_cache, - "$cache_path/lookup_svn_merge"); + "$cache_path/lookup_svn_merge"); memoize 'lookup_svn_merge', - SCALAR_CACHE => 'FAULT', - LIST_CACHE => ['HASH' => \%lookup_svn_merge_cache], - ; + SCALAR_CACHE => 'FAULT', + LIST_CACHE => ['HASH' => \%lookup_svn_merge_cache], + ; tie_for_persistent_memoization(\%check_cherry_pick_cache, - "$cache_path/check_cherry_pick"); + "$cache_path/check_cherry_pick"); memoize 'check_cherry_pick', - SCALAR_CACHE => 'FAULT', - LIST_CACHE => ['HASH' => \%check_cherry_pick_cache], - ; + SCALAR_CACHE => 'FAULT', + LIST_CACHE => ['HASH' => \%check_cherry_pick_cache], + ; tie_for_persistent_memoization(\%has_no_changes_cache, - "$cache_path/has_no_changes"); + "$cache_path/has_no_changes"); memoize 'has_no_changes', - SCALAR_CACHE => ['HASH' => \%has_no_changes_cache], - LIST_CACHE => 'FAULT', - ; + SCALAR_CACHE => ['HASH' => \%has_no_changes_cache], + LIST_CACHE => 'FAULT', + ; + + tie_for_persistent_memoization(\%_rev_list_cache, + "$cache_path/_rev_list"); + memoize '_rev_list', + SCALAR_CACHE => 'FAULT', + LIST_CACHE => ['HASH' => \%_rev_list_cache], + ; + } sub unmemoize_svn_mergeinfo_functions { @@ -1629,6 +1638,7 @@ sub tie_for_persistent_memoization { Memoize::unmemoize 'lookup_svn_merge'; Memoize::unmemoize 'check_cherry_pick'; Memoize::unmemoize 'has_no_changes'; + Memoize::unmemoize '_rev_list'; } sub clear_memoized_mergeinfo_caches { @@ -1959,11 +1969,25 @@ sub rebuild_from_rev_db { unlink $path or croak "unlink: $!"; } +#define a global associate map to record rebuild status +my %rebuildStatus; +#define a global associate map to record rebuild verify status +my %rebuildVerifyStatus; + sub rebuild { my ($self) = @_; my $map_path = $self->map_path; my $partial = (-e $map_path && ! -z $map_path); - return unless ::verify_ref($self->refname.'^0'); + my $verifyKey = $self->refname.'^0'; + if (! exists $rebuildVerifyStatus{$verifyKey} || ! defined $rebuildVerifyStatus{$verifyKey} ) { + my $verifyResult = ::verify_ref($verifyKey); + if ($verifyResult) { + $rebuildVerifyStatus{$verifyKey} = 1; + } + } + if (! exists $rebuildVerifyStatus{$verifyKey}) { + return; + } if (!$partial && ($self->use_svm_props || $self->no_metadata)) { my $rev_db = $self->rev_db_path; $self->rebuild_from_rev_db($rev_db); @@ -1977,10 +2001,21 @@ sub rebuild { print "Rebuilding $map_path ...\n" if (!$partial); my ($base_rev, $head) = ($partial ? $self->rev_map_max_norebuild(1) : (undef, undef)); + my $keyValue = ($head ? "$head.." : "") . $self->refname; + if (exists $rebuildStatus{$keyValue}) { + print "Done rebuilding $map_path\n" if (!$partial || !$head); + my $rev_db_path = $self->rev_db_path; + if (-f $self->rev_db_path) { + unlink $self->rev_db_path or croak "unlink: $!"; + } + $self->unlink_rev_db_symlink; + return; + } my ($log, $ctx) = command_output_pipe(qw/rev-list --pretty=raw --reverse/, - ($head ? "$head.." : "") . $self->refname, + $keyValue, '--'); + $rebuildStatus{$keyValue} = 1; my $metadata_url = $self->metadata_url; remove_username($metadata_url); my $svn_uuid = $self->rewrite_uuid || $self->ra_uuid; -- 1.8.3.2 ``` ## Eric Wong, 2014-01-22 02:56 Subject: Re: [PATCH] improve git svn performance Message-ID: <20140122025630.GA15886@dcvr.yhbt.net> URL: https://gitlist.dev/e/20140122025630.GA15886%40dcvr.yhbt.net In-Reply-To: <1390192672-3827-1-git-send-email-manjian2006@gmail.com> ``` manjian2006@gmail.com wrote: > From: linzj > I am trying to improve git svn's performance according to some > profiling data.As the data showed,_rev_list subroutine and rebuild > subroutine are consuming a large proportion of time.So I improve > _rev_list's performance by memoize its results,and avoid subprocess > invocation by memoize rebuild subroutine's key data.Here's my patch: Hi, I'm interested in this. How much did performance improve by (and how many revisions is the repository)> However, I cannot accept the patch in the current state. The commit message is inadequate; and you need a Signed-off-by and follow existing coding convention/style. See Documentation/SubmittingPatches and Documentation/CodingGuidelines for more info. Some comments inline... > tie_for_persistent_memoization(\%lookup_svn_merge_cache, > - "$cache_path/lookup_svn_merge"); > + "$cache_path/lookup_svn_merge"); > memoize 'lookup_svn_merge', > - SCALAR_CACHE => 'FAULT', > - LIST_CACHE => ['HASH' => \%lookup_svn_merge_cache], > - ; > + SCALAR_CACHE => 'FAULT', > + LIST_CACHE => ['HASH' => \%lookup_svn_merge_cache], > + ; Please no extraneous whitespace changes. > +#define a global associate map to record rebuild status > +my %rebuildStatus; We prefer snake_case variables, not camelCase. > sub rebuild { > my ($self) = @_; > my $map_path = $self->map_path; > my $partial = (-e $map_path && ! -z $map_path); > - return unless ::verify_ref($self->refname.'^0'); > + my $verifyKey = $self->refname.'^0'; > + if (! exists $rebuildVerifyStatus{$verifyKey} || ! defined $rebuildVerifyStatus{$verifyKey} ) { > + my $verifyResult = ::verify_ref($verifyKey); > + if ($verifyResult) { > + $rebuildVerifyStatus{$verifyKey} = 1; > + } > + } > + if (! exists $rebuildVerifyStatus{$verifyKey}) { > + return; > + } Please use tabs for indentation to match surrounding code (most of git is tabs). Thanks. ```