[PATCH] improve git svn performance

[PATCH] improve git svn performance

[manjian2006]

From: linzj <linzj@ucweb.com>

Hi,
  I am trying to improve git svn's performance according to some profiling data.As the data showed,_rev_list subroutine and rebuild subroutine are consuming a large proportion of time.So I improve _rev_list's performance by memoize its results,and avoid subprocess invocation by memoize rebuild subroutine's key data.Here's my patch:
---
 perl/Git/SVN.pm | 63 ++++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 49 insertions(+), 14 deletions(-)

diff --git a/perl/Git/SVN.pm b/perl/Git/SVN.pm
index 5273ee8..3cd1c8f 100644
--- a/perl/Git/SVN.pm
+++ b/perl/Git/SVN.pm
@@ -1599,27 +1599,36 @@ sub tie_for_persistent_memoization {
 		my %lookup_svn_merge_cache;
 		my %check_cherry_pick_cache;
 		my %has_no_changes_cache;
+		my %_rev_list_cache;
 
 		tie_for_persistent_memoization(\%lookup_svn_merge_cache,
-		    "$cache_path/lookup_svn_merge");
+				"$cache_path/lookup_svn_merge");
 		memoize 'lookup_svn_merge',
-			SCALAR_CACHE => 'FAULT',
-			LIST_CACHE => ['HASH' => \%lookup_svn_merge_cache],
-		;
+				SCALAR_CACHE => 'FAULT',
+				LIST_CACHE => ['HASH' => \%lookup_svn_merge_cache],
+				;
 
 		tie_for_persistent_memoization(\%check_cherry_pick_cache,
-		    "$cache_path/check_cherry_pick");
+				"$cache_path/check_cherry_pick");
 		memoize 'check_cherry_pick',
-			SCALAR_CACHE => 'FAULT',
-			LIST_CACHE => ['HASH' => \%check_cherry_pick_cache],
-		;
+				SCALAR_CACHE => 'FAULT',
+				LIST_CACHE => ['HASH' => \%check_cherry_pick_cache],
+				;
 
 		tie_for_persistent_memoization(\%has_no_changes_cache,
-		    "$cache_path/has_no_changes");
+				"$cache_path/has_no_changes");
 		memoize 'has_no_changes',
-			SCALAR_CACHE => ['HASH' => \%has_no_changes_cache],
-			LIST_CACHE => 'FAULT',
-		;
+				SCALAR_CACHE => ['HASH' => \%has_no_changes_cache],
+				LIST_CACHE => 'FAULT',
+				;
+
+		tie_for_persistent_memoization(\%_rev_list_cache,
+				"$cache_path/_rev_list");
+		memoize '_rev_list',
+				SCALAR_CACHE => 'FAULT',
+				LIST_CACHE => ['HASH' => \%_rev_list_cache],
+				;
+
 	}
 
 	sub unmemoize_svn_mergeinfo_functions {
@@ -1629,6 +1638,7 @@ sub tie_for_persistent_memoization {
 		Memoize::unmemoize 'lookup_svn_merge';
 		Memoize::unmemoize 'check_cherry_pick';
 		Memoize::unmemoize 'has_no_changes';
+		Memoize::unmemoize '_rev_list';
 	}
 
 	sub clear_memoized_mergeinfo_caches {
@@ -1959,11 +1969,25 @@ sub rebuild_from_rev_db {
 	unlink $path or croak "unlink: $!";
 }
 
+#define a global associate map to record rebuild status
+my %rebuildStatus;
+#define a global associate map to record rebuild verify status
+my %rebuildVerifyStatus;
+
 sub rebuild {
 	my ($self) = @_;
 	my $map_path = $self->map_path;
 	my $partial = (-e $map_path && ! -z $map_path);
-	return unless ::verify_ref($self->refname.'^0');
+    my $verifyKey = $self->refname.'^0';
+    if (! exists $rebuildVerifyStatus{$verifyKey} || ! defined $rebuildVerifyStatus{$verifyKey} ) {
+        my $verifyResult = ::verify_ref($verifyKey);
+        if ($verifyResult) {
+            $rebuildVerifyStatus{$verifyKey} = 1;
+        }
+    }
+    if (! exists $rebuildVerifyStatus{$verifyKey}) {
+        return;
+    }
 	if (!$partial && ($self->use_svm_props || $self->no_metadata)) {
 		my $rev_db = $self->rev_db_path;
 		$self->rebuild_from_rev_db($rev_db);
@@ -1977,10 +2001,21 @@ sub rebuild {
 	print "Rebuilding $map_path ...\n" if (!$partial);
 	my ($base_rev, $head) = ($partial ? $self->rev_map_max_norebuild(1) :
 		(undef, undef));
+    my $keyValue = ($head ? "$head.." : "") . $self->refname;
+    if (exists $rebuildStatus{$keyValue}) {
+        print "Done rebuilding $map_path\n" if (!$partial || !$head);
+        my $rev_db_path = $self->rev_db_path;
+        if (-f $self->rev_db_path) {
+            unlink $self->rev_db_path or croak "unlink: $!";
+        }
+        $self->unlink_rev_db_symlink;
+		return;
+    }
 	my ($log, $ctx) =
 	    command_output_pipe(qw/rev-list --pretty=raw --reverse/,
-				($head ? "$head.." : "") . $self->refname,
+			    $keyValue,	
 				'--');
+    $rebuildStatus{$keyValue} = 1;
 	my $metadata_url = $self->metadata_url;
 	remove_username($metadata_url);
 	my $svn_uuid = $self->rewrite_uuid || $self->ra_uuid;
-- 
1.8.3.2

Re: [PATCH] improve git svn performance

[Eric Wong]

manjian2006@gmail.com wrote:
> From: linzj <linzj@ucweb.com>
>   I am trying to improve git svn's performance according to some
>   profiling data.As the data showed,_rev_list subroutine and rebuild
>   subroutine are consuming a large proportion of time.So I improve
>   _rev_list's performance by memoize its results,and avoid subprocess
>   invocation by memoize rebuild subroutine's key data.Here's my patch:

Hi, I'm interested in this.  How much did performance improve by
(and how many revisions is the repository)>

However, I cannot accept the patch in the current state.

The commit message is inadequate; and you need a Signed-off-by and follow
existing coding convention/style.  See Documentation/SubmittingPatches
and Documentation/CodingGuidelines for more info.

Some comments inline...

>  		tie_for_persistent_memoization(\%lookup_svn_merge_cache,
> -		    "$cache_path/lookup_svn_merge");
> +				"$cache_path/lookup_svn_merge");
>  		memoize 'lookup_svn_merge',
> -			SCALAR_CACHE => 'FAULT',
> -			LIST_CACHE => ['HASH' => \%lookup_svn_merge_cache],
> -		;
> +				SCALAR_CACHE => 'FAULT',
> +				LIST_CACHE => ['HASH' => \%lookup_svn_merge_cache],
> +				;

Please no extraneous whitespace changes.

> +#define a global associate map to record rebuild status
> +my %rebuildStatus;

We prefer snake_case variables, not camelCase.

>  sub rebuild {
>  	my ($self) = @_;
>  	my $map_path = $self->map_path;
>  	my $partial = (-e $map_path && ! -z $map_path);
> -	return unless ::verify_ref($self->refname.'^0');
> +    my $verifyKey = $self->refname.'^0';
> +    if (! exists $rebuildVerifyStatus{$verifyKey} || ! defined $rebuildVerifyStatus{$verifyKey} ) {
> +        my $verifyResult = ::verify_ref($verifyKey);
> +        if ($verifyResult) {
> +            $rebuildVerifyStatus{$verifyKey} = 1;
> +        }
> +    }
> +    if (! exists $rebuildVerifyStatus{$verifyKey}) {
> +        return;
> +    }

Please use tabs for indentation to match surrounding code
(most of git is tabs).

Thanks.