{"thread":{"id":"25314","subject":"[RFC PATCH] git-svn: fix performance importing tagged subdirectories","startedAt":"2010-10-02T13:01:12Z","lastAt":"2010-10-02T13:01:12Z","messageCount":1,"participants":["David D. Kilzer"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"152289","messageId":"1286024472-2255-1-git-send-email-ddkilzer@kilzer.net","threadId":"25314","inReplyTo":null,"subject":"[RFC PATCH] git-svn: fix performance importing tagged subdirectories","fromName":"David D. Kilzer","fromEmail":"ddkilzer@kilzer.net","sentAt":"2010-10-02T13:01:12Z","receivedAt":"2010-10-02T13:01:12Z","isPatch":true,"sender":{"key":"ddkilzer@kilzer.net","avatar":"https://avatars.githubusercontent.com/u/263571?v=4"},"body":"NOTE: This is an RFC patch because I'm pretty sure I should be\nusing Git::SVN::Ra::can_do_switch() somewhere, and because I am\nvery likely abusing git-svn internal methods in a way that will\nproduce incorrect results in some cases.\n\nDave\n--\n\nWhen an svn repository has multiple related projects checked in\nas individual directories under trunk:\n\n    trunk/project1/\n    trunk/project2/\n    trunk/project3/\n\nand each project subdirectory is tagged instead of tagging\ntrunk:\n\n    [...]\n    tags/project1-204\n    tags/project1-205\n    [...]\n    tags/project2-395\n    tags/project2-396\n    [...]\n    tags/project3-77\n    tags/project3-78\n    [...]\n\nthen git-svn currently imports the entire history of each new\ntag beginning with r1.  This happens because git-svn uses the\nname of the branch or tag when attempting to fast-forward svn\nhistory.  For large svn repositories, the time required to\nimport each additional tag grows exponentially.\n\nA better approach is to search through all known refs for a\nref that has the same repository URL, but with a smaller max\nrevision.  This ref could then be used to seed a new ref for\nthe tag being imported, thus bypassing the majority of the\nwork.\n\nThis approach is implemented by changing find_by_url() to take\nan additional parameter ($rev) that tells it to return a ref\nthat represents the closest match to the desired repo url while\nhaving a revision less than or equal to $rev.  When a brand new\nref is created in other_gs(), the new find_by_url() behavior is\nused to find the closest matching ref and use it as a seed.\n---\n git-svn.perl                          |   45 +++++++++++++++++++++---\n t/t9157-git-svn-subdir-import-perf.sh |   59 +++++++++++++++++++++++++++++++++\n 2 files changed, 98 insertions(+), 6 deletions(-)\n create mode 100755 t/t9157-git-svn-subdir-import-perf.sh\n\ndiff --git a/git-svn.perl b/git-svn.perl\nindex 9b046b6..af46f5f 100755\n--- a/git-svn.perl\n+++ b/git-svn.perl\n@@ -1967,8 +1967,12 @@ sub init_remote_config {\n \t$self->{url} = $url;\n }\n \n-sub find_by_url { # repos_root and, path are optional\n-\tmy ($class, $full_url, $repos_root, $path) = @_;\n+# Finds an exact match for a ref based on $full_url, $repos_root and\n+# $path.  If no exact match is found and if $rev is specified, the\n+# closest match with the same url and a revision <= $rev is returned.\n+# Note that $repos_root, $path and $rev are optional.\n+sub find_by_url {\n+\tmy ($class, $full_url, $repos_root, $path, $rev) = @_;\n \n \treturn undef unless defined $full_url;\n \tremove_username($full_url);\n@@ -1978,6 +1982,7 @@ sub find_by_url { # repos_root and, path are optional\n \t\t$path = $full_url;\n \t\t$path =~ s#^\\Q$repos_root\\E(?:/|$)##;\n \t}\n+\tmy ($closest_gs, $closest_max_rev);\n \tforeach my $repo_id (keys %$remotes) {\n \t\tmy $u = $remotes->{$repo_id}->{url} or next;\n \t\tremove_username($u);\n@@ -2009,11 +2014,22 @@ sub find_by_url { # repos_root and, path are optional\n \t\t\t$p =~ s#^\\Q$z\\E(?:/|$)#$prefix# or next;\n \t\t}\n \t\tforeach my $f (keys %$fetch) {\n-\t\t\tnext if $f ne $p;\n-\t\t\treturn Git::SVN->new($fetch->{$f}, $repo_id, $f);\n+\t\t\tunless ($rev) {\n+\t\t\t\tnext if $f ne $p;\n+\t\t\t\treturn Git::SVN->new($fetch->{$f}, $repo_id, $f);\n+\t\t\t}\n+\t\t\tmy $gs = Git::SVN->new($fetch->{$f}, $repo_id, $f);\n+\t\t\tmy ($max_rev, $max_commit) = $gs->rev_map_max(1);\n+\t\t\tnext if !$max_rev || !$max_commit;\n+\t\t\tmy ($url) = ::cmt_metadata($max_commit);\n+\t\t\tnext if $url ne $full_url || $max_rev > $rev;\n+\t\t\tif (!$closest_gs || $closest_max_rev < $max_rev) {\n+\t\t\t\t$closest_gs = $gs;\n+\t\t\t\t$closest_max_rev = $max_rev;\n+\t\t\t}\n \t\t}\n \t}\n-\tundef;\n+\t$closest_gs && $rev ? $closest_gs : undef;\n }\n \n sub init {\n@@ -2969,18 +2985,35 @@ sub other_gs {\n \t\t\t$u = $url;\n \t\t\t$repo_id = $self->{repo_id};\n \t\t}\n+\t\tmy $max_commit;\n \t\twhile (1) {\n \t\t\t# It is possible to tag two different subdirectories at\n \t\t\t# the same revision.  If the url for an existing ref\n \t\t\t# does not match, we must either find a ref with a\n \t\t\t# matching url or create a new ref by growing a tail.\n \t\t\t$gs = Git::SVN->init($u, $p, $repo_id, $ref_id, 1);\n-\t\t\tmy (undef, $max_commit) = $gs->rev_map_max(1);\n+\t\t\t(undef, $max_commit) = $gs->rev_map_max(1);\n \t\t\tlast if (!$max_commit);\n \t\t\tmy ($url) = ::cmt_metadata($max_commit);\n \t\t\tlast if ($url eq $gs->full_url);\n \t\t\t$ref_id .= '-';\n \t\t}\n+\t\tunless ($max_commit) {\n+\t\t\t# If a brand new ref was created, try to find a matching\n+\t\t\t# ref with the same url and a smaller revision to use as\n+\t\t\t# as a seed.  This avoids reloading the entire history\n+\t\t\t# of the repository when the same subdirectory is tagged\n+\t\t\t# frequently.\n+\t\t\tmy $parent_gs = Git::SVN->find_by_url($new_url, $url,\n+\t\t\t\t$branch_from, $r);\n+\t\t\tif ($parent_gs) {\n+\t\t\t\tmy ($parent_rev, $parent_commit) =\n+\t\t\t\t\t$parent_gs->rev_map_max(1);\n+\t\t\t\t$gs->rev_map_set($parent_rev, $parent_commit);\n+\t\t\t\tprint STDERR \"Using \" . $parent_gs->{path} .\n+\t\t\t\t\t\" as seed: $ref_id\\n\" unless $::_q > 1;\n+\t\t\t}\n+\t\t}\n \t\tprint STDERR \"Initializing parent: $ref_id\\n\" unless $::_q > 1;\n \t}\n \t$gs\ndiff --git a/t/t9157-git-svn-subdir-import-perf.sh b/t/t9157-git-svn-subdir-import-perf.sh\nnew file mode 100755\nindex 0000000..d28d0e0\n--- /dev/null\n+++ b/t/t9157-git-svn-subdir-import-perf.sh\n@@ -0,0 +1,59 @@\n+#!/bin/sh\n+\n+test_description='git svn import subdirectory performance'\n+\n+. ./lib-git-svn.sh\n+\n+test_expect_success 'setup svn repo' '\n+\tmkdir -p import/trunk/subdir &&\n+\tmkdir -p import/branches &&\n+\tmkdir -p import/tags &&\n+\techo \"base\" >import/trunk/subdir/file &&\n+\tsvn_cmd import -m \"import for git svn\" import \"$svnrepo\" &&\n+\trm -rf import &&\n+\n+\tsvn_cmd co \"$svnrepo/trunk\" svn_project &&\n+\tj=4 &&\n+\t(cd svn_project &&\n+\t\ti=1 &&\n+\t\twhile [ $i -le $j ]; do\n+\t\t\techo \"$i\" >>subdir/file &&\n+\t\t\tsvn_cmd ci -m \"trunk change $i\" subdir/file &&\n+\t\t\ti=$(($i+1))\n+\t\tdone\n+\t) &&\n+\n+\tsvn_cmd cp -m \"create tag mytag1\" \"$svnrepo/trunk/subdir\" \"$svnrepo/tags/mytag1\" &&\n+\n+\t(cd svn_project &&\n+\t\ti=$(($j+1)) &&\n+\t\techo \"$i\" >>subdir/file &&\n+\t\tsvn_cmd ci -m \"trunk change $i\" subdir/file\n+\t) &&\n+\n+\tsvn_cmd cp -m \"create tag mytag2\" \"$svnrepo/trunk/subdir\" \"$svnrepo/tags/mytag2\"\n+\n+\t(cd svn_project &&\n+\t\ti=$(($j+2)) &&\n+\t\techo \"$i\" >>subdir/file &&\n+\t\tsvn_cmd ci -m \"trunk change $i\" subdir/file\n+\t) &&\n+\n+\tsvn_cmd cp -m \"create tag mytag3\" \"$svnrepo/trunk/subdir\" \"$svnrepo/tags/mytag3\"\n+'\n+\n+test_expect_success 'import subdirectory performance' '\n+\tgit svn init --stdlayout \"$svnrepo\" git_project &&\n+\tcd git_project &&\n+\tgit svn fetch | tee fetch.txt &&\n+\n+\tgrep \"refs/remotes/tags/mytag2@7\" fetch.txt >actual.txt &&\n+\tgrep \"^r7\" actual.txt >expected.txt &&\n+\tdiff -u expected.txt actual.txt &&\n+\n+\tgit diff --exit-code tags/mytag1..tags/mytag2^^ &&\n+\tgit diff --exit-code tags/mytag1..tags/mytag3^^^ &&\n+\tgit diff --exit-code tags/mytag2..tags/mytag3^^\n+'\n+\n+test_done\n-- \n1.7.2.1.158.gbd3a97\n"}]}