{"thread":{"id":"1700","subject":"[PATCH] Initial import of git-archimport-script","startedAt":"2005-08-30T09:56:52Z","lastAt":"2005-08-30T09:56:52Z","messageCount":1,"participants":["Martin Langhoff"],"isPatch":true,"patchVersion":1,"patchTotal":null},"messages":[{"id":"7948","messageId":"1125395812179-git-send-email-martin@catalyst.net.nz","threadId":"1700","inReplyTo":null,"subject":"[PATCH] Initial import of git-archimport-script","fromName":"Martin Langhoff","fromEmail":"martin@catalyst.net.nz","sentAt":"2005-08-30T09:56:52Z","receivedAt":"2005-08-30T09:56:52Z","isPatch":true,"sender":{"key":"martin@laptop.org","avatar":null},"body":"Imports a project history from one or more Arch repositories, following\nthe branching and tagging across repositories. Note you should import \nseparate projects to separate GIT repositories.\n\nSupported\n - Imports, tags and simple commits.\n - File renames\n - Arch tags\n - Binary files\n - Large trees\n - Multiple repositories\n - Branches\n\nTODO:\n - Allow re-running the import on an already-imported project\n - Follow merges using Arch data\n - Audit shell escaping of filenames\n - Better support for file metadata\n - Better/safer creation of temp directories\n\nUnsupported:\n - Arch 'configuration'\n\nSigned-off-by: Martin Langhoff <martin@catalyst.net.nz>\n\n---\n\ndiff --git a/git-archimport-script b/git-archimport-script\nnew file mode 100755\n--- /dev/null\n+++ b/git-archimport-script\n@@ -0,0 +1,593 @@\n+#!/usr/bin/perl -w\n+#\n+# This tool is copyright (c) 2005, Martin Langhoff.\n+# It is released under the Gnu Public License, version 2.\n+#\n+# The basic idea is to walk the output of tla abrowse, \n+# fetch the changesets and apply them. \n+#\n+=head1 Invocation\n+\n+    git-archimport-script -i <archive>/<branch> [<archive>/<branch>]\n+    [ <archive>/<branch> ]\n+\n+    The script expects you to provide the key roots where it can start the\n+    import from an 'initial import' or 'tag' type of Arch commit. It will\n+    then follow all the branching and tagging within the provided roots.\n+\n+    It will die if it sees branches that have different roots. \n+\n+=head2 TODO\n+\n+ - keep track of merged patches, and mark a git merge when it happens\n+ - smarter rules to parse the archive history \"up\" and \"down\"\n+ - be able to continue an import where we left off\n+ - audit shell-escaping of filenames\n+\n+=head1 Devel tricks\n+\n+Add print in front of the shell commands invoked via backticks. \n+\n+=cut\n+\n+use strict;\n+use warnings;\n+use Getopt::Std;\n+use File::Spec;\n+use File::Temp qw(tempfile);\n+use File::Path qw(mkpath);\n+use File::Basename qw(basename dirname);\n+use String::ShellQuote;\n+use Time::Local;\n+use IO::Socket;\n+use IO::Pipe;\n+use POSIX qw(strftime dup2);\n+use Data::Dumper qw/ Dumper /;\n+use IPC::Open2;\n+\n+$SIG{'PIPE'}=\"IGNORE\";\n+$ENV{'TZ'}=\"UTC\";\n+\n+our($opt_h,$opt_v, $opt_T,\n+    $opt_C,$opt_t, $opt_i);\n+\n+sub usage() {\n+    print STDERR <<END;\n+Usage: ${\\basename $0}     # fetch/update GIT from Arch\n+       [ -h ] [ -v ] [ -i ] [ -T ] \n+       [ -C GIT_repository ] [ -t tempdir ] \n+       repository/arch-branch [ repository/arch-branch] ...\n+END\n+    exit(1);\n+}\n+\n+getopts(\"hviC:t:\") or usage();\n+usage if $opt_h;\n+\n+@ARGV >= 1 or usage();\n+my @arch_roots = @ARGV;\n+\n+my $tmp = $opt_t;\n+$tmp ||= '/tmp';\n+$tmp .= '/git-archimport/';\n+\n+my $git_tree = $opt_C;\n+$git_tree ||= \".\";\n+\n+\n+my @psets  = ();                # the collection\n+\n+foreach my $root (@arch_roots) {\n+    my ($arepo, $abranch) = split(m!/!, $root);\n+    open ABROWSE, \"tla abrowse -f -A $arepo --desc --merges $abranch |\" \n+        or die \"Problems with tla abrowse: $!\";\n+    \n+    my %ps        = ();         # the current one\n+    my $mode      = '';\n+    my $lastseen  = '';\n+    \n+    while (<ABROWSE>) {\n+        chomp;\n+        \n+        # first record padded w 8 spaces\n+        if (s/^\\s{8}\\b//) {\n+            \n+            # store the record we just captured\n+            if (%ps) {\n+                my %temp = %ps; # break references\n+                push (@psets, \\%temp);\n+                %ps = ();\n+            }\n+            \n+            my ($id, $type) = split(m/\\s{3}/, $_);\n+            $ps{id}   = $id;\n+            $ps{repo} = $arepo;\n+\n+            # deal with types\n+            if ($type =~ m/^\\(simple changeset\\)/) {\n+                $ps{type} = 's';\n+            } elsif ($type eq '(initial import)') {\n+                $ps{type} = 'i';\n+            } elsif ($type =~ m/^\\(tag revision of (.+)\\)/) {\n+                $ps{type} = 't';\n+                $ps{tag}  = $1;\n+            } else { \n+                warn \"Unknown type $type\";\n+            }\n+            $lastseen = 'id';\n+        }\n+        \n+        if (s/^\\s{10}//) { \n+            # 10 leading spaces or more \n+            # indicate commit metadata\n+            \n+            # date & author \n+            if ($lastseen eq 'id' && m/^\\d{4}-\\d{2}-\\d{2}/) {\n+                \n+                my ($date, $authoremail) = split(m/\\s{2,}/, $_);\n+                $ps{date}   = $date;\n+                $ps{date}   =~ s/\\bGMT$//; # strip off trailign GMT\n+                if ($ps{date} =~ m/\\b\\w+$/) {\n+                    warn 'Arch dates not in GMT?! - imported dates will be wrong';\n+                }\n+            \n+                $authoremail =~ m/^(.+)\\s(\\S+)$/;\n+                $ps{author} = $1;\n+                $ps{email}  = $2;\n+            \n+                $lastseen = 'date';\n+            \n+            } elsif ($lastseen eq 'date') {\n+                # the only hint is position\n+                # subject is after date\n+                $ps{subj} = $_;\n+                $lastseen = 'subj';\n+            \n+            } elsif ($lastseen eq 'subj' && $_ eq 'merges in:') {\n+                $ps{merges} = [];\n+                $lastseen = 'merges';\n+            \n+            } elsif ($lastseen eq 'merges' && s/^\\s{2}//) {\n+                push (@{$ps{merges}}, $_);\n+            } else {\n+                warn 'more metadata after merges!?';\n+            }\n+            \n+        }\n+    }\n+\n+    if (%ps) {\n+        my %temp = %ps;         # break references\n+        push (@psets, \\%temp);\n+        %ps = ();\n+    }    \n+    close ABROWSE;\n+}                               # end foreach $root\n+\n+## Order patches by time\n+@psets = sort {$a->{date}.$b->{id} cmp $b->{date}.$b->{id}} @psets;\n+\n+#print Dumper \\@psets;\n+\n+##\n+## TODO cleanup irrelevant patches\n+##      and put an initial import\n+##      or a full tag\n+\n+if ($opt_i) {                   # initial import \n+    if ($psets[0]{type} eq 'i' || $psets[0]{type} eq 't') {\n+        print \"Starting import from $psets[0]{id}\\n\";\n+    } else {\n+        die \"Need to start from an import or a tag -- cannot use $psets[0]{id}\";\n+    }\n+    `git-init-db`;\n+    die $! if $?;\n+}\n+\n+# process\n+my $lastbranch = branchname($psets[0]{id}); # only good for initial import\n+my $importseen = $opt_i ? 0 : 1; # start at 1 if opt_i\n+\n+foreach my $ps (@psets) {\n+\n+    $ps->{branch} =  branchname($ps->{id});\n+\n+    #\n+    # ensure we have a clean state \n+    # \n+    if (`git diff-files`) {\n+        die \"Unclean tree when about to process $ps->{id} \" .\n+            \" - did we fail to commit cleanly before?\";\n+    }\n+    die $! if $?;\n+\n+    # \n+    # create the branch if needed\n+    #\n+    if ($ps->{type} eq 'i' && $importseen) {\n+        die \"Should not have more than one 'Initial import' per GIT import\";\n+    }\n+\n+    unless ($opt_i && !$importseen) { # skip for first commit\n+        if ( -e \".git/refs/heads/$ps->{branch}\") {\n+            # we know about this branch\n+            `git checkout    $ps->{branch}`;\n+        } else {\n+            # new branch! we need to verify a few things\n+            die \"Branch on a non-tag!\" unless $ps->{type} eq 't';\n+            my $branchpoint = ptag($ps->{tag});\n+            die \"Tagging from unknown id unsupported: $ps->{tag}\" \n+                unless $branchpoint;\n+            \n+            # find where we are supposed to branch from\n+            `git checkout -b $ps->{branch} $branchpoint`;\n+        } \n+        die $! if $?;\n+    } \n+\n+        \n+    #\n+    # Apply the import/changeset/merge into the working tree\n+    # \n+    if ($ps->{type} eq 'i' || $ps->{type} eq 't') {\n+        $importseen = 1;\n+        apply_import($ps) or die $!;\n+    } elsif ($ps->{type} eq 's') {\n+        apply_cset($ps);\n+    }\n+\n+    #\n+    # prepare update git's index, based on what arch knows\n+    # about the pset, resolve parents, etc\n+    #\n+    my $tree;\n+    \n+    my $commitlog = `tla cat-archive-log -A $ps->{repo} $ps->{id}`; \n+    die \"Error in cat-archive-log: $!\" if $?;\n+        \n+    # parselog will git-add/rm files\n+    # and generally prepare things for the commit\n+    # NOTE: parselog will shell-quote filenames! \n+    my ($sum, $msg, $add, $del, $mod, $ren) = parselog($commitlog);\n+    my $logmessage = \"$sum\\n$msg\";\n+\n+\n+    # imports don't give us good info\n+    # on added files. Shame on them\n+    if ($ps->{type} eq 'i' || $ps->{type} eq 't') { \n+        `find . -type f -print0 | grep -zv '^./.git' | xargs -0 -l100 git-update-cache --add`;\n+        `git-ls-files --deleted -z | xargs --no-run-if-empty -0 -l100 git-update-cache --remove`; \n+    }\n+\n+    if (@$add) {\n+        while (@$add) {\n+            my @slice = splice(@$add, 0, 100);\n+            my $slice = join(' ', @slice);          \n+            `git-update-cache --add $slice`;\n+            die \"Error in git-update-cache --add: $!\" if $?;\n+        }\n+    }\n+    if (@$del) {\n+        foreach my $file (@$del) {\n+            unlink $file or die \"Problems deleting $file : $!\";\n+        }\n+        while (@$del) {\n+            my @slice = splice(@$del, 0, 100);\n+            my $slice = join(' ', @slice);\n+            `git-update-cache --remove $slice`;\n+            die \"Error in git-update-cache --remove: $!\" if $?;\n+        }\n+    }\n+    if (@$ren) {                # renamed\n+        if (@$ren % 2) {\n+            die \"Odd number of entries in rename!?\";\n+        }\n+        ;\n+        while (@$ren) {\n+            my $from = pop @$ren;\n+            my $to   = pop @$ren;           \n+\n+            unless (-d dirname($to)) {\n+                mkpath(dirname($to)); # will die on err\n+            }\n+            #print \"moving $from $to\";\n+            `mv $from $to`;\n+            die \"Error renaming $from $to : $!\" if $?;\n+            `git-update-cache --remove $from`;\n+            die \"Error in git-update-cache --remove: $!\" if $?;\n+            `git-update-cache --add $to`;\n+            die \"Error in git-update-cache --add: $!\" if $?;\n+        }\n+\n+    }\n+    if (@$mod) {                # must be _after_ renames\n+        while (@$mod) {\n+            my @slice = splice(@$mod, 0, 100);\n+            my $slice = join(' ', @slice);\n+            `git-update-cache $slice`;\n+            die \"Error in git-update-cache: $!\" if $?;\n+        }\n+    }\n+\n+    # warn \"errors when running git-update-cache! $!\";\n+    $tree = `git-write-tree`;\n+    die \"cannot write tree $!\" if $?;\n+    chomp $tree;\n+        \n+    \n+    #\n+    # Who's your daddy?\n+    #\n+    my @par;\n+    if ( -e \".git/refs/heads/$ps->{branch}\") {\n+        if (open HEAD, \"<.git/refs/heads/$ps->{branch}\") {\n+            my $p = <HEAD>;\n+            close HEAD;\n+            chomp $p;\n+            push @par, '-p', $p;\n+        } else { \n+            if ($ps->{type} eq 's') {\n+                warn \"Could not find the right head for the branch $ps->{branch}\";\n+            }\n+        }\n+    }\n+    \n+    my $par = join (' ', @par);\n+\n+    #    \n+    # Commit, tag and clean state\n+    #\n+    $ENV{TZ}                  = 'GMT';\n+    $ENV{GIT_AUTHOR_NAME}     = $ps->{author};\n+    $ENV{GIT_AUTHOR_EMAIL}    = $ps->{email};\n+    $ENV{GIT_AUTHOR_DATE}     = $ps->{date};\n+    $ENV{GIT_COMMITTER_NAME}  = $ps->{author};\n+    $ENV{GIT_COMMITTER_EMAIL} = $ps->{email};\n+    $ENV{GIT_COMMITTER_DATE}  = $ps->{date};\n+\n+    my ($pid, $commit_rh, $commit_wh);\n+    $commit_rh = 'commit_rh';\n+    $commit_wh = 'commit_wh';\n+    \n+    $pid = open2(*READER, *WRITER, \"git-commit-tree $tree $par\") \n+        or die $!;\n+    print WRITER $logmessage;   # write\n+    close WRITER;\n+    my $commitid = <READER>;    # read\n+    chomp $commitid;\n+    close READER;\n+    waitpid $pid,0;             # close;\n+\n+    if (length $commitid != 40) {\n+        die \"Something went wrong with the commit! $! $commitid\";\n+    }\n+    #\n+    # Update the branch\n+    # \n+    open  HEAD, \">.git/refs/heads/$ps->{branch}\";\n+    print HEAD $commitid;\n+    close HEAD;\n+    unlink ('.git/HEAD');\n+    symlink(\"refs/heads/$ps->{branch}\",\".git/HEAD\");\n+\n+    # tag accordingly\n+    ptag($ps->{id}, $commitid); # private tag\n+    if ($opt_T || $ps->{type} eq 't' || $ps->{type} eq 'i') {\n+        tag($ps->{id}, $commitid);\n+    }\n+    print \" * Committed $ps->{id}\\n\";\n+    print \"   + tree   $tree\\n\";\n+    print \"   + commit $commitid\\n\";\n+    # print \"   + commit date is  $ps->{date} \\n\";\n+}\n+\n+sub branchname {\n+    my $id = shift;\n+    $id =~ s#^.+?/##;\n+    my @parts = split(m/--/, $id);\n+    return join('--', @parts[0..1]);\n+}\n+\n+sub apply_import {\n+    my $ps = shift;\n+    my $bname = branchname($ps->{id});\n+\n+    `mkdir -p $tmp`;\n+\n+    `tla get -s --no-pristine -A $ps->{repo} $ps->{id} $tmp/import`;\n+    die \"Cannot get import: $!\" if $?;    \n+    `rsync -v --archive --delete --exclude '.git' --exclude '.arch-ids' --exclude '{arch}' $tmp/import/* ./`;\n+    die \"Cannot rsync import:$!\" if $?;\n+    \n+    `rm -fr $tmp/import`;\n+    die \"Cannot remove tempdir: $!\" if $?;\n+    \n+\n+    return 1;\n+}\n+\n+sub apply_cset {\n+    my $ps = shift;\n+\n+    `mkdir -p $tmp`;\n+\n+    # get the changeset\n+    `tla get-changeset  -A $ps->{repo} $ps->{id} $tmp/changeset`;\n+    die \"Cannot get changeset: $!\" if $?;\n+    \n+    # apply patches\n+    if (`find $tmp/changeset/patches -type f -name '*.patch'`) {\n+        # this can be sped up considerably by doing\n+        #    (find | xargs cat) | patch\n+        # but that cna get mucked up by patches\n+        # with missing trailing newlines or the standard \n+        # 'missing newline' flag in the patch - possibly\n+        # produced with an old/buggy diff.\n+        # slow and safe, we invoke patch once per patchfile\n+        `find $tmp/changeset/patches -type f -name '*.patch' -print0 | grep -zv '{arch}' | xargs -iFILE -0 --no-run-if-empty patch -p1 --forward -iFILE`;\n+        die \"Problem applying patches! $!\" if $?;\n+    }\n+\n+    # apply changed binary files\n+    if (my @modified = `find $tmp/changeset/patches -type f -name '*.modified'`) {\n+        foreach my $mod (@modified) {\n+            chomp $mod;\n+            my $orig = $mod;\n+            $orig =~ s/\\.modified$//; # lazy\n+            $orig =~ s!^\\Q$tmp\\E/changeset/patches/!!;\n+            #print \"rsync -p '$mod' '$orig'\";\n+            `rsync -p $mod ./$orig`;\n+            die \"Problem applying binary changes! $!\" if $?;\n+        }\n+    }\n+\n+    # bring in new files\n+    `rsync --archive --exclude '.git' --exclude '.arch-ids' --exclude '{arch}' $tmp/changeset/new-files-archive/* ./`;\n+\n+    # deleted files are hinted from the commitlog processing\n+\n+    `rm -fr $tmp/changeset`;\n+}\n+\n+\n+# =for reference\n+# A log entry looks like \n+# Revision: moodle-org--moodle--1.3.3--patch-15\n+# Archive: arch-eduforge@catalyst.net.nz--2004\n+# Creator: Penny Leach <penny@catalyst.net.nz>\n+# Date: Wed May 25 14:15:34 NZST 2005\n+# Standard-date: 2005-05-25 02:15:34 GMT\n+# New-files: lang/de/.arch-ids/block_glossary_random.php.id\n+#     lang/de/.arch-ids/block_html.php.id\n+# New-directories: lang/de/help/questionnaire\n+#     lang/de/help/questionnaire/.arch-ids\n+# Renamed-files: .arch-ids/db_sears.sql.id db/.arch-ids/db_sears.sql.id\n+#    db_sears.sql db/db_sears.sql\n+# Removed-files: lang/be/docs/.arch-ids/release.html.id\n+#     lang/be/docs/.arch-ids/releaseold.html.id\n+# Modified-files: admin/cron.php admin/delete.php\n+#     admin/editor.html backup/lib.php backup/restore.php\n+# New-patches: arch-eduforge@catalyst.net.nz--2004/moodle-org--moodle--1.3.3--patch-15\n+# Summary: Updating to latest from MOODLE_14_STABLE (1.4.5+)\n+# Keywords:\n+#\n+# Updating yadda tadda tadda madda\n+sub parselog {\n+    my $log = shift;\n+    #print $log;\n+\n+    my (@add, @del, @mod, @ren, @kw, $sum, $msg );\n+\n+    if ($log =~ m/(?:\\n|^)New-files:(.*?)(?=\\n\\w)/s ) {\n+        my $files = $1;\n+        @add = split(m/\\s+/s, $files);\n+    }\n+       \n+    if ($log =~ m/(?:\\n|^)Removed-files:(.*?)(?=\\n\\w)/s ) {\n+        my $files = $1;\n+        @del = split(m/\\s+/s, $files);\n+    }\n+    \n+    if ($log =~ m/(?:\\n|^)Modified-files:(.*?)(?=\\n\\w)/s ) {\n+        my $files = $1;\n+        @mod = split(m/\\s+/s, $files);\n+    }\n+    \n+    if ($log =~ m/(?:\\n|^)Renamed-files:(.*?)(?=\\n\\w)/s ) {\n+        my $files = $1;\n+        @ren = split(m/\\s+/s, $files);\n+    }\n+\n+    $sum ='';\n+    if ($log =~ m/^Summary:(.+?)$/m ) {\n+        $sum = $1;\n+        $sum =~ s/^\\s+//;\n+        $sum =~ s/\\s+$//;\n+    }\n+\n+    $msg = '';\n+    if ($log =~ m/\\n\\n(.+)$/s) {\n+        $msg = $1;\n+        $msg =~ s/^\\s+//;\n+        $msg =~ s/\\s+$//;\n+    }\n+\n+\n+    # cleanup the arrays\n+    foreach my $ref ( (\\@add, \\@del, \\@mod, \\@ren) ) {\n+        my @tmp = ();\n+        while (my $t = pop @$ref) {\n+            next unless length ($t);\n+            next if $t =~ m!\\{arch\\}/!;\n+            next if $t =~ m!\\.arch-ids/!;\n+            next if $t =~ m!\\.arch-inventory$!;\n+            push (@tmp, shell_quote($t));\n+        }\n+        @$ref = @tmp;\n+    }\n+    \n+    #print Dumper [$sum, $msg, \\@add, \\@del, \\@mod, \\@ren]; \n+    return       ($sum, $msg, \\@add, \\@del, \\@mod, \\@ren); \n+}\n+\n+# write/read a tag\n+sub tag {\n+    my ($tag, $commit) = @_;\n+    $tag =~ s|/|--|g; \n+    $tag = shell_quote($tag);\n+    \n+    if ($commit) {\n+        open(C,\">.git/refs/tags/$tag\")\n+            or die \"Cannot create tag $tag: $!\\n\";\n+        print C \"$commit\\n\"\n+            or die \"Cannot write tag $tag: $!\\n\";\n+        close(C)\n+            or die \"Cannot write tag $tag: $!\\n\";\n+        print \"Created tag '$tag' on '$commit'\\n\" if $opt_v;\n+    } else {                    # read\n+        open(C,\"<.git/refs/tags/$tag\")\n+            or die \"Cannot read tag $tag: $!\\n\";\n+        $commit = <C>;\n+        chomp $commit;\n+        die \"Error reading tag $tag: $!\\n\" unless length $commit == 40;\n+        close(C)\n+            or die \"Cannot read tag $tag: $!\\n\";\n+        return $commit;\n+    }\n+}\n+\n+# write/read a private tag\n+# reads fail softly if the tag isn't there\n+sub ptag {\n+    my ($tag, $commit) = @_;\n+    $tag =~ s|/|--|g; \n+    $tag = shell_quote($tag);\n+    \n+    unless (-d '.git/archimport/tags') {\n+        mkpath('.git/archimport/tags');\n+    }\n+\n+    if ($commit) {              # write\n+        open(C,\">.git/archimport/tags/$tag\")\n+            or die \"Cannot create tag $tag: $!\\n\";\n+        print C \"$commit\\n\"\n+            or die \"Cannot write tag $tag: $!\\n\";\n+        close(C)\n+            or die \"Cannot write tag $tag: $!\\n\";\n+    } else {                    # read\n+        # if the tag isn't there, return 0\n+        unless ( -s \".git/archimport/tags/$tag\") {\n+            warn \"Could not find tag $tag -- perhaps it isn't in the repos we have?\\n\" \n+                if $opt_v;\n+            return 0;\n+        }\n+        open(C,\"<.git/archimport/tags/$tag\")\n+            or die \"Cannot read tag $tag: $!\\n\";\n+        $commit = <C>;\n+        chomp $commit;\n+        die \"Error reading tag $tag: $!\\n\" unless length $commit == 40;\n+        close(C)\n+            or die \"Cannot read tag $tag: $!\\n\";\n+        return $commit;\n+    }\n+}\n"}]}