Implements a gate between git and mediawiki, allowing git users to push and pull objects from mediawiki just as one would do with a classic git repository thanks to remote-helpers.
Currently supported commands are :
git clone mediawiki::http://onewiki.com
git pullYou need the following packages installed (available on common
repositories):
libmediawiki-api-perl
libdatetime-format-iso8601-perlRemote helpers have been used in order to be as transparent as possible to the git user.
Mediawiki revisions are downloaded through the Mediawiki API and then fast-imported into git.
Mediawiki revisions and git commits are linked thanks to notes bound to commits.
The import part is done on a refs/mediawiki/<remote> branch before coming to refs/remote/origin/master (Huge thanks to Jonathan Nieder for his help)
For now, the whole wiki is cloned, but it will be possible to clone only some pages: the clone is based on a list of pages which is now all pages.
Signed-off-by: Jérémie Nikaes <jeremie.nikaes@ensimag.imag.fr> Signed-off-by: Arnaud Lacurie <arnaud.lacurie@ensimag.imag.fr> Signed-off-by: Claire Fousse <claire.fousse@ensimag.imag.fr> Signed-off-by: David Amouyal <david.amouyal@ensimag.imag.fr> Signed-off-by: Matthieu Moy <matthieu.moy@grenoble-inp.fr> Signed-off-by: Sylvain Boulmé <sylvain.boulme@imag.fr> --- contrib/mw-to-git/git-remote-mediawiki | 252 ++++++++++++++++++++++++++++ contrib/mw-to-git/git-remote-mediawiki.txt | 7 + 2 files changed, 259 insertions(+), 0 deletions(-) create mode 100755 contrib/mw-to-git/git-remote-mediawiki create mode 100644 contrib/mw-to-git/git-remote-mediawiki.txt
Show changes to 2 files +259 −0
contrib/mw-to-git/git-remote-mediawiki, contrib/mw-to-git/git-remote-mediawiki.txt
diff --git a/contrib/mw-to-git/git-remote-mediawiki b/contrib/mw-to-git/git-remote-mediawiki new file mode 100755 index 0000000..f20edc4 --- /dev/null +++ b/contrib/mw-to-git/git-remote-mediawiki @@ -0,0 +1,252 @@ +#! /usr/bin/perl + +use strict; +use Switch; +use MediaWiki::API; +use Storable qw(freeze thaw); +use DateTime::Format::ISO8601; +use Encode qw(encode_utf8); + +my $remotename = $ARGV[0]; +my $url = $ARGV[1]; + +print STDERR "$url\n"; + +# commands parser +my $loop = 1; +my $entry; +my @cmd; +while ($loop) { + $| = 1; #flush STDOUT + $entry = <STDIN>; + print STDERR $entry; + chomp($entry); + @cmd = undef; + @cmd = split(/ /,$entry); + switch ($cmd[0]) { + case "capabilities" { + if ($cmd[1] eq "") { + mw_capabilities(); + } else { + $loop = 0; + } + } + case "list" { + if ($cmd[2] eq "") { + mw_list($cmd[1]); + } else { + $loop = 0; + } + } + case "import" { + if ($cmd[1] ne "" && $cmd[2] eq "") { + mw_import($url); + } else { + $loop = 0; + } + } + case "option" { + mw_option($cmd[1],$cmd[2]); + } + case "push" { + #check the pattern +<src>:<dist> + my @pushargs = split(/:/,$cmd[1]); + if ($pushargs[1] ne "" && $pushargs[2] eq "" + && (substr($pushargs[0],0,1) eq "+")) { + mw_push(substr($pushargs[0],1),$pushargs[1]); + } else { + $loop = 0; + } + } else { + $loop = 0; + } + } + close(FILE); +} + +########################## Functions ############################## + +sub get_last_local_revision { + # Get last commit sha1 + my $commit_sha1 = `git rev-parse refs/mediawiki/$remotename/master 2>/dev/null`; + + # Get note regarding that commit + chomp($commit_sha1); + my $note = `git notes show $commit_sha1 2>/dev/null`; + my @note_info = split(/ /, $note); + + my $lastrevision_number; + if (!($note_info[0] eq "mediawiki_revision:")) { + print STDERR "No previous mediawiki revision found, fetching from beginning\n"; + $lastrevision_number = 0; + } else { + # Notes are formatted : mediawiki_revision: #number + $lastrevision_number = $note_info[1]; + chomp($lastrevision_number); + print STDERR "Last mediawiki revision found is $lastrevision_number, fetching from here\n"; + } + return $lastrevision_number; +} + +sub mw_capabilities { + # Revisions are imported to the private namespace + # refs/mediawiki/$remotename/ by the helper and fetched into + # refs/remotes/$remotename later by fetch. + print STDOUT "refspec refs/heads/*:refs/mediawiki/$remotename/*\n"; + print STDOUT "import\n"; + print STDOUT "list\n"; + print STDOUT "option\n"; + print STDOUT "push\n"; + print STDOUT "\n"; +} + +sub mw_list { + # MediaWiki do not have branches, we consider one branch arbitrarily + # called master + print STDOUT "? refs/heads/master\n"; + print STDOUT '@'."refs/heads/master HEAD\n"; + print STDOUT "\n"; + +} + +sub mw_option { + print STDERR "not yet implemented \n"; + print STDOUT "unsupported\n"; +} + +sub mw_import { + my @wiki_name = split(/:\/\//,$url); + my $wiki_name = $wiki_name[1]; + + my $mediawiki = MediaWiki::API->new; + $mediawiki->{config}->{api_url} = "$url/api.php"; + + my $pages = $mediawiki->list({ + action => 'query', + list => 'allpages', + aplimit => 500, + }); + if ($pages == undef) { + print STDERR "fatal: '$url' does not appear to be a mediawiki\n"; + print STDERR "fatal: make sure '$url/api.php' is a valid page\n"; + exit; + } + + my @revisions; + print STDERR "Searching revisions...\n"; + my $fetch_from = get_last_local_revision() + 1; + my $n = 1; + foreach my $page (@$pages) { + my $id = $page->{pageid}; + + print STDERR "$n/", scalar(@$pages), ": $page->{title}\n"; + $n++; + + my $query = { + action => 'query', + prop => 'revisions', + rvprop => 'ids', + rvdir => 'newer', + rvstartid => $fetch_from, + rvlimit => 500, + pageids => $page->{pageid}, + }; + + my $revnum = 0; + # Get 500 revisions at a time due to the mediawiki api limit + while (1) { + my $result = $mediawiki->api($query); + + # Parse each of those 500 revisions + foreach my $revision (@{$result->{query}->{pages}->{$id}->{revisions}}) { + my $page_rev_ids; + $page_rev_ids->{pageid} = $page->{pageid}; + $page_rev_ids->{revid} = $revision->{revid}; + push (@revisions, $page_rev_ids); + $revnum++; + } + last unless $result->{'query-continue'}; + $query->{rvstartid} = $result->{'query-continue'}->{revisions}->{rvstartid}; + print "\n"; + } + print STDERR " Found ", $revnum, " revision(s).\n"; + } + + # Creation of the fast-import stream + print STDERR "Fetching & writing export data...\n"; + binmode STDOUT, ':binary'; + $n = 0; + + foreach my $pagerevids (sort {$a->{revid} <=> $b->{revid}} @revisions) { + #fetch the content of the pages + my $query = { + action => 'query', + prop => 'revisions', + rvprop => 'content|timestamp|comment|user|ids', + revids => $pagerevids->{revid}, + }; + + my $result = $mediawiki->api($query); + + my $rev = pop(@{$result->{query}->{pages}->{$pagerevids->{pageid}}->{revisions}}); + + $n++; + my $user = $rev->{user} || 'Anonymous'; + my $dt = DateTime::Format::ISO8601->parse_datetime($rev->{timestamp}); + + my $comment = defined $rev->{comment} ? $rev->{comment} : '*Empty MediaWiki Message*'; + my $title = $result->{query}->{pages}->{$pagerevids->{pageid}}->{title}; + my $content = $rev->{'*'}; + $title =~ y/ /_/; + + print STDERR "$n/", scalar(@revisions), ": Revision n°$pagerevids->{revid} of $title\n"; + + print "commit refs/mediawiki/$remotename/master\n"; + print "mark :$n\n"; + print "committer $user <$user\@$wiki_name> ", $dt->epoch, " +0000\n"; + print "data ", bytes::length(encode_utf8($comment)), "\n", encode_utf8($comment); + # If it's not a clone, needs to know where to start from + if ($fetch_from != 1 && $n == 1) { + print "from refs/mediawiki/$remotename/master^0\n"; + } + print "M 644 inline $title.wiki\n"; + print "data ", bytes::length(encode_utf8($content)), "\n", encode_utf8($content); + print "\n\n"; + + + # mediawiki revision number in the git note + my $note_comment = encode_utf8("note added by git-mediawiki"); + my $note_comment_length = bytes::length($note_comment); + my $note_content = encode_utf8("mediawiki_revision: " . $pagerevids->{revid} . "\n"); + my $note_content_length = bytes::length($note_content); + + if ($fetch_from == 1 && $n == 1) { + print "reset refs/notes/commits\n"; + } + print "commit refs/notes/commits\n"; + print "committer $user <user\@example.com> ", $dt->epoch, " +0000\n"; + print "data ", $note_comment_length, "\n", $note_comment; + if ($fetch_from != 1 && $n == 1) { + print "from refs/notes/commits^0\n"; + } + print "N inline :$n\n"; + print "data ", $note_content_length, "\n", $note_content; + print "\n\n"; + } + + if ($fetch_from == 1) { + if ($n != 0) { + print "reset refs/heads/master\n"; + print "from :$n\n"; + } else { + print STDERR "You appear to have cloned an empty mediawiki\n"; + #What do we have to do here ? If nothing is done, an error is thrown saying that + #HEAD is refering to unknown object 0000000000000000000 + } + } + +} + +sub mw_push { + print STDERR "not yet implemented \n"; +} diff --git a/contrib/mw-to-git/git-remote-mediawiki.txt b/contrib/mw-to-git/git-remote-mediawiki.txt new file mode 100644 index 0000000..4d211f5 --- /dev/null +++ b/contrib/mw-to-git/git-remote-mediawiki.txt @@ -0,0 +1,7 @@ +Git-Mediawiki is a project which aims the creation of a gate +between git and mediawiki, allowing git users to push and pull +objects from mediawiki just as one would do with a classic git +repository thanks to remote-helpers. + +For more information, visit the wiki at +https://github.com/Bibzball/Git-Mediawiki/wiki
-- 1.7.5.3.486.gdc5865
