# [PATCH] gitweb.cgi: Use File::MMagic; "a=blob" action knows the blob/file type 3 messages from 2006-07-08 to 2006-07-09. Participants: Luben Tuikov, Junio C Hamano. Thread: https://gitlist.dev/t/4801 ## Luben Tuikov, 2006-07-08 04:10 Subject: [PATCH] gitweb.cgi: Use File::MMagic; "a=blob" action knows the blob/file type Message-ID: <20060708041021.24704.qmail@web31804.mail.mud.yahoo.com> URL: https://gitlist.dev/e/20060708041021.24704.qmail%40web31804.mail.mud.yahoo.com ``` Use File::MMagic to determine the MIME type of a blob/file. The variable magic_mime_file holds the location of the "magic.mime" file, usually "/usr/share/file/magic.mime". If not defined, the magic numbers internally stored in the File::MMagic module are used. Action "blob" knows the file type: if the file type is not "text/*" then action "blob" defaults to "blob_plain", i.e. the file is downloaded raw for the browser to interpret. If the file type is "text/*", then "blob" defaults to the current "cat -n"-like output, from which you can click "plain", to get the "blob_plain" output. Signed-off-by: Luben Tuikov --- gitweb/gitweb.cgi | 140 +++++++++++++++++++++-------------------------------- 1 files changed, 56 insertions(+), 84 deletions(-) diff --git a/gitweb/gitweb.cgi b/gitweb/gitweb.cgi index cce0753..6798990 100755 --- a/gitweb/gitweb.cgi +++ b/gitweb/gitweb.cgi @@ -14,6 +14,8 @@ use CGI::Util qw(unescape); use CGI::Carp qw(fatalsToBrowser); use Encode; use Fcntl ':mode'; +use File::MMagic; +use FileHandle; binmode STDOUT, ':utf8'; our $cgi = new CGI; @@ -54,9 +56,15 @@ #our $projects_list = $projectroot; our $projects_list = "index/index.aux"; # default blob_plain mimetype and default charset for text/plain blob -our $default_blob_plain_mimetype = 'text/plain'; +our $default_blob_mimetype = 'text/plain'; our $default_text_plain_charset = undef; +# magic_mime_file: if defined this file will be used by File::MMagic +# to guess the file type, else the magic numbers stored internally +# in File::MMagic will be used. Either relative or absolute name +# can be given. E.g. "/usr/share/file/magic.mime". +our $magic_mime_file = undef; + # file to use for guessing MIME types before trying /etc/mime.types # (relative to the current git repository) our $mimetypes_file = undef; @@ -1455,11 +1463,58 @@ sub git_get_hash_by_path { } } +# +# Strangely enough the File::MMagic package, version 1.27, has a bug +# whereby reading from a piped filehandle (e.g. STDIN, or "-|") always +# returns 'text/plain', but reading from a file on a file system (as it +# would be the case for the checktype_filename() method) properly +# determines the file type. +# +sub get_blob_mimetype { + my $blob_file = "$git_temp/blob-$hash"; + if (! -r $blob_file) { + open my $fd_in, "-|", "$gitbin/git-cat-file blob $hash" or return $default_blob_mimetype; + open my $fd_out, "> $blob_file"; + my @file = <$fd_in>; + print $fd_out @file; + close $fd_out; + close $fd_in; + } + my $mm = $magic_mime_file ? File::MMagic->new($magic_mime_file) : new File::MMagic; + my $mime = $mm->checktype_filename($blob_file); + return $mime; +} + +sub git_blob_plain { + open my $fd, "-|", "$gitbin/git-cat-file blob $hash" or return; + my $mimetype = get_blob_mimetype(); + + # save as filename, even when no $file_name is given + my $save_as = "$hash"; + if (defined $file_name) { + $save_as = $file_name; + } elsif ($mimetype =~ m/^text\//) { + $save_as .= '.txt'; + } + + print $cgi->header(-type => "$mimetype", '-content-disposition' => "inline; filename=\"$save_as\""); + undef $/; + binmode STDOUT, ':raw'; + print <$fd>; + binmode STDOUT, ':utf8'; # as set at the beginning of gitweb.cgi + $/ = "\n"; + close $fd; +} + sub git_blob { if (!defined $hash && defined $file_name) { my $base = $hash_base || git_read_head($project); $hash = git_get_hash_by_path($base, $file_name, "blob") || die_error(undef, "Error lookup file."); } + my $mimetype = get_blob_mimetype(); + if ($mimetype !~ m/^text\//) { + return git_blob_plain(); + } my $have_blame = git_get_project_config_bool ('blame'); open my $fd, "-|", "$gitbin/git-cat-file blob $hash" or die_error(undef, "Open failed."); git_header_html(); @@ -1510,89 +1565,6 @@ sub git_blob { git_footer_html(); } -sub mimetype_guess_file { - my $filename = shift; - my $mimemap = shift; - -r $mimemap or return undef; - - my %mimemap; - open(MIME, $mimemap) or return undef; - while () { - my ($mime, $exts) = split(/\t+/); - my @exts = split(/\s+/, $exts); - foreach my $ext (@exts) { - $mimemap{$ext} = $mime; - } - } - close(MIME); - - $filename =~ /\.(.*?)$/; - return $mimemap{$1}; -} - -sub mimetype_guess { - my $filename = shift; - my $mime; - $filename =~ /\./ or return undef; - - if ($mimetypes_file) { - my $file = $mimetypes_file; - #$file =~ m#^/# or $file = "$projectroot/$path/$file"; - $mime = mimetype_guess_file($filename, $file); - } - $mime ||= mimetype_guess_file($filename, '/etc/mime.types'); - return $mime; -} - -sub git_blob_plain_mimetype { - my $fd = shift; - my $filename = shift; - - # just in case - return $default_blob_plain_mimetype unless $fd; - - if ($filename) { - my $mime = mimetype_guess($filename); - $mime and return $mime; - } - - if (-T $fd) { - return 'text/plain' . - ($default_text_plain_charset ? '; charset='.$default_text_plain_charset : ''); - } elsif (! $filename) { - return 'application/octet-stream'; - } elsif ($filename =~ m/\.png$/i) { - return 'image/png'; - } elsif ($filename =~ m/\.gif$/i) { - return 'image/gif'; - } elsif ($filename =~ m/\.jpe?g$/i) { - return 'image/jpeg'; - } else { - return 'application/octet-stream'; - } -} - -sub git_blob_plain { - open my $fd, "-|", "$gitbin/git-cat-file blob $hash" or return; - my $type = git_blob_plain_mimetype($fd, $file_name); - - # save as filename, even when no $file_name is given - my $save_as = "$hash"; - if (defined $file_name) { - $save_as = $file_name; - } elsif ($type =~ m/^text\//) { - $save_as .= '.txt'; - } - - print $cgi->header(-type => "$type", '-content-disposition' => "inline; filename=\"$save_as\""); - undef $/; - binmode STDOUT, ':raw'; - print <$fd>; - binmode STDOUT, ':utf8'; # as set at the beginning of gitweb.cgi - $/ = "\n"; - close $fd; -} - sub git_tree { if (!defined $hash) { $hash = git_read_head($project); -- 1.4.1.g2f3c ``` ## Junio C Hamano, 2006-07-08 06:18 Subject: Re: [PATCH] gitweb.cgi: Use File::MMagic; "a=blob" action knows the blob/file type Message-ID: <7vzmfksjpe.fsf@assigned-by-dhcp.cox.net> URL: https://gitlist.dev/e/7vzmfksjpe.fsf%40assigned-by-dhcp.cox.net In-Reply-To: <20060708041021.24704.qmail@web31804.mail.mud.yahoo.com> ``` Luben Tuikov writes: > Use File::MMagic to determine the MIME type of a blob/file. > The variable magic_mime_file holds the location of the > "magic.mime" file, usually "/usr/share/file/magic.mime". > If not defined, the magic numbers internally stored in the > File::MMagic module are used. I am sorry to ask you this, but would you mind redoing this patch without File::MMagic bits? I think giving "a=blob" an ability to automatically switch to git_blob_plain is a good addition (as is your earlier patch to give a direct link to reach blob_plain from the list), so let's have that part in first. I haven't applied your earlier one but it will appear in "next" shortly. Existing filename based mimetypes_guess should be a lot cheaper than exploding a blob and feeding it to File::MMagic. I was hoping File::MMagic to be used when we cannot guess the content type that way (i.e. when mimetypes_guess returns undef or application/octet-stream). Since the repository owner can correct misidentification by the standard /etc/mime.types by supplying a custom per-repository $mimetypes_file (modulo that the current implementation of mimetype_guess_file does not allow it if the file does not have an extension that is specific enough), File::MMagic might be an overkill, especially if used in the way this patch does. To allow finer grained differentiation that cannot be done with file extensions alone (e.g. some files may have .dat extension but one can be VCD mpeg wrapped in RIFF, and another can be a Z-machine story file), it might be simpler to allow the repository owner to specify full $file_name for such an ambiguous file in their custom $mimetypes_file, and try to match it in mimetype_guess_file sub. That way we may not even need to use File::MMagic. Are there cases where only $hash is given without $file_name? If so we may need to fall back on File::MMagic in such a case after all, but get_blob_mimetype sub copies the whole blob to a temporary file to work around a problem with version 1.27 you state in the comment -- this is way too much (and nobody seems to clean up the tempfile). Looking at magic.mime, I suspect we might be able to get away with the first 4k bytes or so at most (the largest offset except iso9660 image is "Biff5" appearing at 2114 to signal an Excel spreadsheet). ``` ## Luben Tuikov, 2006-07-09 01:17 Subject: Re: [PATCH] gitweb.cgi: Use File::MMagic; "a=blob" action knows the blob/file type Message-ID: <20060709011756.92415.qmail@web31815.mail.mud.yahoo.com> URL: https://gitlist.dev/e/20060709011756.92415.qmail%40web31815.mail.mud.yahoo.com In-Reply-To: <7vzmfksjpe.fsf@assigned-by-dhcp.cox.net> ``` --- Junio C Hamano wrote: > Luben Tuikov writes: > > > Use File::MMagic to determine the MIME type of a blob/file. > > The variable magic_mime_file holds the location of the > > "magic.mime" file, usually "/usr/share/file/magic.mime". > > If not defined, the magic numbers internally stored in the > > File::MMagic module are used. > > I am sorry to ask you this, but would you mind redoing this > patch without File::MMagic bits? Yeah, no problem, will do. > I think giving "a=blob" an > ability to automatically switch to git_blob_plain is a good > addition (as is your earlier patch to give a direct link to > reach blob_plain from the list), so let's have that part in > first. Ok. > I haven't applied your earlier one but it will appear in > "next" shortly. Ok. > Existing filename based mimetypes_guess should be a lot cheaper > than exploding a blob and feeding it to File::MMagic. I was > hoping File::MMagic to be used when we cannot guess the content > type that way (i.e. when mimetypes_guess returns undef or > application/octet-stream). The MIME guessing is used in git_blob_plain(), where the blob is already exploded, since a file descriptor is passed to the MIME guessing routines. Initially I tried using the already opened file descriptor to the blob, but File::MMagic v1.27 always returns "text/plain" on that. I was astounded by this and a 5 line perl script using that same module confirmed my suspicion that the module or "seek" is broken in perl or in FileHandle. I.e. checktype_filename() succeeds, but checktype_filehandle() always returns "text/plain". Had simply reading the file descriptor succeeded, then we'd not need to explode blobs in $git_temp. The other reason I decided to completely go with File::MMagic is that if I'm going to use it to decide the mime type after the default method didn't succeed, why not forget the default method and just use File::MMagic in general -- seemed like a logical simplification. > Since the repository owner can correct misidentification by the > standard /etc/mime.types by supplying a custom per-repository > $mimetypes_file (modulo that the current implementation of > mimetype_guess_file does not allow it if the file does not have > an extension that is specific enough), File::MMagic might be an > overkill, especially if used in the way this patch does. To > allow finer grained differentiation that cannot be done with > file extensions alone (e.g. some files may have .dat extension > but one can be VCD mpeg wrapped in RIFF, and another can be a > Z-machine story file), it might be simpler to allow the > repository owner to specify full $file_name for such an ambiguous > file in their custom $mimetypes_file, and try to match it in > mimetype_guess_file sub. That way we may not even need to use > File::MMagic. This is true, but I wonder how many people are going to go and create their own mime.type files. Most of the time people would do minimal gitweb.cgi changes not even specifying $mimetypes_file nor even $magic_mime_file. > Are there cases where only $hash is given without $file_name? > If so we may need to fall back on File::MMagic in such a case > after all, but get_blob_mimetype sub copies the whole blob to a > temporary file to work around a problem with version 1.27 you > state in the comment -- this is way too much (and nobody seems > to clean up the tempfile). Looking at magic.mime, I suspect we > might be able to get away with the first 4k bytes or so at most > (the largest offset except iso9660 image is "Biff5" appearing at > 2114 to signal an Excel spreadsheet). I was hoping that tmpwatch(8) would clean up the blobs in Linux/UNIX. But we can also delete them after the lookup. Anothing thing is that getting a blob isn't that often and when it happens, after the MIME lookup it is already in the pagecache. Luben ```