{"thread":{"id":"22857","subject":"[PATCH/RFC 1/2 v2] gitweb: Syntax highlighting support","startedAt":"2010-02-28T20:28:09Z","lastAt":"2010-02-28T20:28:10Z","messageCount":2,"participants":["Jakub Narebski"],"isPatch":true,"patchVersion":2,"patchTotal":2},"messages":[{"id":"135894","messageId":"1267388890-30305-1-git-send-email-jnareb@gmail.com","threadId":"22857","inReplyTo":null,"subject":"[PATCH/RFC 1/2 v2] gitweb: Syntax highlighting support","fromName":"Jakub Narebski","fromEmail":"jnareb@gmail.com","sentAt":"2010-02-28T20:28:09Z","receivedAt":"2010-02-28T20:28:09Z","isPatch":true,"sender":{"key":"jnareb@gmail.com","avatar":"https://avatars.githubusercontent.com/u/2706?v=4"},"body":"From: Johannes Schindelin <johannes.schindelin@gmx.de>\n\nIt requires the 'highlight' program to do all the heavy-lifting.\n\nThis is loosely based on Daniel Svensson's and Sham Chukoury's work in\ngitweb-xmms2.git (it cannot be cherry-picked, as gitweb-xmms2 first forked\nwildly, then not contributed back, and then went stale).\n\n[jn: cherry picked from bc1ed6aafd9ee4937559535c66c8bddf1864bec6\n in http://repo.or.cz/w/git/dscho.git, with a few changes]\n\nSigned-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>\nSigned-off-by: Jakub Narebski <jnareb@gmail.com>\n---\nCompared to the version in Dscho repository, it makes gitweb do line\nnumbering by itself, instead of making 'highlight' do it... and then\nrewriting it to conform to the rest of gitweb.  It also leaves tab\nexpansion to gitweb, for 'blob' view to look the same with and without\nsyntax hightlighting enabled.\n\nThe list of supported extensions is widened, and syntax is decided\nbased on basename, not on whole pathname (which might be a mistake).\nAlso the '.in' suffix is stripped if it is present (so 'git.spec.in'\nfile would use 'spec' syntax).\n\nNote that gitweb.css contains default 'highlight' style, but perhaps\nother style would be better fit for deafule gitweb CSS.\n\n gitweb/gitweb.css  |   18 ++++++++++++++\n gitweb/gitweb.perl |   65 +++++++++++++++++++++++++++++++++++++++++++++++++++-\n 2 files changed, 82 insertions(+), 1 deletions(-)\n\ndiff --git a/gitweb/gitweb.css b/gitweb/gitweb.css\nindex 50067f2..4132aab 100644\n--- a/gitweb/gitweb.css\n+++ b/gitweb/gitweb.css\n@@ -572,3 +572,21 @@ span.match {\n div.binary {\n \tfont-style: italic;\n }\n+\n+/* Style definition generated by highlight 2.4.5, http://www.andre-simon.de/ */\n+\n+/* Highlighting theme definition: */\n+\n+.num    { color:#2928ff; }\n+.esc    { color:#ff00ff; }\n+.str    { color:#ff0000; }\n+.dstr   { color:#818100; }\n+.slc    { color:#838183; font-style:italic; }\n+.com    { color:#838183; font-style:italic; }\n+.dir    { color:#008200; }\n+.sym    { color:#000000; }\n+.line   { color:#555555; }\n+.kwa    { color:#000000; font-weight:bold; }\n+.kwb    { color:#830000; }\n+.kwc    { color:#000000; font-weight:bold; }\n+.kwd    { color:#010181; }\ndiff --git a/gitweb/gitweb.perl b/gitweb/gitweb.perl\nindex 32b04a4..5b85581 100755\n--- a/gitweb/gitweb.perl\n+++ b/gitweb/gitweb.perl\n@@ -227,6 +227,36 @@ our %avatar_size = (\n # Leave it undefined (or set to 'undef') to turn off load checking.\n our $maxload = 300;\n \n+# syntax highlighting\n+our %highlight_type = (\n+\t# match by basename\n+\t'SConstruct' => 'py',\n+\t'Program' => 'py',\n+\t'Library' => 'py',\n+\t'Makefile' => 'make',\n+\t# match by extension\n+\t'\\.py$' => 'py', # Python\n+\t'\\.c$' => 'c',\n+\t'\\.h$' => 'c',\n+\t'\\.cpp$' => 'cpp',\n+\t'\\.cxx$' => 'cpp',\n+\t'\\.rb$' => 'ruby',\n+\t'\\.java$' => 'java',\n+\t'\\.css$' => 'css',\n+\t'\\.php3?$' => 'php',\n+\t'\\.sh$' => 'sh', # Bash / shell script\n+\t'\\.pl$' => 'pl', # Perl\n+\t'\\.js$' => 'js', # JavaScript\n+\t'\\.tex$' => 'tex', # TeX and LaTeX\n+\t'\\.bib$' => 'bib', # BibTeX\n+\t'\\.x?html$' => 'xml',\n+\t'\\.xml$' => 'xml',\n+\t'\\.awk$' => 'awk',\n+\t'\\.bat$' => 'bat', # DOS Batch script\n+\t'\\.ini$' => 'ini',\n+\t'\\.spec$' => 'spec', # RPM Spec\n+);\n+\n # You define site-wide feature defaults here; override them with\n # $GITWEB_CONFIG as necessary.\n our %feature = (\n@@ -445,6 +475,19 @@ our %feature = (\n \t'javascript-actions' => {\n \t\t'override' => 0,\n \t\t'default' => [0]},\n+\n+\t# Syntax highlighting support. This is based on Daniel Svensson's\n+\t# and Sham Chukoury's work in gitweb-xmms2.git.\n+\t# It requires the 'highlight' program, and therefore is disabled\n+\t# by default.\n+\n+\t# To enable system wide have in $GITWEB_CONFIG\n+\t# $feature{'highlight'}{'default'} = [1];\n+\n+\t'highlight' => {\n+\t\t'sub' => sub { feature_bool('highlight', @_) },\n+\t\t'override' => 0,\n+\t\t'default' => [0]},\n );\n \n sub gitweb_get_feature {\n@@ -5340,6 +5383,7 @@ sub git_blob {\n \topen my $fd, \"-|\", git_cmd(), \"cat-file\", \"blob\", $hash\n \t\tor die_error(500, \"Couldn't cat $file_name, $hash\");\n \tmy $mimetype = blob_mimetype($fd, $file_name);\n+\t# use 'blob_plain' (aka 'raw') view for files that cannot be displayed\n \tif ($mimetype !~ m!^(?:text/|image/(?:gif|png|jpeg)$)! && -B $fd) {\n \t\tclose $fd;\n \t\treturn git_blob_plain($mimetype);\n@@ -5347,6 +5391,25 @@ sub git_blob {\n \t# we can have blame only for text/* mimetype\n \t$have_blame &&= ($mimetype =~ m!^text/!);\n \n+\tmy $have_highlight = gitweb_check_feature('highlight');\n+\tmy $syntax;\n+\tif ($have_highlight && defined($file_name)) {\n+\t\tmy $basename = basename($file_name, '.in');\n+\t\tforeach my $regexp (keys %highlight_type) {\n+\t\t\tif ($basename =~ /$regexp/) {\n+\t\t\t\t$syntax = $highlight_type{$regexp};\n+\t\t\t\tlast;\n+\t\t\t}\n+\t\t}\n+\n+\t\tif ($syntax) {\n+\t\t\tclose $fd;\n+\t\t\topen $fd, quote_command(git_cmd(), \"cat-file\", \"blob\", $hash).\" | \".\n+\t\t\t          \"highlight --xhtml --fragment -t 8 --syntax $syntax |\"\n+\t\t\t\tor die_error(500, \"Couldn't open file or run syntax highlighter\");\n+\t\t}\n+\t}\n+\n \tgit_header_html(undef, $expires);\n \tmy $formats_nav = '';\n \tif (defined $hash_base && (my %co = parse_commit($hash_base))) {\n@@ -5398,7 +5461,7 @@ sub git_blob {\n \t\t\t$line = untabify($line);\n \t\t\tprintf \"<div class=\\\"pre\\\"><a id=\\\"l%i\\\" href=\\\"\" . href(-replay => 1)\n \t\t\t\t. \"#l%i\\\" class=\\\"linenr\\\">%4i</a> %s</div>\\n\",\n-\t\t\t       $nr, $nr, $nr, esc_html($line, -nbsp=>1);\n+\t\t\t       $nr, $nr, $nr, $syntax ? $line : esc_html($line, -nbsp=>1);\n \t\t}\n \t}\n \tclose $fd\n-- \n1.6.6.1\n"},{"id":"135895","messageId":"1267388890-30305-2-git-send-email-jnareb@gmail.com","threadId":"22857","inReplyTo":"1267388890-30305-1-git-send-email-jnareb@gmail.com","subject":"[PATCH/RFC 2/2] gitweb: Refactor syntax highlighting support","fromName":"Jakub Narebski","fromEmail":"jnareb@gmail.com","sentAt":"2010-02-28T20:28:10Z","receivedAt":"2010-02-28T20:28:10Z","isPatch":true,"sender":{"key":"jnareb@gmail.com","avatar":"https://avatars.githubusercontent.com/u/2706?v=4"},"body":"This refactoring (adding blob_syntax_highlight and blob_highlight\nsubroutines) is meant to make it easier to add support for other syntax\nhighlighing solutions, or make it smarter by not re-running `git\ncat-file` second time.\n\nInstead of looping over list of regexps (keys of %highlight_type hash),\nmake use of the fact that choosing syntax is based either on full\nbasename, or on file extension.\n\nAdd some basic test of syntax highlighting (with 'highlight' as\nprerequisite) to t/t9500-gitweb-standalone-no-errors.sh test.\n\nWhile at it make git_blob Perl style prettier.\n\nSigned-off-by: Jakub Narebski <jnareb@gmail.com>\n---\nThis probably could be squashed together with previous patch.\n\nIt is an RFC, because I am not sure about names blob_syntax_highlight\nand blob_highlight for subroutines.\n\nNote: if 'SConstruct', 'Program' and 'Library' must match path, and\nnot only basename to identify Python (py) program, then current\nsolution of using hash-table instead of loop over regexp to match\nwould be not possible.\n\n[I probably should update comment and add some information to\ngitweb/README and/or gitweb/INSTALL about syntax hightlight support.]\n\n gitweb/gitweb.perl                     |  106 ++++++++++++++++---------------\n t/t9500-gitweb-standalone-no-errors.sh |   29 +++++++++\n 2 files changed, 84 insertions(+), 51 deletions(-)\n\ndiff --git a/gitweb/gitweb.perl b/gitweb/gitweb.perl\nindex 5b85581..528ffe1 100755\n--- a/gitweb/gitweb.perl\n+++ b/gitweb/gitweb.perl\n@@ -227,36 +227,6 @@ our %avatar_size = (\n # Leave it undefined (or set to 'undef') to turn off load checking.\n our $maxload = 300;\n \n-# syntax highlighting\n-our %highlight_type = (\n-\t# match by basename\n-\t'SConstruct' => 'py',\n-\t'Program' => 'py',\n-\t'Library' => 'py',\n-\t'Makefile' => 'make',\n-\t# match by extension\n-\t'\\.py$' => 'py', # Python\n-\t'\\.c$' => 'c',\n-\t'\\.h$' => 'c',\n-\t'\\.cpp$' => 'cpp',\n-\t'\\.cxx$' => 'cpp',\n-\t'\\.rb$' => 'ruby',\n-\t'\\.java$' => 'java',\n-\t'\\.css$' => 'css',\n-\t'\\.php3?$' => 'php',\n-\t'\\.sh$' => 'sh', # Bash / shell script\n-\t'\\.pl$' => 'pl', # Perl\n-\t'\\.js$' => 'js', # JavaScript\n-\t'\\.tex$' => 'tex', # TeX and LaTeX\n-\t'\\.bib$' => 'bib', # BibTeX\n-\t'\\.x?html$' => 'xml',\n-\t'\\.xml$' => 'xml',\n-\t'\\.awk$' => 'awk',\n-\t'\\.bat$' => 'bat', # DOS Batch script\n-\t'\\.ini$' => 'ini',\n-\t'\\.spec$' => 'spec', # RPM Spec\n-);\n-\n # You define site-wide feature defaults here; override them with\n # $GITWEB_CONFIG as necessary.\n our %feature = (\n@@ -3192,6 +3162,55 @@ sub blob_contenttype {\n \treturn $type;\n }\n \n+sub blob_syntax_highlight {\n+\tmy ($highlight, $mimetype, $file_name) = @_;\n+\treturn undef unless ($highlight && defined $file_name);\n+\n+\t# match by basename\n+\tmy %highlight_basename = (\n+\t\t#'SConstruct' => 'py',\n+\t\t#'Program' => 'py',\n+\t\t#'Library' => 'py',\n+\t\t'Makefile' => 'make',\n+\t);\n+\t# match by extension\n+\tmy %highlight_ext = (\n+\t\t# see files in /usr/share/highlight/langDefs/ directory\n+\t\tmap { $_ => $_ }\n+\t\tqw(py c cpp rb java css php sh pl js tex bib xml awk bat ini spec tcl),\n+\t\t# alternate extensions, see /etc/highlight/filetypes.conf\n+\t\t'h' => 'c',\n+\t\t'cxx' => 'cpp', 'c++' => 'cpp',\n+\t\t'php3' => 'php', 'php4' => 'php',\n+\t\t'perl' => 'pl', 'pm' => 'pl',\n+\t\t'mak' => 'make',\n+\t\t'xhtml' => 'xml', 'html' => 'xml', 'htm' => 'xml',\n+\t);\n+\n+\tmy $basename = basename($file_name, '.in');\n+\treturn $highlight_basename{$basename}\n+\t\tif exists $highlight_basename{$basename};\n+\n+\t$basename =~ /\\.([^.]*)$/;\n+\tmy $ext = $1 or return undef;\n+\treturn $highlight_ext{$ext}\n+\t\tif exists $highlight_ext{$ext};\n+\n+\treturn undef;\n+}\n+\n+sub blob_highlight {\n+\tmy ($fd, $highlight, $syntax) = @_;\n+\treturn $fd unless ($highlight && defined $syntax);\n+\n+\tclose $fd\n+\t\tor die_error(404, \"Reading blob failed\");\n+\topen $fd, quote_command(git_cmd(), \"cat-file\", \"blob\", $hash).\" | \".\n+\t          \"highlight --xhtml --fragment --syntax $syntax |\"\n+\t\tor die_error(500, \"Couldn't open file or run syntax highlighter\");\n+\treturn $fd;\n+}\n+\n ## ======================================================================\n ## functions printing HTML: header, footer, error page\n \n@@ -5391,24 +5410,10 @@ sub git_blob {\n \t# we can have blame only for text/* mimetype\n \t$have_blame &&= ($mimetype =~ m!^text/!);\n \n-\tmy $have_highlight = gitweb_check_feature('highlight');\n-\tmy $syntax;\n-\tif ($have_highlight && defined($file_name)) {\n-\t\tmy $basename = basename($file_name, '.in');\n-\t\tforeach my $regexp (keys %highlight_type) {\n-\t\t\tif ($basename =~ /$regexp/) {\n-\t\t\t\t$syntax = $highlight_type{$regexp};\n-\t\t\t\tlast;\n-\t\t\t}\n-\t\t}\n-\n-\t\tif ($syntax) {\n-\t\t\tclose $fd;\n-\t\t\topen $fd, quote_command(git_cmd(), \"cat-file\", \"blob\", $hash).\" | \".\n-\t\t\t          \"highlight --xhtml --fragment -t 8 --syntax $syntax |\"\n-\t\t\t\tor die_error(500, \"Couldn't open file or run syntax highlighter\");\n-\t\t}\n-\t}\n+\tmy $highlight = gitweb_check_feature('highlight');\n+\tmy $syntax = blob_syntax_highlight($highlight, $mimetype, $file_name);\n+\t$fd = blob_highlight($fd, $highlight, $syntax)\n+\t\tif $syntax;\n \n \tgit_header_html(undef, $expires);\n \tmy $formats_nav = '';\n@@ -5459,9 +5464,8 @@ sub git_blob {\n \t\t\tchomp $line;\n \t\t\t$nr++;\n \t\t\t$line = untabify($line);\n-\t\t\tprintf \"<div class=\\\"pre\\\"><a id=\\\"l%i\\\" href=\\\"\" . href(-replay => 1)\n-\t\t\t\t. \"#l%i\\\" class=\\\"linenr\\\">%4i</a> %s</div>\\n\",\n-\t\t\t       $nr, $nr, $nr, $syntax ? $line : esc_html($line, -nbsp=>1);\n+\t\t\tprintf qq!<div class=\"pre\"><a id=\"l%i\" href=\"%s#l%i\" class=\"linenr\">%4i</a> %s</div>\\n!,\n+\t\t\t       $nr, href(-replay => 1), $nr, $nr, $syntax ? $line : esc_html($line, -nbsp=>1);\n \t\t}\n \t}\n \tclose $fd\ndiff --git a/t/t9500-gitweb-standalone-no-errors.sh b/t/t9500-gitweb-standalone-no-errors.sh\nindex 2fc7fdb..0fe9beb 100755\n--- a/t/t9500-gitweb-standalone-no-errors.sh\n+++ b/t/t9500-gitweb-standalone-no-errors.sh\n@@ -639,4 +639,33 @@ test_expect_success \\\n \t gitweb_run \"p=.git;a=summary\"'\n test_debug 'cat gitweb.log'\n \n+# ----------------------------------------------------------------------\n+# syntax highlighting\n+\n+cat >>gitweb_config.perl <<\\EOF\n+$feature{'highlight'}{'override'} = 1;\n+EOF\n+\n+highlight --version >/dev/null 2>&1\n+if [ $? -eq 127 ]; then\n+\tsay \"Skipping syntax highlighting test, because 'highlight' was not found\"\n+else\n+\ttest_set_prereq HIGHLIGHT\n+fi\n+\n+test_expect_success HIGHLIGHT \\\n+\t'syntax highlighting (no highlight)' \\\n+\t'git config gitweb.highlight yes &&\n+\t gitweb_run \"p=.git;a=blob;f=file\"'\n+test_debug 'cat gitweb.log'\n+\n+test_expect_success HIGHLIGHT \\\n+\t'syntax highlighting (highlighted)' \\\n+\t'git config gitweb.highlight yes &&\n+\t echo \"#!/usr/bin/sh\" > test.sh &&\n+\t git add test.sh &&\n+\t git commit -m \"Add test.sh\" &&\n+\t gitweb_run \"p=.git;a=blob;f=test.sh\"'\n+test_debug 'cat gitweb.log'\n+\n test_done\n-- \n1.6.6.1\n"}]}