git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v4 7/7] color-words: make regex configurable via attributes

From
Thomas Rast <trast@student.ethz.ch>
Date
Jan 17, 2009, 16:29 UTC
Message-ID
<1232209788-10408-8-git-send-email-trast@student.ethz.ch>
In-Reply-To
<1232209788-10408-7-git-send-email-trast@student.ethz.ch>

Make the --color-words splitting regular expression configurable via the diff driver's 'wordregex' attribute. The user can then set the driver on a file in .gitattributes. If a regex is given on the command line, it overrides the driver's setting.

We also provide built-in regexes for the languages that already had funcname patterns, and add an appropriate diff driver entry for C/++. (The patterns are designed to run UTF-8 sequences into a single chunk to make sure they remain readable.)

Signed-off-by: Thomas Rast <trast@student.ethz.ch>
---
 Documentation/diff-options.txt  |    4 ++
 Documentation/gitattributes.txt |   21 ++++++++++
 diff.c                          |   10 +++++
 t/t4034-diff-words.sh           |   36 ++++++++++++++++++
 userdiff.c                      |   78 +++++++++++++++++++++++++++++++-------
 userdiff.h                      |    1 +
 6 files changed, 135 insertions(+), 15 deletions(-)
diff --git a/Documentation/diff-options.txt b/Documentation/diff-options.txt
index 8689a92..1edb82e 100644
--- a/Documentation/diff-options.txt
+++ b/Documentation/diff-options.txt
@@ -102,6 +102,10 @@ differences.  You may want to append `|[^[:space:]]` to your regular
 expression to make sure that it matches all non-whitespace characters.
 A match that contains a newline is silently truncated(!) at the
 newline.
++
+The regex can also be set via a diff driver, see
+linkgit:gitattributes[1]; giving it explicitly overrides any diff
+driver setting.
 
 --no-renames::
 	Turn off rename detection, even when the configuration
diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt
index 8af22ec..ba3ba12 100644
--- a/Documentation/gitattributes.txt
+++ b/Documentation/gitattributes.txt
@@ -317,6 +317,8 @@ patterns are available:
 
 - `bibtex` suitable for files with BibTeX coded references.
 
+- `cpp` suitable for source code in the C and C++ languages.
+
 - `html` suitable for HTML/XHTML documents.
 
 - `java` suitable for source code in the Java language.
@@ -334,6 +336,25 @@ patterns are available:
 - `tex` suitable for source code for LaTeX documents.
 
 
+Customizing word diff
+^^^^^^^^^^^^^^^^^^^^^
+
+You can customize the rules that `git diff --color-words` uses to
+split words in a line, by specifying an appropriate regular expression
+in the "diff.*.wordregex" configuration variable.  For example, in TeX
+a backslash followed by a sequence of letters forms a command, but
+several such commands can be run together without intervening
+whitespace.  To separate them, use a regular expression such as
+
+------------------------
+[diff "tex"]
+	wordregex = "\\\\[a-zA-Z]+|[{}]|\\\\.|[^\\{}[:space:]]+"
+------------------------
+
+A built-in pattern is provided for all languages listed in the
+previous section.
+
+
 Performing text diffs of binary files
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/diff.c b/diff.c
index 00c661f..9fcde96 100644
--- a/diff.c
+++ b/diff.c
@@ -1380,6 +1380,12 @@ int diff_filespec_is_binary(struct diff_filespec *one)
 	return one->driver->funcname.pattern ? &one->driver->funcname : NULL;
 }
 
+static const char *userdiff_word_regex(struct diff_filespec *one)
+{
+	diff_filespec_load_driver(one);
+	return one->driver->word_regex;
+}
+
 void diff_set_mnemonic_prefix(struct diff_options *options, const char *a, const char *b)
 {
 	if (!options->a_prefix)
@@ -1540,6 +1546,10 @@ static void builtin_diff(const char *name_a,
 			ecbdata.diff_words =
 				xcalloc(1, sizeof(struct diff_words_data));
 			ecbdata.diff_words->file = o->file;
+			if (!o->word_regex)
+				o->word_regex = userdiff_word_regex(one);
+			if (!o->word_regex)
+				o->word_regex = userdiff_word_regex(two);
 			if (o->word_regex) {
 				ecbdata.diff_words->word_regex = (regex_t *)
 					xmalloc(sizeof(regex_t));
diff --git a/t/t4034-diff-words.sh b/t/t4034-diff-words.sh
index 4873486..744221b 100755
--- a/t/t4034-diff-words.sh
+++ b/t/t4034-diff-words.sh
@@ -84,6 +84,41 @@ test_expect_success 'word diff with a regular expression' '
 
 '
 
+test_expect_success 'set a diff driver' '
+	git config diff.testdriver.wordregex "[^[:space:]]" &&
+	cat <<EOF > .gitattributes
+pre diff=testdriver
+post diff=testdriver
+EOF
+'
+
+test_expect_success 'option overrides default' '
+
+	word_diff --color-words="[a-z]+"
+
+'
+
+cat > expect <<\EOF
+<WHITE>diff --git a/pre b/post<RESET>
+<WHITE>index 330b04f..5ed8eff 100644<RESET>
+<WHITE>--- a/pre<RESET>
+<WHITE>+++ b/post<RESET>
+<BROWN>@@ -1,3 +1,7 @@<RESET>
+h(4)<GREEN>,hh[44]<RESET>
+<RESET>
+a = b + c<RESET>
+
+<GREEN>aa = a<RESET>
+
+<GREEN>aeff = aeff * ( aaa )<RESET>
+EOF
+
+test_expect_success 'use default supplied by driver' '
+
+	word_diff --color-words
+
+'
+
 echo 'aaa (aaa)' > pre
 echo 'aaa (aaa) aaa' > post
 
@@ -100,6 +135,7 @@ test_expect_success 'test parsing words for newline' '
 
 	word_diff --color-words="a+"
 
+
 '
 
 echo '(:' > pre
diff --git a/userdiff.c b/userdiff.c
index 3681062..2b55509 100644
--- a/userdiff.c
+++ b/userdiff.c
@@ -6,14 +6,20 @@
 static int ndrivers;
 static int drivers_alloc;
 
-#define FUNCNAME(name, pattern) \
-	{ name, NULL, -1, { pattern, REG_EXTENDED } }
+#define PATTERNS(name, pattern, wordregex)			\
+	{ name, NULL, -1, { pattern, REG_EXTENDED }, wordregex }
 static struct userdiff_driver builtin_drivers[] = {
-FUNCNAME("html", "^[ \t]*(<[Hh][1-6][ \t].*>.*)$"),
-FUNCNAME("java",
+PATTERNS("html", "^[ \t]*(<[Hh][1-6][ \t].*>.*)$",
+	 "[^<>= \t]+|[^[:space:]]|[\x80-\xff]+"),
+PATTERNS("java",
 	 "!^[ \t]*(catch|do|for|if|instanceof|new|return|switch|throw|while)\n"
-	 "^[ \t]*(([ \t]*[A-Za-z_][A-Za-z_0-9]*){2,}[ \t]*\\([^;]*)$"),
-FUNCNAME("objc",
+	 "^[ \t]*(([ \t]*[A-Za-z_][A-Za-z_0-9]*){2,}[ \t]*\\([^;]*)$",
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]="
+	 "|--|\\+\\+|<<=?|>>>?=?|&&|\\|\\|"
+	 "|[^[:space:]]|[\x80-\xff]+"),
+PATTERNS("objc",
 	 /* Negate C statements that can look like functions */
 	 "!^[ \t]*(do|for|if|else|return|switch|while)\n"
 	 /* Objective-C methods */
@@ -21,20 +27,60 @@
 	 /* C functions */
 	 "^[ \t]*(([ \t]*[A-Za-z_][A-Za-z_0-9]*){2,}[ \t]*\\([^;]*)$\n"
 	 /* Objective-C class/protocol definitions */
-	 "^(@(implementation|interface|protocol)[ \t].*)$"),
-FUNCNAME("pascal",
+	 "^(@(implementation|interface|protocol)[ \t].*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"
+	 "|[^[:space:]]|[\x80-\xff]+"),
+PATTERNS("pascal",
 	 "^((procedure|function|constructor|destructor|interface|"
 		"implementation|initialization|finalization)[ \t]*.*)$"
 	 "\n"
-	 "^(.*=[ \t]*(class|record).*)$"),
-FUNCNAME("php", "^[\t ]*((function|class).*)"),
-FUNCNAME("python", "^[ \t]*((class|def)[ \t].*)$"),
-FUNCNAME("ruby", "^[ \t]*((class|module|def)[ \t].*)$"),
-FUNCNAME("bibtex", "(@[a-zA-Z]{1,}[ \t]*\\{{0,1}[ \t]*[^ \t\"@',\\#}{~%]*).*$"),
-FUNCNAME("tex", "^(\\\\((sub)*section|chapter|part)\\*{0,1}\\{.*)$"),
+	 "^(.*=[ \t]*(class|record).*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
+	 "|<>|<=|>=|:=|\\.\\."
+	 "|[^[:space:]]|[\x80-\xff]+"),
+PATTERNS("php", "^[\t ]*((function|class).*)",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
+	 "|[-+*/<>%&^|=!.]=|--|\\+\\+|<<=?|>>=?|===|&&|\\|\\||::|->"
+	 "|[^[:space:]]|[\x80-\xff]+"),
+PATTERNS("python", "^[ \t]*((class|def)[ \t].*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[jJlL]?|0[xX]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|//=?|<<=?|>>=?|\\*\\*=?"
+	 "|[^[:space:]|[\x80-\xff]+"),
+	 /* -- */
+PATTERNS("ruby", "^[ \t]*((class|module|def)[ \t].*)$",
+	 /* -- */
+	 "(@|@@|\\$)?[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+|\\?(\\\\C-)?(\\\\M-)?."
+	 "|//=?|[-+*/<>%&^|=!]=|<<=?|>>=?|===|\\.{1,3}|::|[!=]~"
+	 "|[^[:space:]|[\x80-\xff]+"),
+PATTERNS("bibtex", "(@[a-zA-Z]{1,}[ \t]*\\{{0,1}[ \t]*[^ \t\"@',\\#}{~%]*).*$",
+	 "[={}\"]|[^={}\" \t]+"),
+PATTERNS("tex", "^(\\\\((sub)*section|chapter|part)\\*{0,1}\\{.*)$",
+	 "\\\\[a-zA-Z@]+|\\\\.|[a-zA-Z0-9\x80-\xff]+|[^[:space:]]"),
+PATTERNS("cpp",
+	 /* Jump targets or access declarations */
+	 "!^[ \t]*[A-Za-z_][A-Za-z_0-9]*:.*$\n"
+	 /* C/++ functions/methods at top level */
+	 "^([A-Za-z_][A-Za-z_0-9]*([ \t]+[A-Za-z_][A-Za-z_0-9]*([ \t]*::[ \t]*[^[:space:]]+)?){1,}[ \t]*\\([^;]*)$\n"
+	 /* compound type at top level */
+	 "^((struct|class|enum)[^;]*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"
+	 "|[^[:space:]]|[\x80-\xff]+"),
 { "default", NULL, -1, { NULL, 0 } },
 };
-#undef FUNCNAME
+#undef PATTERNS
 
 static struct userdiff_driver driver_true = {
 	"diff=true",
@@ -134,6 +180,8 @@ int userdiff_config(const char *k, const char *v)
 		return parse_string(&drv->external, k, v);
 	if ((drv = parse_driver(k, v, "textconv")))
 		return parse_string(&drv->textconv, k, v);
+	if ((drv = parse_driver(k, v, "wordregex")))
+		return parse_string(&drv->word_regex, k, v);
 
 	return 0;
 }
diff --git a/userdiff.h b/userdiff.h
index ba29457..c315159 100644
--- a/userdiff.h
+++ b/userdiff.h
@@ -11,6 +11,7 @@ struct userdiff_driver {
 	const char *external;
 	int binary;
 	struct userdiff_funcname funcname;
+	const char *word_regex;
 	const char *textconv;
 };
 

-- 
1.6.1.315.g92577
Previous: Thomas RastNext: Santi Béjar
Message 51 of 109 in “refactor the --color-words to make it more hackable”
  1. 0/4 refactor the --color-words to make it more hackableJohannes Schindelin, Jan 11, 2009
  2. 1/4 Add color_fwrite(), a function coloring each line individuallyJohannes Schindelin, Jan 11, 2009
  3. Junio C HamanoJan 11, 2009
  4. Johannes SchindelinJan 11, 2009
  5. 1/4 Add color_fwrite(), a function coloring each line individuallyJohannes Schindelin, Jan 11, 2009
  6. Jakub NarebskiJan 12, 2009
  7. 2/4 color-words: refactor word splitting and use ALLOC_GROW()Johannes Schindelin, Jan 11, 2009
  8. 3/4 color-words: refactor to allow for 0-character word boundariesJohannes Schindelin, Jan 11, 2009
  9. Junio C HamanoJan 11, 2009
  10. Johannes SchindelinJan 11, 2009
  11. Thomas RastJan 12, 2009
  12. Junio C HamanoJan 12, 2009
  13. 0/4 color-words improvementsJohannes Schindelin, Jan 14, 2009
  14. 1/4 Add color_fwrite_lines(), a function coloring each line individuallyJohannes Schindelin, Jan 14, 2009
  15. 2/4 color-words: refactor word splitting and use ALLOC_GROW()Johannes Schindelin, Jan 14, 2009
  16. 3/4 color-words: change algorithm to allow for 0-character word boundariesJohannes Schindelin, Jan 14, 2009
  17. Johannes SchindelinJan 14, 2009
  18. 4/4 color-words: take an optional regular expression describing wordsJohannes Schindelin, Jan 14, 2009
  19. Thomas RastJan 14, 2009
  20. Teemu LikonenJan 14, 2009
  21. Teemu LikonenJan 14, 2009
  22. Johannes SchindelinJan 14, 2009
  23. Johannes SchindelinJan 14, 2009
  24. Thomas RastJan 14, 2009
  25. Johannes SchindelinJan 14, 2009
  26. Thomas RastJan 14, 2009
  27. Boyd Stephen Smith Jr.Jan 14, 2009
  28. Teemu LikonenJan 15, 2009
  29. Johannes SchindelinJan 15, 2009
  30. Teemu LikonenJan 15, 2009
  31. Thomas RastJan 15, 2009
  32. Junio C HamanoJan 15, 2009
  33. Johannes SchindelinJan 15, 2009
  34. Santi BéjarJan 16, 2009
  35. Junio C HamanoJan 16, 2009
  36. Boyd Stephen Smith Jr.Jan 16, 2009
  37. Johannes SchindelinJan 16, 2009
  38. Santi BéjarJan 16, 2009
  39. Johannes SchindelinJan 16, 2009
  40. Santi BéjarJan 16, 2009
  41. Johannes SchindelinJan 16, 2009
  42. Thomas RastJan 16, 2009
  43. Johannes SchindelinJan 16, 2009
  44. 0/7 customizable --color-wordsThomas Rast, Jan 17, 2009
  45. 1/7 Add color_fwrite_lines(), a function coloring each line individuallyThomas Rast, Jan 17, 2009
  46. 2/7 color-words: refactor word splitting and use ALLOC_GROW()Thomas Rast, Jan 17, 2009
  47. 3/7 color-words: change algorithm to allow for 0-character word boundariesThomas Rast, Jan 17, 2009
  48. 4/7 color-words: take an optional regular expression describing wordsThomas Rast, Jan 17, 2009
  49. 5/7 color-words: enable REG_NEWLINE to help userThomas Rast, Jan 17, 2009
  50. 6/7 color-words: expand docs with precise semanticsThomas Rast, Jan 17, 2009
  51. 7/7 color-words: make regex configurable via attributesThomas Rast, Jan 17, 2009
  52. Santi BéjarJan 18, 2009
  53. Santi BéjarJan 18, 2009
  54. Santi BéjarJan 19, 2009
  55. Johannes SchindelinJan 19, 2009
  56. Add tests for diff.color-words configuration option.Boyd Stephen Smith Jr., Jan 20, 2009
  57. diff: Support diff.color-words config optionBoyd Stephen Smith Jr., Jan 20, 2009
  58. Junio C HamanoJan 20, 2009
  59. Markus HeidelbergJan 20, 2009
  60. Boyd Stephen Smith Jr.Jan 20, 2009
  61. Johannes SchindelinJan 20, 2009
  62. Junio C HamanoJan 21, 2009
  63. Markus HeidelbergJan 21, 2009
  64. Johannes SchindelinJan 20, 2009
  65. Boyd Stephen Smith Jr.Jan 20, 2009
  66. Johannes SchindelinJan 20, 2009
  67. Junio C HamanoJan 20, 2009
  68. Johannes SchindelinJan 20, 2009
  69. Junio C HamanoJan 20, 2009
  70. Johannes SchindelinJan 20, 2009
  71. color-words: Support diff.color-words config optionBoyd Stephen Smith Jr., Jan 21, 2009
  72. Change the spelling of "wordregex".Boyd Stephen Smith Jr., Jan 21, 2009
  73. Johannes SchindelinJan 21, 2009
  74. Thomas RastJan 21, 2009
  75. Boyd Stephen Smith Jr.Jan 21, 2009
  76. Johannes SchindelinJan 21, 2009
  77. Boyd Stephen Smith Jr.Jan 21, 2009
  78. Junio C HamanoJan 21, 2009
  79. Jakub NarebskiJan 20, 2009
  80. Johannes SchindelinJan 20, 2009
  81. Boyd Stephen Smith Jr.Jan 20, 2009
  82. Johannes SchindelinJan 20, 2009
  83. Boyd Stephen Smith Jr.Jan 16, 2009
  84. color-words: make regex configurable via attributesThomas Rast, Jan 14, 2009
  85. Johannes SchindelinJan 14, 2009
  86. Thomas RastJan 14, 2009
  87. 3/4 color-words: change algorithm to allow for 0-character word boundariesJohannes Schindelin, Jan 14, 2009
  88. 4/4 color-words: take an optional regular expression describing wordsJohannes Schindelin, Jan 14, 2009
  89. Thomas RastJan 15, 2009
  90. Johannes SchindelinJan 15, 2009
  91. Johannes SchindelinJan 15, 2009
  92. Thomas RastJan 15, 2009
  93. Thomas RastJan 15, 2009
  94. Johannes SchindelinJan 15, 2009
  95. 1/4 color-words: fix quoting in t4034Thomas Rast, Jan 14, 2009
  96. 2/4 color-words: enable REG_NEWLINE to help userThomas Rast, Jan 14, 2009
  97. 3/4 color-words: expand docs with precise semanticsThomas Rast, Jan 14, 2009
  98. 4/4 color-words: make regex configurable via attributesThomas Rast, Jan 14, 2009
  99. Johannes SchindelinJan 15, 2009
  100. Johannes SchindelinJan 15, 2009
  101. Johannes SchindelinJan 14, 2009
  102. Thomas RastJan 14, 2009
  103. Johannes SchindelinJan 14, 2009
  104. Thomas RastJan 14, 2009
  105. 4/4 color-words: take an optional regular expression describing wordsJohannes Schindelin, Jan 11, 2009
  106. Thomas RastJan 11, 2009
  107. Johannes SchindelinJan 11, 2009
  108. Thomas RastJan 12, 2009
  109. Santi BéjarJan 14, 2009

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.