git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH] color-words: make regex configurable via attributes

From
Thomas Rast <trast@student.ethz.ch>
Date
Jan 14, 2009, 19:46 UTC
Message-ID
<1231962401-26974-1-git-send-email-trast@student.ethz.ch>
In-Reply-To
<alpine.DEB.1.00.0901141840100.3586@pacific.mpi-cbg.de>

Make the --color-words splitting regular expression configurable via the diff driver's 'wordregex' attribute. The user can then set the driver on a file in .gitattributes. If a regex is given on the command line, it overrides the driver's setting.

We also provide built-in regexes for the languages that already had funcname patterns, and add an appropriate diff driver entry for C/++. (The patterns are designed to run UTF-8 sequences into a single chunk to make sure they remain readable.)

Signed-off-by: Thomas Rast <trast@student.ethz.ch>
---

This is the old 3/4 combined with a test similar to the one it had in the old 4/4, built on top of Dscho's take 3. I researched the operators for each language, but the identifier and number formats may be off in some cases.

 Documentation/diff-options.txt  |    3 +
 Documentation/gitattributes.txt |   21 ++++++++++
 diff.c                          |   10 +++++
 t/t4034-diff-words.sh           |   40 ++++++++++++++++++++
 userdiff.c                      |   78 +++++++++++++++++++++++++++++++-------
 userdiff.h                      |    1 +
 6 files changed, 138 insertions(+), 15 deletions(-)
diff --git a/Documentation/diff-options.txt b/Documentation/diff-options.txt
index 2c1fa4b..ef0e2f5 100644
--- a/Documentation/diff-options.txt
+++ b/Documentation/diff-options.txt
@@ -97,6 +97,9 @@ endif::git-format-patch[]
 Optionally, you can pass a regular expression that tells Git what the
 words are that you are looking for; The default is to interpret any
 stretch of non-whitespace as a word.
+The regex can also be set via a diff driver, see
+linkgit:gitattributes[1]; giving it explicitly overrides any diff
+driver setting.
 
 --no-renames::
 	Turn off rename detection, even when the configuration
diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt
index 8af22ec..17707ba 100644
--- a/Documentation/gitattributes.txt
+++ b/Documentation/gitattributes.txt
@@ -317,6 +317,8 @@ patterns are available:
 
 - `bibtex` suitable for files with BibTeX coded references.
 
+- `cpp` suitable for source code in the C and C++ languages.
+
 - `html` suitable for HTML/XHTML documents.
 
 - `java` suitable for source code in the Java language.
@@ -334,6 +336,25 @@ patterns are available:
 - `tex` suitable for source code for LaTeX documents.
 
 
+Customizing word diff
+^^^^^^^^^^^^^^^^^^^^^
+
+You can customize the rules that `git diff --color-words` uses to
+split words in a line, by specifying an appropriate regular expression
+in the "diff.*.wordregex" configuration variable.  For example, in TeX
+a backslash followed by a sequence of letters forms a command, but
+several such commands can be run together without intervening
+whitespace.  To separate them, use a regular expression such as
+
+------------------------
+[diff "tex"]
+	wordregex = "\\\\[a-zA-Z]+|[{}]|\\\\.|[^\\{}[:space:]]+"
+------------------------
+
+A built-in pattern is provided for all languages listed in the last
+section.
+
+
 Performing text diffs of binary files
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/diff.c b/diff.c
index eb67431..08bdc86 100644
--- a/diff.c
+++ b/diff.c
@@ -1372,6 +1372,12 @@ int diff_filespec_is_binary(struct diff_filespec *one)
 	return one->driver->funcname.pattern ? &one->driver->funcname : NULL;
 }
 
+static const char *userdiff_word_regex(struct diff_filespec *one)
+{
+	diff_filespec_load_driver(one);
+	return one->driver->word_regex;
+}
+
 void diff_set_mnemonic_prefix(struct diff_options *options, const char *a, const char *b)
 {
 	if (!options->a_prefix)
@@ -1532,6 +1538,10 @@ static void builtin_diff(const char *name_a,
 			ecbdata.diff_words =
 				xcalloc(1, sizeof(struct diff_words_data));
 			ecbdata.diff_words->file = o->file;
+			if (!o->word_regex)
+				o->word_regex = userdiff_word_regex(one);
+			if (!o->word_regex)
+				o->word_regex = userdiff_word_regex(two);
 			if (o->word_regex) {
 				ecbdata.diff_words->word_regex = (regex_t *)
 					xmalloc(sizeof(regex_t));
diff --git a/t/t4034-diff-words.sh b/t/t4034-diff-words.sh
index 0ed7e53..d6731d1 100755
--- a/t/t4034-diff-words.sh
+++ b/t/t4034-diff-words.sh
@@ -83,4 +83,44 @@ test_expect_success 'word diff with a regular expression' '
 
 '
 
+cat > expect-by-chars <<\EOF
+<WHITE>diff --git a/pre b/post<RESET>
+<WHITE>index 330b04f..5ed8eff 100644<RESET>
+<WHITE>--- a/pre<RESET>
+<WHITE>+++ b/post<RESET>
+<BROWN>@@ -1,3 +1,7 @@<RESET>
+<RED>h(4)<RESET><GREEN>h(4),hh[44]<RESET>
+<RESET>
+a = b + c<RESET>
+
+<GREEN>aa = a<RESET>
+
+<GREEN>aeff = aeff * ( aaa )<RESET>
+EOF
+
+test_expect_success 'set a diff driver' '
+	git config diff.testdriver.wordregex "[^[:space:]]" &&
+	cat <<EOF > .gitattributes
+test_* diff=testdriver
+EOF
+'
+
+test_expect_success 'use default supplied by driver' '
+
+	test_must_fail git diff --no-index --color-words \
+		pre post > output &&
+	decrypt_color < output > output.decrypted &&
+	test_cmp expect-by-chars output.decrypted
+
+'
+
+test_expect_success 'option overrides default' '
+
+	test_must_fail git diff --no-index --color-words="[a-z]+" \
+		pre post > output &&
+	decrypt_color < output > output.decrypted &&
+	test_cmp expect output.decrypted
+
+'
+
 test_done
diff --git a/userdiff.c b/userdiff.c
index 3681062..79f9cb9 100644
--- a/userdiff.c
+++ b/userdiff.c
@@ -6,14 +6,20 @@
 static int ndrivers;
 static int drivers_alloc;
 
-#define FUNCNAME(name, pattern) \
-	{ name, NULL, -1, { pattern, REG_EXTENDED } }
+#define PATTERNS(name, pattern, wordregex)			\
+	{ name, NULL, -1, { pattern, REG_EXTENDED }, NULL, wordregex }
 static struct userdiff_driver builtin_drivers[] = {
-FUNCNAME("html", "^[ \t]*(<[Hh][1-6][ \t].*>.*)$"),
-FUNCNAME("java",
+PATTERNS("html", "^[ \t]*(<[Hh][1-6][ \t].*>.*)$",
+	 "[^<>= \t]+|[^[:space:]]|[\x80-\xff]+"),
+PATTERNS("java",
 	 "!^[ \t]*(catch|do|for|if|instanceof|new|return|switch|throw|while)\n"
-	 "^[ \t]*(([ \t]*[A-Za-z_][A-Za-z_0-9]*){2,}[ \t]*\\([^;]*)$"),
-FUNCNAME("objc",
+	 "^[ \t]*(([ \t]*[A-Za-z_][A-Za-z_0-9]*){2,}[ \t]*\\([^;]*)$",
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]="
+	 "|--|\\+\\+|<<=?|>>>?=?|&&|\\|\\|"
+	 "|[^[:space:]]|[\x80-\xff]+"),
+PATTERNS("objc",
 	 /* Negate C statements that can look like functions */
 	 "!^[ \t]*(do|for|if|else|return|switch|while)\n"
 	 /* Objective-C methods */
@@ -21,20 +27,60 @@
 	 /* C functions */
 	 "^[ \t]*(([ \t]*[A-Za-z_][A-Za-z_0-9]*){2,}[ \t]*\\([^;]*)$\n"
 	 /* Objective-C class/protocol definitions */
-	 "^(@(implementation|interface|protocol)[ \t].*)$"),
-FUNCNAME("pascal",
+	 "^(@(implementation|interface|protocol)[ \t].*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"
+	 "|[^[:space:]]|[\x80-\xff]+"),
+PATTERNS("pascal",
 	 "^((procedure|function|constructor|destructor|interface|"
 		"implementation|initialization|finalization)[ \t]*.*)$"
 	 "\n"
-	 "^(.*=[ \t]*(class|record).*)$"),
-FUNCNAME("php", "^[\t ]*((function|class).*)"),
-FUNCNAME("python", "^[ \t]*((class|def)[ \t].*)$"),
-FUNCNAME("ruby", "^[ \t]*((class|module|def)[ \t].*)$"),
-FUNCNAME("bibtex", "(@[a-zA-Z]{1,}[ \t]*\\{{0,1}[ \t]*[^ \t\"@',\\#}{~%]*).*$"),
-FUNCNAME("tex", "^(\\\\((sub)*section|chapter|part)\\*{0,1}\\{.*)$"),
+	 "^(.*=[ \t]*(class|record).*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
+	 "|<>|<=|>=|:=|\\.\\."
+	 "|[^[:space:]]|[\x80-\xff]+"),
+PATTERNS("php", "^[\t ]*((function|class).*)",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
+	 "|[-+*/<>%&^|=!.]=|--|\\+\\+|<<=?|>>=?|===|&&|\\|\\||::|->"
+	 "|[^[:space:]]|[\x80-\xff]+"),
+PATTERNS("python", "^[ \t]*((class|def)[ \t].*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[jJlL]?|0[xX]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|//=?|<<=?|>>=?|\\*\\*=?"
+	 "|[^[:space:]|[\x80-\xff]+"),
+	 /* -- */
+PATTERNS("ruby", "^[ \t]*((class|module|def)[ \t].*)$",
+	 /* -- */
+	 "(@|@@|\\$)?[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+|\\?(\\\\C-)?(\\\\M-)?."
+	 "|//=?|[-+*/<>%&^|=!]=|<<=?|>>=?|===|\\.{1,3}|::|[!=]~"
+	 "|[^[:space:]|[\x80-\xff]+"),
+PATTERNS("bibtex", "(@[a-zA-Z]{1,}[ \t]*\\{{0,1}[ \t]*[^ \t\"@',\\#}{~%]*).*$",
+	 "[={}\"]|[^={}\" \t]+"),
+PATTERNS("tex", "^(\\\\((sub)*section|chapter|part)\\*{0,1}\\{.*)$",
+	 "\\\\[a-zA-Z@]+|[{}]|\\\\.|[^\\{} \t]+"),
+PATTERNS("cpp",
+	 /* Jump targets or access declarations */
+	 "!^[ \t]*[A-Za-z_][A-Za-z_0-9]*:.*$\n"
+	 /* C functions at top level */
+	 "^([A-Za-z_][A-Za-z_0-9]*([ \t]+[A-Za-z_][A-Za-z_0-9]*){1,}[ \t]*\\([^;]*)$\n"
+	 /* compound type at top level */
+	 "^((struct|class|enum)[^;]*)$",
+	 /* -- */
+	 "[a-zA-Z_][a-zA-Z0-9_]*"
+	 "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
+	 "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"
+	 "|[^[:space:]]|[\x80-\xff]+"),
 { "default", NULL, -1, { NULL, 0 } },
 };
-#undef FUNCNAME
+#undef PATTERNS
 
 static struct userdiff_driver driver_true = {
 	"diff=true",
@@ -134,6 +180,8 @@ int userdiff_config(const char *k, const char *v)
 		return parse_string(&drv->external, k, v);
 	if ((drv = parse_driver(k, v, "textconv")))
 		return parse_string(&drv->textconv, k, v);
+	if ((drv = parse_driver(k, v, "wordregex")))
+		return parse_string(&drv->word_regex, k, v);
 
 	return 0;
 }
diff --git a/userdiff.h b/userdiff.h
index ba29457..2aab13e 100644
--- a/userdiff.h
+++ b/userdiff.h
@@ -12,6 +12,7 @@ struct userdiff_driver {
 	int binary;
 	struct userdiff_funcname funcname;
 	const char *textconv;
+	const char *word_regex;
 };
 
 int userdiff_config(const char *k, const char *v);

-- 
1.6.1.140.ge720e.dirty
Previous: Boyd Stephen Smith Jr.Next: Johannes Schindelin
Message 84 of 109 in “refactor the --color-words to make it more hackable”
  1. 0/4 refactor the --color-words to make it more hackableJohannes Schindelin, Jan 11, 2009
  2. 1/4 Add color_fwrite(), a function coloring each line individuallyJohannes Schindelin, Jan 11, 2009
  3. Junio C HamanoJan 11, 2009
  4. Johannes SchindelinJan 11, 2009
  5. 1/4 Add color_fwrite(), a function coloring each line individuallyJohannes Schindelin, Jan 11, 2009
  6. Jakub NarebskiJan 12, 2009
  7. 2/4 color-words: refactor word splitting and use ALLOC_GROW()Johannes Schindelin, Jan 11, 2009
  8. 3/4 color-words: refactor to allow for 0-character word boundariesJohannes Schindelin, Jan 11, 2009
  9. Junio C HamanoJan 11, 2009
  10. Johannes SchindelinJan 11, 2009
  11. Thomas RastJan 12, 2009
  12. Junio C HamanoJan 12, 2009
  13. 0/4 color-words improvementsJohannes Schindelin, Jan 14, 2009
  14. 1/4 Add color_fwrite_lines(), a function coloring each line individuallyJohannes Schindelin, Jan 14, 2009
  15. 2/4 color-words: refactor word splitting and use ALLOC_GROW()Johannes Schindelin, Jan 14, 2009
  16. 3/4 color-words: change algorithm to allow for 0-character word boundariesJohannes Schindelin, Jan 14, 2009
  17. Johannes SchindelinJan 14, 2009
  18. 4/4 color-words: take an optional regular expression describing wordsJohannes Schindelin, Jan 14, 2009
  19. Thomas RastJan 14, 2009
  20. Teemu LikonenJan 14, 2009
  21. Teemu LikonenJan 14, 2009
  22. Johannes SchindelinJan 14, 2009
  23. Johannes SchindelinJan 14, 2009
  24. Thomas RastJan 14, 2009
  25. Johannes SchindelinJan 14, 2009
  26. Thomas RastJan 14, 2009
  27. Boyd Stephen Smith Jr.Jan 14, 2009
  28. Teemu LikonenJan 15, 2009
  29. Johannes SchindelinJan 15, 2009
  30. Teemu LikonenJan 15, 2009
  31. Thomas RastJan 15, 2009
  32. Junio C HamanoJan 15, 2009
  33. Johannes SchindelinJan 15, 2009
  34. Santi BéjarJan 16, 2009
  35. Junio C HamanoJan 16, 2009
  36. Boyd Stephen Smith Jr.Jan 16, 2009
  37. Johannes SchindelinJan 16, 2009
  38. Santi BéjarJan 16, 2009
  39. Johannes SchindelinJan 16, 2009
  40. Santi BéjarJan 16, 2009
  41. Johannes SchindelinJan 16, 2009
  42. Thomas RastJan 16, 2009
  43. Johannes SchindelinJan 16, 2009
  44. 0/7 customizable --color-wordsThomas Rast, Jan 17, 2009
  45. 1/7 Add color_fwrite_lines(), a function coloring each line individuallyThomas Rast, Jan 17, 2009
  46. 2/7 color-words: refactor word splitting and use ALLOC_GROW()Thomas Rast, Jan 17, 2009
  47. 3/7 color-words: change algorithm to allow for 0-character word boundariesThomas Rast, Jan 17, 2009
  48. 4/7 color-words: take an optional regular expression describing wordsThomas Rast, Jan 17, 2009
  49. 5/7 color-words: enable REG_NEWLINE to help userThomas Rast, Jan 17, 2009
  50. 6/7 color-words: expand docs with precise semanticsThomas Rast, Jan 17, 2009
  51. 7/7 color-words: make regex configurable via attributesThomas Rast, Jan 17, 2009
  52. Santi BéjarJan 18, 2009
  53. Santi BéjarJan 18, 2009
  54. Santi BéjarJan 19, 2009
  55. Johannes SchindelinJan 19, 2009
  56. Add tests for diff.color-words configuration option.Boyd Stephen Smith Jr., Jan 20, 2009
  57. diff: Support diff.color-words config optionBoyd Stephen Smith Jr., Jan 20, 2009
  58. Junio C HamanoJan 20, 2009
  59. Markus HeidelbergJan 20, 2009
  60. Boyd Stephen Smith Jr.Jan 20, 2009
  61. Johannes SchindelinJan 20, 2009
  62. Junio C HamanoJan 21, 2009
  63. Markus HeidelbergJan 21, 2009
  64. Johannes SchindelinJan 20, 2009
  65. Boyd Stephen Smith Jr.Jan 20, 2009
  66. Johannes SchindelinJan 20, 2009
  67. Junio C HamanoJan 20, 2009
  68. Johannes SchindelinJan 20, 2009
  69. Junio C HamanoJan 20, 2009
  70. Johannes SchindelinJan 20, 2009
  71. color-words: Support diff.color-words config optionBoyd Stephen Smith Jr., Jan 21, 2009
  72. Change the spelling of "wordregex".Boyd Stephen Smith Jr., Jan 21, 2009
  73. Johannes SchindelinJan 21, 2009
  74. Thomas RastJan 21, 2009
  75. Boyd Stephen Smith Jr.Jan 21, 2009
  76. Johannes SchindelinJan 21, 2009
  77. Boyd Stephen Smith Jr.Jan 21, 2009
  78. Junio C HamanoJan 21, 2009
  79. Jakub NarebskiJan 20, 2009
  80. Johannes SchindelinJan 20, 2009
  81. Boyd Stephen Smith Jr.Jan 20, 2009
  82. Johannes SchindelinJan 20, 2009
  83. Boyd Stephen Smith Jr.Jan 16, 2009
  84. color-words: make regex configurable via attributesThomas Rast, Jan 14, 2009
  85. Johannes SchindelinJan 14, 2009
  86. Thomas RastJan 14, 2009
  87. 3/4 color-words: change algorithm to allow for 0-character word boundariesJohannes Schindelin, Jan 14, 2009
  88. 4/4 color-words: take an optional regular expression describing wordsJohannes Schindelin, Jan 14, 2009
  89. Thomas RastJan 15, 2009
  90. Johannes SchindelinJan 15, 2009
  91. Johannes SchindelinJan 15, 2009
  92. Thomas RastJan 15, 2009
  93. Thomas RastJan 15, 2009
  94. Johannes SchindelinJan 15, 2009
  95. 1/4 color-words: fix quoting in t4034Thomas Rast, Jan 14, 2009
  96. 2/4 color-words: enable REG_NEWLINE to help userThomas Rast, Jan 14, 2009
  97. 3/4 color-words: expand docs with precise semanticsThomas Rast, Jan 14, 2009
  98. 4/4 color-words: make regex configurable via attributesThomas Rast, Jan 14, 2009
  99. Johannes SchindelinJan 15, 2009
  100. Johannes SchindelinJan 15, 2009
  101. Johannes SchindelinJan 14, 2009
  102. Thomas RastJan 14, 2009
  103. Johannes SchindelinJan 14, 2009
  104. Thomas RastJan 14, 2009
  105. 4/4 color-words: take an optional regular expression describing wordsJohannes Schindelin, Jan 11, 2009
  106. Thomas RastJan 11, 2009
  107. Johannes SchindelinJan 11, 2009
  108. Thomas RastJan 12, 2009
  109. Santi BéjarJan 14, 2009

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.