git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v2 05/12] grep/icase: avoid kwsset when -F is specified

From
Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Date
Jun 25, 2016, 05:22 UTC
Message-ID
<20160625052238.13615-6-pclouds@gmail.com>
In-Reply-To
<20160625052238.13615-1-pclouds@gmail.com>

Similar to the previous commit, we can't use kws on icase search outside ascii range. But we can't simply pass the pattern to regcomp/pcre like the previous commit because it may contain regex special characters, so we need to quote the regex first.

To avoid misquote traps that could lead to undefined behavior, we always stick to basic regex engine in this case. We don't need fancy features for grepping a literal string anyway.

basic_regex_quote_buf() assumes that if the pattern is in a multibyte encoding, ascii chars must be unambiguously encoded as single bytes. This is true at least for UTF-8. For others, let's wait until people yell up. Chances are nobody uses multibyte, non utf-8 charsets anymore.

Noticed-by: Plamen Totev <plamen.totev@abv.bg>
Helped-by: René Scharfe <l.s.r@web.de>
Helped-by: Eric Sunshine <sunshine@sunshineco.com>
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 grep.c                          | 24 +++++++++++++++++++++++-
 quote.c                         | 37 +++++++++++++++++++++++++++++++++++++
 quote.h                         |  1 +
 t/t7812-grep-icase-non-ascii.sh | 26 ++++++++++++++++++++++++++
 4 files changed, 87 insertions(+), 1 deletion(-)
diff --git a/grep.c b/grep.c
index 451275d..5a8c52a 100644
--- a/grep.c
+++ b/grep.c
@@ -5,6 +5,7 @@
 #include "diff.h"
 #include "diffcore.h"
 #include "commit.h"
+#include "quote.h"
 
 static int grep_source_load(struct grep_source *gs);
 static int grep_source_is_binary(struct grep_source *gs);
@@ -397,6 +398,24 @@ static int is_fixed(const char *s, size_t len)
 	return 1;
 }
 
+static void compile_fixed_regexp(struct grep_pat *p, struct grep_opt *opt)
+{
+	struct strbuf sb = STRBUF_INIT;
+	int err;
+
+	basic_regex_quote_buf(&sb, p->pattern);
+	err = regcomp(&p->regexp, sb.buf, opt->regflags & ~REG_EXTENDED);
+	if (opt->debug)
+		fprintf(stderr, "fixed %s\n", sb.buf);
+	strbuf_release(&sb);
+	if (err) {
+		char errbuf[1024];
+		regerror(err, &p->regexp, errbuf, sizeof(errbuf));
+		regfree(&p->regexp);
+		compile_regexp_failed(p, errbuf);
+	}
+}
+
 static void compile_regexp(struct grep_pat *p, struct grep_opt *opt)
 {
 	int icase, ascii_only;
@@ -408,7 +427,7 @@ static void compile_regexp(struct grep_pat *p, struct grep_opt *opt)
 	ascii_only     = !has_non_ascii(p->pattern);
 
 	if (opt->fixed)
-		p->fixed = 1;
+		p->fixed = !icase || ascii_only;
 	else if ((!icase || ascii_only) &&
 		 is_fixed(p->pattern, p->patternlen))
 		p->fixed = 1;
@@ -423,6 +442,9 @@ static void compile_regexp(struct grep_pat *p, struct grep_opt *opt)
 		kwsincr(p->kws, p->pattern, p->patternlen);
 		kwsprep(p->kws);
 		return;
+	} else if (opt->fixed) {
+		compile_fixed_regexp(p, opt);
+		return;
 	}
 
 	if (opt->pcre) {
diff --git a/quote.c b/quote.c
index fe884d2..c67adb7 100644
--- a/quote.c
+++ b/quote.c
@@ -440,3 +440,40 @@ void tcl_quote_buf(struct strbuf *sb, const char *src)
 	}
 	strbuf_addch(sb, '"');
 }
+
+void basic_regex_quote_buf(struct strbuf *sb, const char *src)
+{
+	char c;
+
+	if (*src == '^') {
+		/* only beginning '^' is special and needs quoting */
+		strbuf_addch(sb, '\\');
+		strbuf_addch(sb, *src++);
+	}
+	if (*src == '*')
+		/* beginning '*' is not special, no quoting */
+		strbuf_addch(sb, *src++);
+
+	while ((c = *src++)) {
+		switch (c) {
+		case '[':
+		case '.':
+		case '\\':
+		case '*':
+			strbuf_addch(sb, '\\');
+			strbuf_addch(sb, c);
+			break;
+
+		case '$':
+			/* only the end '$' is special and needs quoting */
+			if (*src == '\0')
+				strbuf_addch(sb, '\\');
+			strbuf_addch(sb, c);
+			break;
+
+		default:
+			strbuf_addch(sb, c);
+			break;
+		}
+	}
+}
diff --git a/quote.h b/quote.h
index 99e04d3..362d315 100644
--- a/quote.h
+++ b/quote.h
@@ -67,5 +67,6 @@ extern char *quote_path_relative(const char *in, const char *prefix,
 extern void perl_quote_buf(struct strbuf *sb, const char *src);
 extern void python_quote_buf(struct strbuf *sb, const char *src);
 extern void tcl_quote_buf(struct strbuf *sb, const char *src);
+extern void basic_regex_quote_buf(struct strbuf *sb, const char *src);
 
 #endif
diff --git a/t/t7812-grep-icase-non-ascii.sh b/t/t7812-grep-icase-non-ascii.sh
index b78a774..1929809 100755
--- a/t/t7812-grep-icase-non-ascii.sh
+++ b/t/t7812-grep-icase-non-ascii.sh
@@ -20,4 +20,30 @@ test_expect_success REGEX_LOCALE 'grep literal string, no -F' '
 	git grep -i "TILRAUN: HALLÓ HEIMUR!"
 '
 
+test_expect_success REGEX_LOCALE 'grep literal string, with -F' '
+	git grep --debug -i -F "TILRAUN: Halló Heimur!"  2>&1 >/dev/null |
+		 grep fixed >debug1 &&
+	test_write_lines "fixed TILRAUN: Halló Heimur!" >expect1 &&
+	test_cmp expect1 debug1 &&
+
+	git grep --debug -i -F "TILRAUN: HALLÓ HEIMUR!"  2>&1 >/dev/null |
+		 grep fixed >debug2 &&
+	test_write_lines "fixed TILRAUN: HALLÓ HEIMUR!" >expect2 &&
+	test_cmp expect2 debug2
+'
+
+test_expect_success REGEX_LOCALE 'grep string with regex, with -F' '
+	test_write_lines "^*TILR^AUN:.* \\Halló \$He[]imur!\$" >file &&
+
+	git grep --debug -i -F "^*TILR^AUN:.* \\Halló \$He[]imur!\$" 2>&1 >/dev/null |
+		 grep fixed >debug1 &&
+	test_write_lines "fixed \\^*TILR^AUN:\\.\\* \\\\Halló \$He\\[]imur!\\\$" >expect1 &&
+	test_cmp expect1 debug1 &&
+
+	git grep --debug -i -F "^*TILR^AUN:.* \\HALLÓ \$HE[]IMUR!\$"  2>&1 >/dev/null |
+		 grep fixed >debug2 &&
+	test_write_lines "fixed \\^*TILR^AUN:\\.\\* \\\\HALLÓ \$HE\\[]IMUR!\\\$" >expect2 &&
+	test_cmp expect2 debug2
+'
+
 test_done

-- 
2.8.2.526.g02eed6d
Previous: Nguyễn Thái Ngọc DuyNext: Nguyễn Thái Ngọc Duy
Message 24 of 39 in “nd/icase updates”
  1. 00/11 nd/icase updatesNguyễn Thái Ngọc Duy, Jun 23, 2016
  2. 02/11 test-regex: isolate the bug test codeNguyễn Thái Ngọc Duy, Jun 23, 2016
  3. 05/11 grep/icase: avoid kwsset when -F is specifiedNguyễn Thái Ngọc Duy, Jun 23, 2016
  4. Junio C HamanoJun 23, 2016
  5. 09/11 diffcore-pickaxe: "share" regex error handling codeNguyễn Thái Ngọc Duy, Jun 23, 2016
  6. Jeff KingJun 23, 2016
  7. 11/11 grep.c: reuse "icase" variableNguyễn Thái Ngọc Duy, Jun 23, 2016
  8. 03/11 test-regex: expose full regcomp() to the command lineNguyễn Thái Ngọc Duy, Jun 23, 2016
  9. 08/11 grep/pcre: support utf-8Nguyễn Thái Ngọc Duy, Jun 23, 2016
  10. 10/11 diffcore-pickaxe: support case insensitive match on non-asciiNguyễn Thái Ngọc Duy, Jun 23, 2016
  11. 07/11 gettext: add is_utf8_locale()Nguyễn Thái Ngọc Duy, Jun 23, 2016
  12. 01/11 grep: break down an "if" stmt in preparation for next changesNguyễn Thái Ngọc Duy, Jun 23, 2016
  13. 04/11 grep/icase: avoid kwsset on literal non-ascii stringsNguyễn Thái Ngọc Duy, Jun 23, 2016
  14. Junio C HamanoJun 23, 2016
  15. 06/11 grep/pcre: prepare locale-dependent tables for icase matchingNguyễn Thái Ngọc Duy, Jun 23, 2016
  16. Junio C HamanoJun 23, 2016
  17. 00/12 nd/icase updatesNguyễn Thái Ngọc Duy, Jun 25, 2016
  18. 03/12 test-regex: expose full regcomp() to the command lineNguyễn Thái Ngọc Duy, Jun 25, 2016
  19. 04/12 grep/icase: avoid kwsset on literal non-ascii stringsNguyễn Thái Ngọc Duy, Jun 25, 2016
  20. 07/12 grep/pcre: prepare locale-dependent tables for icase matchingNguyễn Thái Ngọc Duy, Jun 25, 2016
  21. 06/12 grep: rewrite an if/else condition to avoid duplicate expressionNguyễn Thái Ngọc Duy, Jun 25, 2016
  22. 10/12 diffcore-pickaxe: Add regcomp_or_die()Nguyễn Thái Ngọc Duy, Jun 25, 2016
  23. 09/12 grep/pcre: support utf-8Nguyễn Thái Ngọc Duy, Jun 25, 2016
  24. 05/12 grep/icase: avoid kwsset when -F is specifiedNguyễn Thái Ngọc Duy, Jun 25, 2016
  25. 12/12 grep.c: reuse "icase" variableNguyễn Thái Ngọc Duy, Jun 25, 2016
  26. 08/12 gettext: add is_utf8_locale()Nguyễn Thái Ngọc Duy, Jun 25, 2016
  27. 01/12 grep: break down an "if" stmt in preparation for next changesNguyễn Thái Ngọc Duy, Jun 25, 2016
  28. 11/12 diffcore-pickaxe: support case insensitive match on non-asciiNguyễn Thái Ngọc Duy, Jun 25, 2016
  29. 02/12 test-regex: isolate the bug test codeNguyễn Thái Ngọc Duy, Jun 25, 2016
  30. Junio C HamanoJun 27, 2016
  31. Junio C HamanoJun 27, 2016
  32. Duy NguyenJun 30, 2016
  33. Junio C HamanoJul 1, 2016
  34. Duy NguyenJul 1, 2016
  35. Junio C HamanoJul 1, 2016
  36. Junio C HamanoJul 1, 2016
  37. Junio C HamanoJul 1, 2016
  38. Junio C HamanoJul 1, 2016
  39. Junio C HamanoJul 1, 2016

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.