git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 3/4] t/greplint: move parsing and token check to library

From
Tyler Cipriani <tyler@tylercipriani.com>
Date
Oct 25, 2026, 06:35 UTC
Message-ID
<20261025063508.3026136-4-tyler@tylercipriani.com>
In-Reply-To
<20261025063508.3026136-1-tyler@tylercipriani.com>

Move greplint.pl's parsing and token check work to a library. Modify the greplint.pl to use the library. Break out actual checks into check_tokens() sub.

In preparation to combine the work of chainlint and greplint into a single parser invocation.

Signed-off-by: Tyler Cipriani <tyler@tylercipriani.com>
---
 t/greplint.pl                      | 249 +----------------------------
 t/{greplint.pl => lib-greplint.pl} |  69 ++++----
 2 files changed, 37 insertions(+), 281 deletions(-)
 copy t/{greplint.pl => lib-greplint.pl} (87%)
diff --git a/t/greplint.pl b/t/greplint.pl
index 68b6eba78d..e277647f11 100644
--- a/t/greplint.pl
+++ b/t/greplint.pl
@@ -1,257 +1,16 @@
 #!/usr/bin/env perl
 
-# Detect bare 'grep' used as a test assertion where 'test_grep'
-# should be used, and '! test_grep' where 'test_grep !' should
-# be used.
-#
-# The shared shell parser tokenizes test bodies so that 'grep'
-# inside heredocs, command substitutions like $(grep ...), and
-# quoted strings is collapsed into a single token and never seen
-# by our check.  A line-oriented approach would need to track
-# heredoc delimiters, nested $() depth, and cross-line pipe
-# state to avoid false positives on patterns like:
-#
-#   write_script foo.sh <<-\EOF
-#   grep pattern file    # data, not an assertion
-#   EOF
-#
-# The Lexer already handles these.
-
+# Detect bare 'grep' used as a test assertion
 use warnings;
 use strict;
 use File::Basename;
 do(dirname($0) . "/lib-shell-parser.pl")
 	or die "$0: failed to load lib-shell-parser.pl: $@$!\n";
+do(dirname($0) . "/lib-greplint.pl")
+	or die "$0: failed to load lib-greplint.pl: $@$!\n";
 
 my $exit_code = 0;
 
-# GrepLintParser inherits ScriptParser's ability to find
-# test_expect_success/failure blocks and call check_test()
-# on each body.  We override check_test() to walk the token
-# stream looking for bare grep assertions.
-package GrepLintParser;
-
-our @ISA = ('ScriptParser');
-
-# After these tokens, the next token is a command word.
-# For example, in 'echo foo && grep bar file', the 'grep'
-# after '&&' is at command position and should be flagged.
-my %cmd_start = map { $_ => 1 } qw(&& || ; ;; do then else elif), "\n", '{', '(';
-
-# Tokens indicating grep's output is piped or redirected.
-my %filter_op = map { $_ => 1 } qw(| > >> <);
-
-# A token is at "command word" position if the shell would
-# interpret it as a program name rather than an argument.
-# Only 'grep' at command position is an assertion we should
-# flag; 'grep' as an argument ('test_must_fail grep') or
-# value ('for cmd in grep sed') is not.
-sub is_command_word {
-	my ($tokens, $pos) = @_;
-	return 1 if $pos == 0;
-	for (my $j = $pos - 1; $j >= 0; $j--) {
-		my $t = $tokens->[$j]->[0];
-		# After a separator or pipe, a new command starts.
-		return 1 if $cmd_start{$t} || $t eq '|';
-		# After '}' or ')', what follows is a separator or
-		# redirect on the compound command, not a new command.
-		return 0 if $t eq '}' || $t eq ')';
-		# '!' is a prefix that does not consume command
-		# position; keep scanning to find what precedes it.
-		next if $t eq '!';
-		# Any other word means we are past the command word.
-		return 0;
-	}
-	return 1;
-}
-
-# lint_ok() reports whether a bare grep carries a trailing
-# '# lint-ok' comment telling this linter to skip it.
-#
-# In practice this is needed for just one case: a grep acting
-# as a data filter whose output is consumed by a redirect or
-# pipe on an enclosing compound command (such as a subshell or
-# brace group) rather than by grep's own pipeline, e.g.
-#
-#	( grep ... && # lint-ok
-#	  sed ... ) >out
-#
-#	{ grep ... || : # lint-ok
-#	} >out
-#
-# is_filter() only scans grep's own pipeline: it stops at the
-# separator before the compound command closes and never sees
-# the outer redirect, so it would flag such a grep as an
-# assertion.  A grep that really is an assertion is better
-# written as test_grep (or a guarded test_grep when the file's
-# presence is conditional) than annotated with lint-ok.
-sub lint_ok {
-	my ($raw_lines, $ln) = @_;
-	if ($ln < 1 || $ln > @$raw_lines) {
-		warn "lint_ok: line number $ln out of range (1.." .
-		    scalar(@$raw_lines) . ")\n";
-		return 0;
-	}
-	return $raw_lines->[$ln - 1] =~ /lint-ok/;
-}
-
-# Grep is a filter (not an assertion) if it receives piped
-# input or sends its output to a pipe or redirect.  Check
-# both directions from grep's position in the token stream.
-sub is_filter {
-	my ($tokens, $pos) = @_;
-	# Backward: is grep receiving piped input?
-	# Newlines don't break pipes ('cmd |\n grep' is one
-	# pipeline), so skip past them.
-	for (my $j = $pos - 1; $j >= 0; $j--) {
-		my $t = $tokens->[$j]->[0];
-		return 1 if $t eq '|';
-		next if $t eq "\n";
-		last if $cmd_start{$t} || $t eq '}' || $t eq ')';
-	}
-	# Forward: is grep piping or redirecting output?
-	# Unlike the backward scan, we do not skip newlines here:
-	# a bare newline is a command boundary, and redirects or
-	# pipes must appear on the same line as grep (or after a
-	# line continuation, which the Lexer consumes).
-	for (my $j = $pos + 1; $j < @$tokens; $j++) {
-		my $t = $tokens->[$j]->[0];
-		return 0 if $cmd_start{$t};
-		return 1 if $filter_op{$t};
-	}
-	return 0;
-}
-
-# Map a body-relative line number to a file line number.
-# For double-quoted bodies, backslash-continuation lines
-# (\<newline>) are consumed by the Lexer without appearing
-# in the body text, so the inner parser sees fewer lines
-# than the source file has.  We walk the source lines to
-# count continuations and adjust accordingly.
-sub body_to_file_line {
-	my ($body_lineno, $body_token, $raw_lines, $body_start) = @_;
-	my $body_text = $body_token->[0];
-	my $body_end_line = $body_token->[4];
-	unless ($body_start && $body_start >= 1) {
-		warn "body_start is not a positive integer\n";
-		return $body_lineno;
-	}
-	my $file_lineno = $body_lineno + $body_start - 1;
-	# Only double-quoted bodies have line splices.
-	return $file_lineno unless $body_text =~ /^"/;
-	my $adj = 0;
-	my $lines_seen = 0;
-	unless ($body_end_line && $body_end_line >= $body_start) {
-		warn "body_end_line is not set for double-quoted body\n";
-		return $file_lineno;
-	}
-	my $end = $body_end_line;
-	if ($end > @$raw_lines) {
-		warn "body_end_line ($end) exceeds file length (" .
-		    scalar(@$raw_lines) . ")\n";
-		return $file_lineno;
-	}
-	my $src_ln = $body_start;
-	while ($src_ln <= $end && $lines_seen < $body_lineno) {
-		my $line = $raw_lines->[$src_ln - 1];
-		# Odd trailing backslashes = continuation (\<nl>).
-		# Even = escaped backslashes (\\), not a continuation.
-		if ($line =~ /(\\*)$/ && length($1) % 2 == 1) {
-			$adj++;
-		} else {
-			$lines_seen++;
-		}
-		$src_ln++;
-	}
-	if ($lines_seen < $body_lineno) {
-		warn "body_lineno ($body_lineno) not found within body range " .
-		    "($body_start..$end)\n";
-	}
-	return $file_lineno + $adj;
-}
-
-# ScriptParser calls this for each test body found in the script.
-sub check_test {
-	my $self = shift @_;
-	my $title = ScriptParser::unwrap(shift @_);
-	my $body_token = shift @_;
-	my $body_start = $body_token->[3];
-	my $body = ScriptParser::unwrap($body_token);
-	# Handle heredoc-style test bodies:
-	#   test_expect_success 'title' - <<\EOF
-	#   grep pattern file
-	#   EOF
-	# The '-' signals that the body follows as a heredoc.
-	if ($body eq '-') {
-		my $herebody = shift @_;
-		if ($herebody) {
-			$body = $herebody->{content};
-			$body_start = $herebody->{start_line};
-		}
-	}
-	return unless $body;
-	# We can never match a bare "grep" if there isn't one in the first
-	# place. So we can skip all of the parsing effort. Technically you
-	# can have:
-	#
-	#   gr\
-	#   ep
-	#
-	# but that is sufficiently insane that I think we can ignore it.
-	return unless $body =~ /grep/;
-
-	my $raw_lines = $self->{raw_lines};
-
-	# The outer parser gives us the body as an opaque string.
-	# Parse it to get individual tokens with command boundaries.
-	my $parser = ShellParser->new(\$body);
-	my @tokens = $parser->parse();
-
-	my $file = $self->{file};
-
-	for (my $i = 0; $i < @tokens; $i++) {
-		my $text = $tokens[$i]->[0];
-		next unless is_command_word(\@tokens, $i);
-
-		my $token_lineno = $tokens[$i]->[3];
-		unless (defined($token_lineno) && $token_lineno >= 1) {
-			warn "token has no line number\n";
-			next;
-		}
-		my $file_lineno = body_to_file_line(
-			$token_lineno,
-			$body_token, $raw_lines, $body_start);
-
-		# '!' negates the exit code without consuming command
-		# position.  '! test_grep' is an anti-pattern because
-		# test_grep only prints diagnostics on grep failure,
-		# and '!' inverts after that decision is already made.
-		if ($text eq '!') {
-			if ($i + 1 < @tokens &&
-			    $tokens[$i + 1]->[0] eq 'test_grep' &&
-			    !lint_ok($raw_lines, $file_lineno)) {
-				print "$file:$file_lineno: error: ",
-				    'use "test_grep !" instead of ',
-				    '"! test_grep"', "\n";
-				$exit_code = 1;
-			}
-			next;
-		}
-
-		# Bare grep as a command (not a filter) is a test
-		# assertion that should use test_grep for better
-		# failure diagnostics.
-		if ($text eq 'grep' &&
-		    !is_filter(\@tokens, $i) &&
-		    !lint_ok($raw_lines, $file_lineno)) {
-			print "$file:$file_lineno: error: ",
-			    "bare grep outside pipeline ",
-			    "(use test_grep)\n";
-			$exit_code = 1;
-		}
-	}
-}
-
 package main;
 
 for my $file (@ARGV) {
@@ -263,5 +22,7 @@ package main;
 	$parser->{file} = $file;
 	$parser->{raw_lines} = \@raw_lines;
 	$parser->parse();
+	print "$_\n" for @{$parser->{output}};
+	$exit_code ||= $parser->{nerrs} > 0;
 }
 exit $exit_code;
diff --git a/t/greplint.pl b/t/lib-greplint.pl
similarity index 87%
copy from t/greplint.pl
copy to t/lib-greplint.pl
index 68b6eba78d..16b85b8d3b 100644
--- a/t/greplint.pl
+++ b/t/lib-greplint.pl
@@ -1,5 +1,3 @@
-#!/usr/bin/env perl
-
 # Detect bare 'grep' used as a test assertion where 'test_grep'
 # should be used, and '! test_grep' where 'test_grep !' should
 # be used.
@@ -19,16 +17,13 @@
 
 use warnings;
 use strict;
-use File::Basename;
-do(dirname($0) . "/lib-shell-parser.pl")
-	or die "$0: failed to load lib-shell-parser.pl: $@$!\n";
-
-my $exit_code = 0;
 
 # GrepLintParser inherits ScriptParser's ability to find
 # test_expect_success/failure blocks and call check_test()
 # on each body.  We override check_test() to walk the token
-# stream looking for bare grep assertions.
+# stream looking for bare grep assertions vs. test_grep.
+#
+# Requires lib-shell-parser.pl to be loaded.
 package GrepLintParser;
 
 our @ISA = ('ScriptParser');
@@ -207,13 +202,25 @@ sub check_test {
 	my $parser = ShellParser->new(\$body);
 	my @tokens = $parser->parse();
 
-	my $file = $self->{file};
+	my @errors = check_tokens(
+		$self->{file}, $raw_lines,
+		$body_token, $body_start, \@tokens);
 
-	for (my $i = 0; $i < @tokens; $i++) {
-		my $text = $tokens[$i]->[0];
-		next unless is_command_word(\@tokens, $i);
+	push(@{$self->{output}}, @errors);
+	$self->{nerrs} += @errors;
+}
 
-		my $token_lineno = $tokens[$i]->[3];
+# check_tokens() walks a parsed test body and returns a list of
+# error messages.
+sub check_tokens {
+	my ($file, $raw_lines, $body_token, $body_start, $tokens) = @_;
+	my @errors;
+
+	for (my $i = 0; $i < @$tokens; $i++) {
+		my $text = $tokens->[$i]->[0];
+		next unless is_command_word($tokens, $i);
+
+		my $token_lineno = $tokens->[$i]->[3];
 		unless (defined($token_lineno) && $token_lineno >= 1) {
 			warn "token has no line number\n";
 			next;
@@ -227,13 +234,12 @@ sub check_test {
 		# test_grep only prints diagnostics on grep failure,
 		# and '!' inverts after that decision is already made.
 		if ($text eq '!') {
-			if ($i + 1 < @tokens &&
-			    $tokens[$i + 1]->[0] eq 'test_grep' &&
+			if ($i + 1 < @$tokens &&
+			    $tokens->[$i + 1]->[0] eq 'test_grep' &&
 			    !lint_ok($raw_lines, $file_lineno)) {
-				print "$file:$file_lineno: error: ",
-				    'use "test_grep !" instead of ',
-				    '"! test_grep"', "\n";
-				$exit_code = 1;
+				push(@errors, "$file:$file_lineno: error: " .
+				    'use "test_grep !" instead of ' .
+				    '"! test_grep"');
 			}
 			next;
 		}
@@ -242,26 +248,15 @@ sub check_test {
 		# assertion that should use test_grep for better
 		# failure diagnostics.
 		if ($text eq 'grep' &&
-		    !is_filter(\@tokens, $i) &&
+		    !is_filter($tokens, $i) &&
 		    !lint_ok($raw_lines, $file_lineno)) {
-			print "$file:$file_lineno: error: ",
-			    "bare grep outside pipeline ",
-			    "(use test_grep)\n";
-			$exit_code = 1;
+			push(@errors, "$file:$file_lineno: error: " .
+			    "bare grep outside pipeline " .
+			    "(use test_grep)");
 		}
 	}
+
+	return @errors;
 }
 
-package main;
-
-for my $file (@ARGV) {
-	open(my $fh, '<:unix:crlf', $file) or die "$0: $file: $!\n";
-	my @raw_lines = <$fh>;
-	close $fh;
-	my $s = join('', @raw_lines);
-	my $parser = GrepLintParser->new(\$s);
-	$parser->{file} = $file;
-	$parser->{raw_lines} = \@raw_lines;
-	$parser->parse();
-}
-exit $exit_code;
+1;
-- 
2.47.3
Previous: Tyler CiprianiNext: Tyler Cipriani
Message 6 of 7 in “test-grep-lint is uncomfortably slow”
  1. Junio C HamanoOct 6, 2026
  2. Jeff KingOct 14, 2026
  3. 0/4 t: run greplint checks inside chainlintTyler Cipriani, Oct 25, 2026
  4. 1/4 t/interop: replace bare grep with test_grepTyler Cipriani, Oct 25, 2026
  5. 2/4 t/greplint: skip test bodies without the word grepTyler Cipriani, Oct 25, 2026
  6. 3/4 t/greplint: move parsing and token check to libraryTyler Cipriani, Oct 25, 2026
  7. 4/4 t/chainlint: add --greplint to run greplintTyler Cipriani, Oct 25, 2026

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.