[PATCH 3/4] t/greplint: move parsing and token check to library
Move greplint.pl's parsing and token check work to a library. Modify the greplint.pl to use the library. Break out actual checks into check_tokens() sub.
In preparation to combine the work of chainlint and greplint into a single parser invocation.
Signed-off-by: Tyler Cipriani <tyler@tylercipriani.com>
---
t/greplint.pl | 249 +----------------------------
t/{greplint.pl => lib-greplint.pl} | 69 ++++----
2 files changed, 37 insertions(+), 281 deletions(-)
copy t/{greplint.pl => lib-greplint.pl} (87%)diff --git a/t/greplint.pl b/t/greplint.pl
index 68b6eba78d..e277647f11 100644
--- a/t/greplint.pl
+++ b/t/greplint.pl
@@ -1,257 +1,16 @@
#!/usr/bin/env perl
-# Detect bare 'grep' used as a test assertion where 'test_grep'
-# should be used, and '! test_grep' where 'test_grep !' should
-# be used.
-#
-# The shared shell parser tokenizes test bodies so that 'grep'
-# inside heredocs, command substitutions like $(grep ...), and
-# quoted strings is collapsed into a single token and never seen
-# by our check. A line-oriented approach would need to track
-# heredoc delimiters, nested $() depth, and cross-line pipe
-# state to avoid false positives on patterns like:
-#
-# write_script foo.sh <<-\EOF
-# grep pattern file # data, not an assertion
-# EOF
-#
-# The Lexer already handles these.
-
+# Detect bare 'grep' used as a test assertion
use warnings;
use strict;
use File::Basename;
do(dirname($0) . "/lib-shell-parser.pl")
or die "$0: failed to load lib-shell-parser.pl: $@$!\n";
+do(dirname($0) . "/lib-greplint.pl")
+ or die "$0: failed to load lib-greplint.pl: $@$!\n";
my $exit_code = 0;
-# GrepLintParser inherits ScriptParser's ability to find
-# test_expect_success/failure blocks and call check_test()
-# on each body. We override check_test() to walk the token
-# stream looking for bare grep assertions.
-package GrepLintParser;
-
-our @ISA = ('ScriptParser');
-
-# After these tokens, the next token is a command word.
-# For example, in 'echo foo && grep bar file', the 'grep'
-# after '&&' is at command position and should be flagged.
-my %cmd_start = map { $_ => 1 } qw(&& || ; ;; do then else elif), "\n", '{', '(';
-
-# Tokens indicating grep's output is piped or redirected.
-my %filter_op = map { $_ => 1 } qw(| > >> <);
-
-# A token is at "command word" position if the shell would
-# interpret it as a program name rather than an argument.
-# Only 'grep' at command position is an assertion we should
-# flag; 'grep' as an argument ('test_must_fail grep') or
-# value ('for cmd in grep sed') is not.
-sub is_command_word {
- my ($tokens, $pos) = @_;
- return 1 if $pos == 0;
- for (my $j = $pos - 1; $j >= 0; $j--) {
- my $t = $tokens->[$j]->[0];
- # After a separator or pipe, a new command starts.
- return 1 if $cmd_start{$t} || $t eq '|';
- # After '}' or ')', what follows is a separator or
- # redirect on the compound command, not a new command.
- return 0 if $t eq '}' || $t eq ')';
- # '!' is a prefix that does not consume command
- # position; keep scanning to find what precedes it.
- next if $t eq '!';
- # Any other word means we are past the command word.
- return 0;
- }
- return 1;
-}
-
-# lint_ok() reports whether a bare grep carries a trailing
-# '# lint-ok' comment telling this linter to skip it.
-#
-# In practice this is needed for just one case: a grep acting
-# as a data filter whose output is consumed by a redirect or
-# pipe on an enclosing compound command (such as a subshell or
-# brace group) rather than by grep's own pipeline, e.g.
-#
-# ( grep ... && # lint-ok
-# sed ... ) >out
-#
-# { grep ... || : # lint-ok
-# } >out
-#
-# is_filter() only scans grep's own pipeline: it stops at the
-# separator before the compound command closes and never sees
-# the outer redirect, so it would flag such a grep as an
-# assertion. A grep that really is an assertion is better
-# written as test_grep (or a guarded test_grep when the file's
-# presence is conditional) than annotated with lint-ok.
-sub lint_ok {
- my ($raw_lines, $ln) = @_;
- if ($ln < 1 || $ln > @$raw_lines) {
- warn "lint_ok: line number $ln out of range (1.." .
- scalar(@$raw_lines) . ")\n";
- return 0;
- }
- return $raw_lines->[$ln - 1] =~ /lint-ok/;
-}
-
-# Grep is a filter (not an assertion) if it receives piped
-# input or sends its output to a pipe or redirect. Check
-# both directions from grep's position in the token stream.
-sub is_filter {
- my ($tokens, $pos) = @_;
- # Backward: is grep receiving piped input?
- # Newlines don't break pipes ('cmd |\n grep' is one
- # pipeline), so skip past them.
- for (my $j = $pos - 1; $j >= 0; $j--) {
- my $t = $tokens->[$j]->[0];
- return 1 if $t eq '|';
- next if $t eq "\n";
- last if $cmd_start{$t} || $t eq '}' || $t eq ')';
- }
- # Forward: is grep piping or redirecting output?
- # Unlike the backward scan, we do not skip newlines here:
- # a bare newline is a command boundary, and redirects or
- # pipes must appear on the same line as grep (or after a
- # line continuation, which the Lexer consumes).
- for (my $j = $pos + 1; $j < @$tokens; $j++) {
- my $t = $tokens->[$j]->[0];
- return 0 if $cmd_start{$t};
- return 1 if $filter_op{$t};
- }
- return 0;
-}
-
-# Map a body-relative line number to a file line number.
-# For double-quoted bodies, backslash-continuation lines
-# (\<newline>) are consumed by the Lexer without appearing
-# in the body text, so the inner parser sees fewer lines
-# than the source file has. We walk the source lines to
-# count continuations and adjust accordingly.
-sub body_to_file_line {
- my ($body_lineno, $body_token, $raw_lines, $body_start) = @_;
- my $body_text = $body_token->[0];
- my $body_end_line = $body_token->[4];
- unless ($body_start && $body_start >= 1) {
- warn "body_start is not a positive integer\n";
- return $body_lineno;
- }
- my $file_lineno = $body_lineno + $body_start - 1;
- # Only double-quoted bodies have line splices.
- return $file_lineno unless $body_text =~ /^"/;
- my $adj = 0;
- my $lines_seen = 0;
- unless ($body_end_line && $body_end_line >= $body_start) {
- warn "body_end_line is not set for double-quoted body\n";
- return $file_lineno;
- }
- my $end = $body_end_line;
- if ($end > @$raw_lines) {
- warn "body_end_line ($end) exceeds file length (" .
- scalar(@$raw_lines) . ")\n";
- return $file_lineno;
- }
- my $src_ln = $body_start;
- while ($src_ln <= $end && $lines_seen < $body_lineno) {
- my $line = $raw_lines->[$src_ln - 1];
- # Odd trailing backslashes = continuation (\<nl>).
- # Even = escaped backslashes (\\), not a continuation.
- if ($line =~ /(\\*)$/ && length($1) % 2 == 1) {
- $adj++;
- } else {
- $lines_seen++;
- }
- $src_ln++;
- }
- if ($lines_seen < $body_lineno) {
- warn "body_lineno ($body_lineno) not found within body range " .
- "($body_start..$end)\n";
- }
- return $file_lineno + $adj;
-}
-
-# ScriptParser calls this for each test body found in the script.
-sub check_test {
- my $self = shift @_;
- my $title = ScriptParser::unwrap(shift @_);
- my $body_token = shift @_;
- my $body_start = $body_token->[3];
- my $body = ScriptParser::unwrap($body_token);
- # Handle heredoc-style test bodies:
- # test_expect_success 'title' - <<\EOF
- # grep pattern file
- # EOF
- # The '-' signals that the body follows as a heredoc.
- if ($body eq '-') {
- my $herebody = shift @_;
- if ($herebody) {
- $body = $herebody->{content};
- $body_start = $herebody->{start_line};
- }
- }
- return unless $body;
- # We can never match a bare "grep" if there isn't one in the first
- # place. So we can skip all of the parsing effort. Technically you
- # can have:
- #
- # gr\
- # ep
- #
- # but that is sufficiently insane that I think we can ignore it.
- return unless $body =~ /grep/;
-
- my $raw_lines = $self->{raw_lines};
-
- # The outer parser gives us the body as an opaque string.
- # Parse it to get individual tokens with command boundaries.
- my $parser = ShellParser->new(\$body);
- my @tokens = $parser->parse();
-
- my $file = $self->{file};
-
- for (my $i = 0; $i < @tokens; $i++) {
- my $text = $tokens[$i]->[0];
- next unless is_command_word(\@tokens, $i);
-
- my $token_lineno = $tokens[$i]->[3];
- unless (defined($token_lineno) && $token_lineno >= 1) {
- warn "token has no line number\n";
- next;
- }
- my $file_lineno = body_to_file_line(
- $token_lineno,
- $body_token, $raw_lines, $body_start);
-
- # '!' negates the exit code without consuming command
- # position. '! test_grep' is an anti-pattern because
- # test_grep only prints diagnostics on grep failure,
- # and '!' inverts after that decision is already made.
- if ($text eq '!') {
- if ($i + 1 < @tokens &&
- $tokens[$i + 1]->[0] eq 'test_grep' &&
- !lint_ok($raw_lines, $file_lineno)) {
- print "$file:$file_lineno: error: ",
- 'use "test_grep !" instead of ',
- '"! test_grep"', "\n";
- $exit_code = 1;
- }
- next;
- }
-
- # Bare grep as a command (not a filter) is a test
- # assertion that should use test_grep for better
- # failure diagnostics.
- if ($text eq 'grep' &&
- !is_filter(\@tokens, $i) &&
- !lint_ok($raw_lines, $file_lineno)) {
- print "$file:$file_lineno: error: ",
- "bare grep outside pipeline ",
- "(use test_grep)\n";
- $exit_code = 1;
- }
- }
-}
-
package main;
for my $file (@ARGV) {
@@ -263,5 +22,7 @@ package main;
$parser->{file} = $file;
$parser->{raw_lines} = \@raw_lines;
$parser->parse();
+ print "$_\n" for @{$parser->{output}};
+ $exit_code ||= $parser->{nerrs} > 0;
}
exit $exit_code;
diff --git a/t/greplint.pl b/t/lib-greplint.pl
similarity index 87%
copy from t/greplint.pl
copy to t/lib-greplint.pl
index 68b6eba78d..16b85b8d3b 100644
--- a/t/greplint.pl
+++ b/t/lib-greplint.pl
@@ -1,5 +1,3 @@
-#!/usr/bin/env perl
-
# Detect bare 'grep' used as a test assertion where 'test_grep'
# should be used, and '! test_grep' where 'test_grep !' should
# be used.
@@ -19,16 +17,13 @@
use warnings;
use strict;
-use File::Basename;
-do(dirname($0) . "/lib-shell-parser.pl")
- or die "$0: failed to load lib-shell-parser.pl: $@$!\n";
-
-my $exit_code = 0;
# GrepLintParser inherits ScriptParser's ability to find
# test_expect_success/failure blocks and call check_test()
# on each body. We override check_test() to walk the token
-# stream looking for bare grep assertions.
+# stream looking for bare grep assertions vs. test_grep.
+#
+# Requires lib-shell-parser.pl to be loaded.
package GrepLintParser;
our @ISA = ('ScriptParser');
@@ -207,13 +202,25 @@ sub check_test {
my $parser = ShellParser->new(\$body);
my @tokens = $parser->parse();
- my $file = $self->{file};
+ my @errors = check_tokens(
+ $self->{file}, $raw_lines,
+ $body_token, $body_start, \@tokens);
- for (my $i = 0; $i < @tokens; $i++) {
- my $text = $tokens[$i]->[0];
- next unless is_command_word(\@tokens, $i);
+ push(@{$self->{output}}, @errors);
+ $self->{nerrs} += @errors;
+}
- my $token_lineno = $tokens[$i]->[3];
+# check_tokens() walks a parsed test body and returns a list of
+# error messages.
+sub check_tokens {
+ my ($file, $raw_lines, $body_token, $body_start, $tokens) = @_;
+ my @errors;
+
+ for (my $i = 0; $i < @$tokens; $i++) {
+ my $text = $tokens->[$i]->[0];
+ next unless is_command_word($tokens, $i);
+
+ my $token_lineno = $tokens->[$i]->[3];
unless (defined($token_lineno) && $token_lineno >= 1) {
warn "token has no line number\n";
next;
@@ -227,13 +234,12 @@ sub check_test {
# test_grep only prints diagnostics on grep failure,
# and '!' inverts after that decision is already made.
if ($text eq '!') {
- if ($i + 1 < @tokens &&
- $tokens[$i + 1]->[0] eq 'test_grep' &&
+ if ($i + 1 < @$tokens &&
+ $tokens->[$i + 1]->[0] eq 'test_grep' &&
!lint_ok($raw_lines, $file_lineno)) {
- print "$file:$file_lineno: error: ",
- 'use "test_grep !" instead of ',
- '"! test_grep"', "\n";
- $exit_code = 1;
+ push(@errors, "$file:$file_lineno: error: " .
+ 'use "test_grep !" instead of ' .
+ '"! test_grep"');
}
next;
}
@@ -242,26 +248,15 @@ sub check_test {
# assertion that should use test_grep for better
# failure diagnostics.
if ($text eq 'grep' &&
- !is_filter(\@tokens, $i) &&
+ !is_filter($tokens, $i) &&
!lint_ok($raw_lines, $file_lineno)) {
- print "$file:$file_lineno: error: ",
- "bare grep outside pipeline ",
- "(use test_grep)\n";
- $exit_code = 1;
+ push(@errors, "$file:$file_lineno: error: " .
+ "bare grep outside pipeline " .
+ "(use test_grep)");
}
}
+
+ return @errors;
}
-package main;
-
-for my $file (@ARGV) {
- open(my $fh, '<:unix:crlf', $file) or die "$0: $file: $!\n";
- my @raw_lines = <$fh>;
- close $fh;
- my $s = join('', @raw_lines);
- my $parser = GrepLintParser->new(\$s);
- $parser->{file} = $file;
- $parser->{raw_lines} = \@raw_lines;
- $parser->parse();
-}
-exit $exit_code;
+1;
--
2.47.3