diff options
22 files changed, 206 insertions, 66 deletions
diff --git a/t/chainlint.pl b/t/chainlint.pl index 976db4b8a0..7972c5bbe6 100755 --- a/t/chainlint.pl +++ b/t/chainlint.pl @@ -75,7 +75,9 @@ sub scan_heredoc_tag { my $self = shift @_; ${$self->{buff}} =~ /\G(-?)/gc; my $indented = $1; - my $tag = $self->scan_token(); + my $token = $self->scan_token(); + return "<<$indented" unless $token; + my $tag = $token->[0]; $tag =~ s/['"\\]//g; push(@{$self->{heretags}}, $indented ? "\t$tag" : "$tag"); return "<<$indented$tag"; @@ -149,7 +151,7 @@ sub scan_dollar { my $self = shift @_; my $b = $self->{buff}; return $self->scan_balanced('(', ')') if $$b =~ /\G\((?=\()/gc; # $((...)) - return '(' . join(' ', $self->scan_subst()) . ')' if $$b =~ /\G\(/gc; # $(...) + return '(' . join(' ', map {$_->[0]} $self->scan_subst()) . ')' if $$b =~ /\G\(/gc; # $(...) return $self->scan_balanced('{', '}') if $$b =~ /\G\{/gc; # ${...} return $1 if $$b =~ /\G(\w+)/gc; # $var return $1 if $$b =~ /\G([@*#?$!0-9-])/gc; # $*, $1, $$, etc. @@ -170,16 +172,18 @@ sub scan_token { my $self = shift @_; my $b = $self->{buff}; my $token = ''; + my $start; RESTART: $$b =~ /\G[ \t]+/gc; # skip whitespace (but not newline) - return "\n" if $$b =~ /\G#[^\n]*(?:\n|\z)/gc; # comment + $start = pos($$b) || 0; + return ["\n", $start, pos($$b)] if $$b =~ /\G#[^\n]*(?:\n|\z)/gc; # comment while (1) { # slurp up non-special characters $token .= $1 if $$b =~ /\G([^\\;&|<>(){}'"\$\s]+)/gc; # handle special characters last unless $$b =~ /\G(.)/sgc; my $c = $1; - last if $c =~ /^[ \t]$/; # whitespace ends token + pos($$b)--, last if $c =~ /^[ \t]$/; # whitespace ends token pos($$b)--, last if length($token) && $c =~ /^[;&|<>(){}\n]$/; $token .= $self->scan_sqstring(), next if $c eq "'"; $token .= $self->scan_dqstring(), next if $c eq '"'; @@ -197,7 +201,7 @@ RESTART: } die("internal error scanning character '$c'\n"); } - return length($token) ? $token : undef; + return length($token) ? [$token, $start, pos($$b)] : undef; } # ShellParser parses POSIX shell scripts (with minor extensions for Bash). It @@ -239,14 +243,14 @@ sub stop_at { my ($self, $token) = @_; return 1 unless defined($token); my $stop = ${$self->{stop}}[-1] if @{$self->{stop}}; - return defined($stop) && $token =~ $stop; + return defined($stop) && $token->[0] =~ $stop; } sub expect { my ($self, $expect) = @_; my $token = $self->next_token(); - return $token if defined($token) && $token eq $expect; - push(@{$self->{output}}, "?!ERR?! expected '$expect' but found '" . (defined($token) ? $token : "<end-of-input>") . "'\n"); + return $token if defined($token) && $token->[0] eq $expect; + push(@{$self->{output}}, "?!ERR?! expected '$expect' but found '" . (defined($token) ? $token->[0] : "<end-of-input>") . "'\n"); $self->untoken($token) if defined($token); return (); } @@ -255,7 +259,7 @@ sub optional_newlines { my $self = shift @_; my @tokens; while (my $token = $self->peek()) { - last unless $token eq "\n"; + last unless $token->[0] eq "\n"; push(@tokens, $self->next_token()); } return @tokens; @@ -278,7 +282,7 @@ sub parse_case_pattern { my @tokens; while (defined(my $token = $self->next_token())) { push(@tokens, $token); - last if $token eq ')'; + last if $token->[0] eq ')'; } return @tokens; } @@ -293,13 +297,13 @@ sub parse_case { $self->optional_newlines()); while (1) { my $token = $self->peek(); - last unless defined($token) && $token ne 'esac'; + last unless defined($token) && $token->[0] ne 'esac'; push(@tokens, $self->parse_case_pattern(), $self->optional_newlines(), $self->parse(qr/^(?:;;|esac)$/)); # item body $token = $self->peek(); - last unless defined($token) && $token ne 'esac'; + last unless defined($token) && $token->[0] ne 'esac'; push(@tokens, $self->expect(';;'), $self->optional_newlines()); @@ -315,7 +319,7 @@ sub parse_for { $self->next_token(), # variable $self->optional_newlines()); my $token = $self->peek(); - if (defined($token) && $token eq 'in') { + if (defined($token) && $token->[0] eq 'in') { push(@tokens, $self->expect('in'), $self->optional_newlines()); @@ -339,11 +343,11 @@ sub parse_if { $self->optional_newlines(), $self->parse(qr/^(?:elif|else|fi)$/)); # if/elif body my $token = $self->peek(); - last unless defined($token) && $token eq 'elif'; + last unless defined($token) && $token->[0] eq 'elif'; push(@tokens, $self->expect('elif')); } my $token = $self->peek(); - if (defined($token) && $token eq 'else') { + if (defined($token) && $token->[0] eq 'else') { push(@tokens, $self->expect('else'), $self->optional_newlines(), @@ -380,7 +384,7 @@ sub parse_bash_array_assignment { my @tokens = $self->expect('('); while (defined(my $token = $self->next_token())) { push(@tokens, $token); - last if $token eq ')'; + last if $token->[0] eq ')'; } return @tokens; } @@ -398,29 +402,31 @@ sub parse_cmd { my $self = shift @_; my $cmd = $self->next_token(); return () unless defined($cmd); - return $cmd if $cmd eq "\n"; + return $cmd if $cmd->[0] eq "\n"; my $token; my @tokens = $cmd; - if ($cmd eq '!') { + if ($cmd->[0] eq '!') { push(@tokens, $self->parse_cmd()); return @tokens; - } elsif (my $f = $compound{$cmd}) { + } elsif (my $f = $compound{$cmd->[0]}) { push(@tokens, $self->$f()); - } elsif (defined($token = $self->peek()) && $token eq '(') { - if ($cmd !~ /\w=$/) { + } elsif (defined($token = $self->peek()) && $token->[0] eq '(') { + if ($cmd->[0] !~ /\w=$/) { push(@tokens, $self->parse_func()); return @tokens; } - $tokens[-1] .= join(' ', $self->parse_bash_array_assignment()); + my @array = $self->parse_bash_array_assignment(); + $tokens[-1]->[0] .= join(' ', map {$_->[0]} @array); + $tokens[-1]->[2] = $array[$#array][2] if @array; } while (defined(my $token = $self->next_token())) { $self->untoken($token), last if $self->stop_at($token); push(@tokens, $token); - last if $token =~ /^(?:[;&\n|]|&&|\|\|)$/; + last if $token->[0] =~ /^(?:[;&\n|]|&&|\|\|)$/; } - push(@tokens, $self->next_token()) if $tokens[-1] ne "\n" && defined($token = $self->peek()) && $token eq "\n"; + push(@tokens, $self->next_token()) if $tokens[-1]->[0] ne "\n" && defined($token = $self->peek()) && $token->[0] eq "\n"; return @tokens; } @@ -453,11 +459,18 @@ package TestParser; use base 'ShellParser'; +sub new { + my $class = shift @_; + my $self = $class->SUPER::new(@_); + $self->{problems} = []; + return $self; +} + sub find_non_nl { my $tokens = shift @_; my $n = shift @_; $n = $#$tokens if !defined($n); - $n-- while $n >= 0 && $$tokens[$n] eq "\n"; + $n-- while $n >= 0 && $$tokens[$n]->[0] eq "\n"; return $n; } @@ -467,7 +480,7 @@ sub ends_with { for my $needle (reverse(@$needles)) { return undef if $n < 0; $n = find_non_nl($tokens, $n), next if $needle eq "\n"; - return undef if $$tokens[$n] !~ $needle; + return undef if $$tokens[$n]->[0] !~ $needle; $n--; } return 1; @@ -486,13 +499,13 @@ sub parse_loop_body { my $self = shift @_; my @tokens = $self->SUPER::parse_loop_body(@_); # did loop signal failure via "|| return" or "|| exit"? - return @tokens if !@tokens || grep(/^(?:return|exit|\$\?)$/, @tokens); + return @tokens if !@tokens || grep {$_->[0] =~ /^(?:return|exit|\$\?)$/} @tokens; # did loop upstream of a pipe signal failure via "|| echo 'impossible # text'" as the final command in the loop body? return @tokens if ends_with(\@tokens, [qr/^\|\|$/, "\n", qr/^echo$/, qr/^.+$/]); # flag missing "return/exit" handling explicit failure in loop body my $n = find_non_nl(\@tokens); - splice(@tokens, $n + 1, 0, '?!LOOP?!'); + push(@{$self->{problems}}, ['LOOP', $tokens[$n]]); return @tokens; } @@ -505,8 +518,13 @@ my @safe_endings = ( sub accumulate { my ($self, $tokens, $cmd) = @_; + my $problems = $self->{problems}; + + # no previous command to check for missing "&&" goto DONE unless @$tokens; - goto DONE if @$cmd == 1 && $$cmd[0] eq "\n"; + + # new command is empty line; can't yet check if previous is missing "&&" + goto DONE if @$cmd == 1 && $$cmd[0]->[0] eq "\n"; # did previous command end with "&&", "|", "|| return" or similar? goto DONE if match_ending($tokens, \@safe_endings); @@ -514,20 +532,20 @@ sub accumulate { # if this command handles "$?" specially, then okay for previous # command to be missing "&&" for my $token (@$cmd) { - goto DONE if $token =~ /\$\?/; + goto DONE if $token->[0] =~ /\$\?/; } # if this command is "false", "return 1", or "exit 1" (which signal # failure explicitly), then okay for all preceding commands to be # missing "&&" - if ($$cmd[0] =~ /^(?:false|return|exit)$/) { - @$tokens = grep(!/^\?!AMP\?!$/, @$tokens); + if ($$cmd[0]->[0] =~ /^(?:false|return|exit)$/) { + @$problems = grep {$_->[0] ne 'AMP'} @$problems; goto DONE; } # flag missing "&&" at end of previous command my $n = find_non_nl($tokens); - splice(@$tokens, $n + 1, 0, '?!AMP?!') unless $n < 0; + push(@$problems, ['AMP', $tokens->[$n]]) unless $n < 0; DONE: $self->SUPER::accumulate($tokens, $cmd); @@ -553,7 +571,7 @@ sub new { # composition of multiple strings and non-string character runs; for instance, # `"test body"` unwraps to `test body`; `word"a b"42'c d'` to `worda b42c d` sub unwrap { - my $token = @_ ? shift @_ : $_; + my $token = (@_ ? shift @_ : $_)->[0]; # simple case: 'sqstring' or "dqstring" return $token if $token =~ s/^'([^']*)'$/$1/; return $token if $token =~ s/^"([^"]*)"$/$1/; @@ -584,12 +602,21 @@ sub check_test { $self->{ntests}++; my $parser = TestParser->new(\$body); my @tokens = $parser->parse(); - return unless $emit_all || grep(/\?![^?]+\?!/, @tokens); + my $problems = $parser->{problems}; + return unless $emit_all || @$problems; my $c = main::fd_colors(1); - my $checked = join(' ', @tokens); + my $start = 0; + my $checked = ''; + for (sort {$a->[1]->[2] <=> $b->[1]->[2]} @$problems) { + my ($label, $token) = @$_; + my $pos = $token->[2]; + $checked .= substr($body, $start, $pos - $start) . " ?!$label?! "; + $start = $pos; + } + $checked .= substr($body, $start); $checked =~ s/^\n//; - $checked =~ s/^ //mg; - $checked =~ s/ $//mg; + $checked =~ s/(\s) \?!/$1?!/mg; + $checked =~ s/\?! (\s)/?!$1/mg; $checked =~ s/(\?![^?]+\?!)/$c->{rev}$c->{red}$1$c->{reset}/mg; $checked .= "\n" unless $checked =~ /\n$/; push(@{$self->{output}}, "$c->{blue}# chainlint: $title$c->{reset}\n$checked"); @@ -598,9 +625,9 @@ sub check_test { sub parse_cmd { my $self = shift @_; my @tokens = $self->SUPER::parse_cmd(); - return @tokens unless @tokens && $tokens[0] =~ /^test_expect_(?:success|failure)$/; + return @tokens unless @tokens && $tokens[0]->[0] =~ /^test_expect_(?:success|failure)$/; my $n = $#tokens; - $n-- while $n >= 0 && $tokens[$n] =~ /^(?:[;&\n|]|&&|\|\|)$/; + $n-- while $n >= 0 && $tokens[$n]->[0] =~ /^(?:[;&\n|]|&&|\|\|)$/; $self->check_test($tokens[1], $tokens[2]) if $n == 2; # title body $self->check_test($tokens[2], $tokens[3]) if $n > 2; # prereq title body return @tokens; diff --git a/t/chainlint/block-comment.expect b/t/chainlint/block-comment.expect index d10b2eeaf2..df2beea888 100644 --- a/t/chainlint/block-comment.expect +++ b/t/chainlint/block-comment.expect @@ -1,6 +1,8 @@ ( { + # show a echo a && + # show b echo b } ) diff --git a/t/chainlint/case-comment.expect b/t/chainlint/case-comment.expect index 1e4b054bda..641c157b98 100644 --- a/t/chainlint/case-comment.expect +++ b/t/chainlint/case-comment.expect @@ -1,7 +1,10 @@ ( case "$x" in + # found foo x) foo ;; + # found other *) + # treat it as bar bar ;; esac diff --git a/t/chainlint/close-subshell.expect b/t/chainlint/close-subshell.expect index 0f87db9ae6..2192a2870a 100644 --- a/t/chainlint/close-subshell.expect +++ b/t/chainlint/close-subshell.expect @@ -15,7 +15,8 @@ ) | wuzzle && ( bop -) | fazz fozz && +) | fazz \ + fozz && ( bup ) | diff --git a/t/chainlint/comment.expect b/t/chainlint/comment.expect index f76fde1ffb..a68f1f9d7c 100644 --- a/t/chainlint/comment.expect +++ b/t/chainlint/comment.expect @@ -1,4 +1,8 @@ ( + # comment 1 nothing && + # comment 2 something + # comment 3 + # comment 4 ) diff --git a/t/chainlint/double-here-doc.expect b/t/chainlint/double-here-doc.expect index 75477bb1ad..cd584a4357 100644 --- a/t/chainlint/double-here-doc.expect +++ b/t/chainlint/double-here-doc.expect @@ -1,2 +1,12 @@ -run_sub_test_lib_test_err run-inv-range-start "--run invalid range start" --run="a-5" <<-EOF && -check_sub_test_lib_test_err run-inv-range-start <<-EOF_OUT 3 <<-EOF_ERR +run_sub_test_lib_test_err run-inv-range-start \ + "--run invalid range start" \ + --run="a-5" <<-\EOF && +test_expect_success "passing test #1" "true" +test_done +EOF +check_sub_test_lib_test_err run-inv-range-start \ + <<-\EOF_OUT 3<<-EOF_ERR +> FATAL: Unexpected exit with code 1 +EOF_OUT +> error: --run: invalid non-numeric in range start: ${SQ}a-5${SQ} +EOF_ERR diff --git a/t/chainlint/empty-here-doc.expect b/t/chainlint/empty-here-doc.expect index f42f2d41ba..e8733c97c6 100644 --- a/t/chainlint/empty-here-doc.expect +++ b/t/chainlint/empty-here-doc.expect @@ -1,3 +1,4 @@ git ls-tree $tree path > current && -cat > expected <<EOF && +cat > expected <<\EOF && +EOF test_output diff --git a/t/chainlint/for-loop.expect b/t/chainlint/for-loop.expect index a5810c9bdd..d65c82129a 100644 --- a/t/chainlint/for-loop.expect +++ b/t/chainlint/for-loop.expect @@ -2,7 +2,9 @@ for i in a b c do echo $i ?!AMP?! - cat <<-EOF ?!LOOP?! + cat <<-\EOF ?!LOOP?! + bar + EOF done ?!AMP?! for i in a b c; do echo $i && diff --git a/t/chainlint/here-doc-close-subshell.expect b/t/chainlint/here-doc-close-subshell.expect index 2af9ced71c..7d9c2b5607 100644 --- a/t/chainlint/here-doc-close-subshell.expect +++ b/t/chainlint/here-doc-close-subshell.expect @@ -1,2 +1,4 @@ ( - cat <<-INPUT) + cat <<-\INPUT) + fizz + INPUT diff --git a/t/chainlint/here-doc-indent-operator.expect b/t/chainlint/here-doc-indent-operator.expect index fb6cf7285d..f92a7ce999 100644 --- a/t/chainlint/here-doc-indent-operator.expect +++ b/t/chainlint/here-doc-indent-operator.expect @@ -1,5 +1,11 @@ -cat > expect <<-EOF && +cat >expect <<- EOF && +header: 43475048 1 $(test_oid oid_version) $NUM_CHUNKS 0 +num_commits: $1 +chunks: oid_fanout oid_lookup commit_metadata generation_data bloom_indexes bloom_data +EOF -cat > expect <<-EOF ?!AMP?! +cat >expect << -EOF ?!AMP?! +this is not indented +-EOF cleanup diff --git a/t/chainlint/here-doc-multi-line-command-subst.expect b/t/chainlint/here-doc-multi-line-command-subst.expect index f8b3aa73c4..b7364c82c8 100644 --- a/t/chainlint/here-doc-multi-line-command-subst.expect +++ b/t/chainlint/here-doc-multi-line-command-subst.expect @@ -1,5 +1,8 @@ ( - x=$(bobble <<-END && + x=$(bobble <<-\END && + fossil + vegetable + END wiffle) ?!AMP?! echo $x ) diff --git a/t/chainlint/here-doc-multi-line-string.expect b/t/chainlint/here-doc-multi-line-string.expect index be64b26869..6c13bdcbfb 100644 --- a/t/chainlint/here-doc-multi-line-string.expect +++ b/t/chainlint/here-doc-multi-line-string.expect @@ -1,5 +1,7 @@ ( - cat <<-TXT && echo "multi-line + cat <<-\TXT && echo "multi-line string" ?!AMP?! + fizzle + TXT bap ) diff --git a/t/chainlint/here-doc.expect b/t/chainlint/here-doc.expect index 110059ba58..1df3f78282 100644 --- a/t/chainlint/here-doc.expect +++ b/t/chainlint/here-doc.expect @@ -1,7 +1,25 @@ -boodle wobba gorgo snoot wafta snurb <<EOF && +boodle wobba \ + gorgo snoot \ + wafta snurb <<EOF && +quoth the raven, +nevermore... +EOF cat <<-Arbitrary_Tag_42 >foo && +snoz +boz +woz +Arbitrary_Tag_42 -cat <<zump >boo && +cat <<"zump" >boo && +snoz +boz +woz +zump -horticulture <<EOF +horticulture <<\EOF +gomez +morticia +wednesday +pugsly +EOF diff --git a/t/chainlint/if-then-else.expect b/t/chainlint/if-then-else.expect index 44d86c3597..cbaaf857d4 100644 --- a/t/chainlint/if-then-else.expect +++ b/t/chainlint/if-then-else.expect @@ -8,7 +8,9 @@ echo foo else echo foo && - cat <<-EOF + cat <<-\EOF + bar + EOF fi ?!AMP?! echo poodle ) && diff --git a/t/chainlint/incomplete-line.expect b/t/chainlint/incomplete-line.expect index ffac8f9018..134d3a14f5 100644 --- a/t/chainlint/incomplete-line.expect +++ b/t/chainlint/incomplete-line.expect @@ -1,4 +1,10 @@ -line 1 line 2 line 3 line 4 && +line 1 \ +line 2 \ +line 3 \ +line 4 && ( - line 5 line 6 line 7 line 8 + line 5 \ + line 6 \ + line 7 \ + line 8 ) diff --git a/t/chainlint/inline-comment.expect b/t/chainlint/inline-comment.expect index dd0dace077..6bad218530 100644 --- a/t/chainlint/inline-comment.expect +++ b/t/chainlint/inline-comment.expect @@ -1,6 +1,6 @@ ( - foobar && - barfoo ?!AMP?! + foobar && # comment 1 + barfoo ?!AMP?! # wrong position for && flibble "not a # comment" ) && diff --git a/t/chainlint/loop-detect-status.expect b/t/chainlint/loop-detect-status.expect index 0ad23bb35e..24da9e86d5 100644 --- a/t/chainlint/loop-detect-status.expect +++ b/t/chainlint/loop-detect-status.expect @@ -2,7 +2,7 @@ do printf "Generating blob $i/$blobcount\r" >& 2 && printf "blob\nmark :$i\ndata $blobsize\n" && - + #test-tool genrandom $i $blobsize && printf "%-${blobsize}s" $i && echo "M 100644 :$i $i" >> commit && i=$(($i+1)) || diff --git a/t/chainlint/nested-here-doc.expect b/t/chainlint/nested-here-doc.expect index e3bef63f75..29b3832a98 100644 --- a/t/chainlint/nested-here-doc.expect +++ b/t/chainlint/nested-here-doc.expect @@ -1,7 +1,30 @@ cat <<ARBITRARY >foop && +naddle +fub <<EOF + nozzle + noodle +EOF +formp +ARBITRARY ( - cat <<-INPUT_END && - cat <<-EOT ?!AMP?! + cat <<-\INPUT_END && + fish are mice + but geese go slow + data <<EOF + perl is lerp + and nothing else + EOF + toink + INPUT_END + + cat <<-\EOT ?!AMP?! + text goes here + data <<EOF + data goes here + EOF + more test here + EOT + foobar ) diff --git a/t/chainlint/nested-subshell-comment.expect b/t/chainlint/nested-subshell-comment.expect index be4b27a305..9138cf386d 100644 --- a/t/chainlint/nested-subshell-comment.expect +++ b/t/chainlint/nested-subshell-comment.expect @@ -2,6 +2,8 @@ foo && ( bar && + # bottles wobble while fiddles gobble + # minor numbers of cows (or do they?) baz && snaff ) ?!AMP?! diff --git a/t/chainlint/subshell-here-doc.expect b/t/chainlint/subshell-here-doc.expect index 029d129299..52789278d1 100644 --- a/t/chainlint/subshell-here-doc.expect +++ b/t/chainlint/subshell-here-doc.expect @@ -1,10 +1,30 @@ ( - echo wobba gorgo snoot wafta snurb <<-EOF && + echo wobba \ + gorgo snoot \ + wafta snurb <<-EOF && + quoth the raven, + nevermore... + EOF + cat <<EOF >bip ?!AMP?! - echo <<-EOF >bop + fish fly high +EOF + + echo <<-\EOF >bop + gomez + morticia + wednesday + pugsly + EOF ) && ( - cat <<-ARBITRARY >bup && - cat <<-ARBITRARY3 >bup3 && + cat <<-\ARBITRARY >bup && + glink + FIZZ + ARBITRARY + cat <<-"ARBITRARY3" >bup3 && + glink + FIZZ + ARBITRARY3 meep ) diff --git a/t/chainlint/t7900-subtree.expect b/t/chainlint/t7900-subtree.expect index 69167da2f2..71b3b3bc20 100644 --- a/t/chainlint/t7900-subtree.expect +++ b/t/chainlint/t7900-subtree.expect @@ -4,12 +4,16 @@ sub2 sub3 sub4" && chks_sub=$(cat <<TXT | sed "s,^,sub dir/," +$chks +TXT ) && chkms="main-sub1 main-sub2 main-sub3 main-sub4" && chkms_sub=$(cat <<TXT | sed "s,^,sub dir/," +$chkms +TXT ) && subfiles=$(git ls-files) && check_equal "$subfiles" "$chkms diff --git a/t/chainlint/while-loop.expect b/t/chainlint/while-loop.expect index f272aa21fe..1f5eaea0fd 100644 --- a/t/chainlint/while-loop.expect +++ b/t/chainlint/while-loop.expect @@ -2,7 +2,9 @@ while true do echo foo ?!AMP?! - cat <<-EOF ?!LOOP?! + cat <<-\EOF ?!LOOP?! + bar + EOF done ?!AMP?! while true; do echo foo && |