Skip to content

Commit

Permalink
Implemented verbose mode which prints each test on separate line.
Browse files Browse the repository at this point in the history
This is handy for environments with non-standard (buffered) handling of
standard output, for example Github Actions (where a progress of tests
cannot be seen until until end-of-line appears, which in standard `console`
mode happens only when all tests finish) or Docker Compose logging output,
where in standard `console` mode each finished test's dot is printed
alone on separate line.

Or the "verbose" mode can be handy just to see a more detailed
progress of tests in all environments, because it outputs something
like this:

```
路 1/85 Framework/Assert.contains.phpt OK in 0.14 s
路 2/85 CodeCoverage/PhpParser.parse.edge.phpt OK in 0.17 s
路 3/85 CodeCoverage/PhpParser.parse.lines-of-code.phpt SKIPPED in 0.18 s
路 4/85 CodeCoverage/PhpParser.parse.lines.phpt FAILED in 0.19 s
...
```

Also, "cider mode" now shows a lemon emoji for skipped tests.
  • Loading branch information
smuuf committed Aug 12, 2023
1 parent 925b93b commit 415bb2b
Show file tree
Hide file tree
Showing 7 changed files with 226 additions and 37 deletions.
3 changes: 2 additions & 1 deletion phpstan.neon
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
parameters:
level: 5

paths:
- src
typeAliases:
Alias_TestResultState: 'Tester\Runner\Test::Passed|Tester\Runner\Test::Skipped|Tester\Runner\Test::Failed|Tester\Runner\Test::Prepared'
3 changes: 2 additions & 1 deletion readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,8 @@ Options:
-s Show information about skipped tests.
--stop-on-fail Stop execution upon the first failure.
-j <num> Run <num> jobs in parallel (default: 8).
-o <console|tap|junit|none> Specify output format.
-o <console|verbose|tap|junit|none>
Specify output format.
-w | --watch <path> Watch directory.
-i | --info Show tests environment info and exit.
--setup <path> Script for runner setup.
Expand Down
33 changes: 24 additions & 9 deletions src/Runner/CliTester.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
use Tester\Dumper;
use Tester\Environment;
use Tester\Helpers;

use Tester\Runner\Output\ConsolePrinter;

/**
* CLI Tester.
Expand Down Expand Up @@ -112,7 +112,7 @@ private function loadOptions(): CommandLine
-s Show information about skipped tests.
--stop-on-fail Stop execution upon the first failure.
-j <num> Run <num> jobs in parallel (default: 8).
-o <console|tap|junit|log|none> (e.g. -o junit:output.xml)
-o <console|verbose|tap|junit|log|none> (e.g. -o junit:output.xml)
Specify one or more output formats with optional file name.
-w | --watch <path> Watch directory.
-i | --info Show tests environment info and exit.
Expand Down Expand Up @@ -219,18 +219,14 @@ private function createRunner(): Runner
$runner->setTempDirectory($this->options['--temp']);

if ($this->stdoutFormat === null) {
$runner->outputHandlers[] = new Output\ConsolePrinter(
$runner,
(bool) $this->options['-s'],
'php://output',
(bool) $this->options['--cider']
);
$runner->outputHandlers[] = $this->buildConsolePrinter($runner, 'php://output', false);
}

foreach ($this->options['-o'] as $output) {
[$format, $file] = $output;
match ($format) {
'console' => $runner->outputHandlers[] = new Output\ConsolePrinter($runner, (bool) $this->options['-s'], $file, (bool) $this->options['--cider']),
'console' => $runner->outputHandlers[] = $this->buildConsolePrinter($runner, $file, false),
'verbose' => $runner->outputHandlers[] = $this->buildConsolePrinter($runner, $file, true),
'tap' => $runner->outputHandlers[] = new Output\TapPrinter($file),
'junit' => $runner->outputHandlers[] = new Output\JUnitPrinter($file),
'log' => $runner->outputHandlers[] = new Output\Logger($runner, $file),
Expand All @@ -248,6 +244,25 @@ private function createRunner(): Runner
return $runner;
}

/**
* Builds and returns a new `ConsolePrinter`.
* @param bool $lineMode If `true`, reports each finished test on separate line.
*/
private function buildConsolePrinter(
Runner $runner,
?string $file,
bool $lineMode,
): ConsolePrinter
{
return new Output\ConsolePrinter(
$runner,
(bool) $this->options['-s'],
$file,
(bool) $this->options['--cider'],
$lineMode,
);
}


private function prepareCodeCoverage(Runner $runner): string
{
Expand Down
129 changes: 105 additions & 24 deletions src/Runner/Output/ConsolePrinter.php
Original file line number Diff line number Diff line change
Expand Up @@ -20,46 +20,66 @@
*/
class ConsolePrinter implements Tester\Runner\OutputHandler
{
private Runner $runner;

/** @var resource */
private $file;
private bool $displaySkipped = false;
private string $buffer;

/** @var list<string> */
private array $buffer;

/**
* @phpstan-var array<Alias_TestResultState, string>
* @var array<int, string>
*/
private array $symbols;

/**
* @phpstan-var array<Alias_TestResultState, int>
* @var array<int, string>
*/
private array $results = [
Test::Passed => 0,
Test::Skipped => 0,
Test::Failed => 0,
];

private float $time;
private int $count;
private array $results;
private ?string $baseDir;
private array $symbols;

private int $resultsCount = 0;

/**
* @param bool $lineMode If `true`, reports each finished test on separate line.
*/
public function __construct(
Runner $runner,
bool $displaySkipped = false,
private Runner $runner,
private bool $displaySkipped = false,
?string $file = null,
bool $ciderMode = false
bool $ciderMode = false,
private bool $lineMode = false,
) {
$this->runner = $runner;
$this->displaySkipped = $displaySkipped;
$this->file = fopen($file ?: 'php://output', 'w');

$this->symbols = [
Test::Passed => $ciderMode ? Dumper::color('green', '馃崕') : '.',
Test::Skipped => 's',
Test::Failed => $ciderMode ? Dumper::color('red', '馃崕') : Dumper::color('white/red', 'F'),
Test::Passed => $this->lineMode ? Dumper::color('lime', 'OK') : '.',
Test::Skipped => $this->lineMode ? Dumper::color('yellow', 'SKIP') : 's',
Test::Failed => $this->lineMode ? Dumper::color('white/red', 'FAIL') : Dumper::color('white/red', 'F'),
];

if ($ciderMode) {
$this->symbols[Test::Passed] = '馃崗';
$this->symbols[Test::Skipped] = '馃崑';
$this->symbols[Test::Failed] = '馃崕';
}
}


public function begin(): void
{
$this->count = 0;
$this->buffer = '';
$this->buffer = [];
$this->baseDir = null;
$this->results = [
Test::Passed => 0,
Test::Skipped => 0,
Test::Failed => 0,
];
$this->time = -microtime(true);
fwrite($this->file, $this->runner->getInterpreter()->getShortInfo()
. ' | ' . $this->runner->getInterpreter()->getCommandLine()
Expand Down Expand Up @@ -94,15 +114,17 @@ public function prepare(Test $test): void
public function finish(Test $test): void
{
$this->results[$test->getResult()]++;
fwrite($this->file, $this->symbols[$test->getResult()]);
$this->lineMode
? $this->printFinishedTestLine($test)
: $this->printFinishedTestDot($test);

$title = ($test->title ? "$test->title | " : '') . substr($test->getSignature(), strlen($this->baseDir));
$message = ' ' . str_replace("\n", "\n ", trim((string) $test->message)) . "\n\n";
$message = preg_replace('/^ $/m', '', $message);
if ($test->getResult() === Test::Failed) {
$this->buffer .= Dumper::color('red', "-- FAILED: $title") . "\n$message";
$this->buffer[] = Dumper::color('red', "-- FAILED: $title") . "\n$message";
} elseif ($test->getResult() === Test::Skipped && $this->displaySkipped) {
$this->buffer .= "-- Skipped: $title\n$message";
$this->buffer[] = "-- Skipped: $title\n$message";
}
}

Expand All @@ -111,14 +133,73 @@ public function end(): void
{
$run = array_sum($this->results);
fwrite($this->file, !$this->count ? "No tests found\n" :
"\n\n" . $this->buffer . "\n"
"\n\n" . implode('', $this->buffer) . "\n"
. ($this->results[Test::Failed] ? Dumper::color('white/red') . 'FAILURES!' : Dumper::color('white/green') . 'OK')
. " ($this->count test" . ($this->count > 1 ? 's' : '') . ', '
. ($this->results[Test::Failed] ? $this->results[Test::Failed] . ' failure' . ($this->results[Test::Failed] > 1 ? 's' : '') . ', ' : '')
. ($this->results[Test::Skipped] ? $this->results[Test::Skipped] . ' skipped, ' : '')
. ($this->count !== $run ? ($this->count - $run) . ' not run, ' : '')
. sprintf('%0.1f', $this->time + microtime(true)) . ' seconds)' . Dumper::color() . "\n");

$this->buffer = '';
$this->buffer = [];
$this->resultsCount = 0;
}


private function printFinishedTestDot(Test $test): void
{
fwrite($this->file, $this->symbols[$test->getResult()]);
}


private function printFinishedTestLine(Test $test): void
{
$this->resultsCount++;
$result = $test->getResult();

$shortFilePath = str_replace($this->baseDir, '', $test->getFile());
$shortDirPath = dirname($shortFilePath) . DIRECTORY_SEPARATOR;
$basename = basename($shortFilePath);

// Filename.
if ($result === Test::Failed) {
$fileText = Dumper::color('red', $shortDirPath) . Dumper::color('white/red', $basename);
} else {
$fileText = Dumper::color('gray', $shortDirPath) . Dumper::color('silver', $basename);
}

// Line header.
$header = "";
// Test's title.
$titleText = $test->title
? Dumper::color('fuchsia', " [$test->title]")
: '';

// Print test's message only if it's not failed (those will be printed
// after all tests are finished and will contain detailed info about the
// failed test).
$message = '';
if ($result !== Test::Failed && $test->message) {
$message = $test->message;
$indent = str_repeat(' ', mb_strlen($header));

if (preg_match('#\n#', $message)) {
$message = "\n$indent" . preg_replace('#\r?\n#', '\0' . $indent, $message);
} else {
$message = Dumper::color('olive', "[$message]");
}
}

$output = sprintf(
"%s%s %s %s %s %s\n",
$header,
"{$this->resultsCount}/{$this->count}",
"$fileText{$titleText}",
$this->symbols[$result],
Dumper::color('gray', sprintf("in %.2f s", $test->getDuration())),
$message,
);

fwrite($this->file, $output);
}
}
19 changes: 18 additions & 1 deletion src/Runner/Test.php
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,23 @@ class Test
PASSED = self::Passed,
SKIPPED = self::Skipped;

private const PossibleResults = [
self::Prepared,
self::Failed,
self::Passed,
self::Skipped,
];

public ?string $title;
public ?string $message = null;
public string $stdout = '';
public string $stderr = '';
private string $file;
private int $result = self::Prepared;
private ?float $duration = null;

/** @phpstan-var Alias_TestResultState */
private int $result = self::Prepared;

/** @var string[]|string[][] */
private $args = [];

Expand Down Expand Up @@ -70,6 +79,9 @@ public function getSignature(): string
}


/**
* @phpstan-return Alias_TestResultState
*/
public function getResult(): int
{
return $this->result;
Expand Down Expand Up @@ -123,6 +135,7 @@ public function withArguments(array $args): self


/**
* @phpstan-param Alias_TestResultState $result
* @return static
*/
public function withResult(int $result, ?string $message, ?float $duration = null): self
Expand All @@ -131,6 +144,10 @@ public function withResult(int $result, ?string $message, ?float $duration = nul
throw new \LogicException("Result of test is already set to $this->result with message '$this->message'.");
}

if (!in_array($result, self::PossibleResults, true)) {
throw new \LogicException("Invalid test result $result");
}

$me = clone $this;
$me->result = $result;
$me->message = $message;
Expand Down
73 changes: 73 additions & 0 deletions tests/RunnerOutput/OutputHandlers.expect.verbose.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
%a% | %a% | 1 thread

路 1/%d% ./01-basic.fail.phptx FAIL in %f% s
路 2/%d% ./01-basic.pass.phptx OK in %f% s
路 3/%d% ./01-basic.skip.phptx SKIP in %f% s
路 4/%d% ./02-title.fail.phptx [Title for output handlers] FAIL in %f% s
路 5/%d% ./02-title.pass.phptx [Title for output handlers] OK in %f% s
路 6/%d% ./02-title.skip.phptx [Title for output handlers] SKIP in %f% s
路 7/%d% ./03-message.fail.phptx FAIL in %f% s
路 8/%d% ./03-message.skip.phptx SKIP in %f% s
Multi
line
message.
路 9/%d% ./04-args.fail.phptx FAIL in %f% s
路 10/%d% ./04-args.pass.phptx OK in %f% s
路 11/%d% ./04-args.skip.phptx SKIP in %f% s
Multi
line
message.


-- FAILED: 01-basic.fail.phptx
Multi
line
stdout.Failed:

in %a%01-basic.fail.phptx(%d%) Tester\Assert::fail('');

STDERR:
Multi
line
stderr.

-- FAILED: Title for output handlers | 02-title.fail.phptx
Multi
line
stdout.Failed:

in %a%02-title.fail.phptx(%d%) Tester\Assert::fail('');

STDERR:
Multi
line
stderr.

-- FAILED: 03-message.fail.phptx
Multi
line
stdout.Failed: Multi
line
message.

in %a%03-message.fail.phptx(%d%) Tester\Assert::fail("Multi\nline\nmessage.");

STDERR:
Multi
line
stderr.

-- FAILED: 04-args.fail.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
Multi
line
stdout.Failed:

in %a%04-args.fail.phptx(%d%) Tester\Assert::fail('');

STDERR:
Multi
line
stderr.


FAILURES! (11 tests, 4 failures, 4 skipped, %a% seconds)

0 comments on commit 415bb2b

Please sign in to comment.