Skip to content

Commit

Permalink
[SUREFIRE-1345] Support flakyFailure and flakyError in TestSuiteXmlPa…
Browse files Browse the repository at this point in the history 
…rser

This closes #700
  • Loading branch information
@gsmet @michael-o
gsmet authored and michael-o committed Dec 22, 2023
1 parent 6eb7d7e commit ef4f2bf
Show file tree
Hide file tree
Showing 4 changed files with 558 additions and 0 deletions.
88 changes: 88 additions & 0 deletions ...-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
*/
package org.apache.maven.plugins.surefire.report;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import static org.apache.maven.shared.utils.StringUtils.isNotBlank;

/**
Expand Down Expand Up @@ -48,6 +52,10 @@ public final class ReportTestCase {

private boolean hasSkipped;

private List<FlakyError> flakyErrors = new ArrayList<>();

private List<FlakyFailure> flakyFailures = new ArrayList<>();

public String getName() {
return name;
}
Expand Down Expand Up @@ -161,6 +169,26 @@ public boolean hasSkipped() {
return hasSkipped;
}

public boolean hasFlakes() {
return !flakyErrors.isEmpty() || !flakyFailures.isEmpty();
}

public void addFlakyError(FlakyError flakyError) {
flakyErrors.add(flakyError);
}

public List<FlakyError> getFlakyErrors() {
return Collections.unmodifiableList(flakyErrors);
}

public void addFlakyFailure(FlakyFailure flakyFailure) {
flakyFailures.add(flakyFailure);
}

public List<FlakyFailure> getFlakyFailures() {
return Collections.unmodifiableList(flakyFailures);
}

/**
* {@inheritDoc}
*/
Expand All @@ -173,4 +201,64 @@ private ReportTestCase setFailureType(String failureType) {
this.failureType = failureType;
return this;
}

public static class FlakyFailure {

private String message;

private String type;

private String stackTrace;

FlakyFailure(String message, String type) {
this.message = message;
this.type = type;
}

public String getMessage() {
return message;
}

public String getType() {
return type;
}

public void setStackTrace(String stackTrace) {
this.stackTrace = stackTrace;
}

public String getStackTrace() {
return stackTrace;
}
}

public static class FlakyError {

private String message;

private String type;

private String stackTrace;

FlakyError(String message, String type) {
this.message = message;
this.type = type;
}

public String getMessage() {
return message;
}

public String getType() {
return type;
}

public void setStackTrace(String stackTrace) {
this.stackTrace = stackTrace;
}

public String getStackTrace() {
return stackTrace;
}
}
}
32 changes: 32 additions & 0 deletions ...ort-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ public final class TestSuiteXmlParser extends DefaultHandler {

private ReportTestCase testCase;

private ReportTestCase.FlakyFailure testCaseFlakyFailure;

private ReportTestCase.FlakyError testCaseFlakyError;

private boolean valid;

private boolean parseContent;
Expand Down Expand Up @@ -131,6 +135,8 @@ public void startElement(String uri, String localName, String qName, Attributes
currentElement = new StringBuilder();

testCase = new ReportTestCase().setName(attributes.getValue("name"));
testCaseFlakyFailure = null;
testCaseFlakyError = null;

String fullClassName = attributes.getValue("classname");

Expand Down Expand Up @@ -177,9 +183,19 @@ public void startElement(String uri, String localName, String qName, Attributes
currentSuite.incrementNumberOfSkipped();
break;
case "flakyFailure":
testCaseFlakyFailure = new ReportTestCase.FlakyFailure(
attributes.getValue("message"), attributes.getValue("type"));
currentSuite.incrementNumberOfFlakes();
break;
case "flakyError":
testCaseFlakyError = new ReportTestCase.FlakyError(
attributes.getValue("message"), attributes.getValue("type"));
currentSuite.incrementNumberOfFlakes();
break;
case "stackTrace":
currentElement = new StringBuilder();
parseContent = true;
break;
case "failsafe-summary":
valid = false;
break;
Expand Down Expand Up @@ -217,6 +233,22 @@ public void endElement(String uri, String localName, String qName) throws SAXExc
throw new SAXException("Failed to parse time value", e);
}
break;
case "flakyFailure":
testCase.addFlakyFailure(testCaseFlakyFailure);
testCaseFlakyFailure = null;
break;
case "flakyError":
testCase.addFlakyError(testCaseFlakyError);
testCaseFlakyError = null;
break;
case "stackTrace":
if (testCaseFlakyFailure != null) {
testCaseFlakyFailure.setStackTrace(currentElement.toString());
}
if (testCaseFlakyError != null) {
testCaseFlakyError.setStackTrace(currentElement.toString());
}
break;
default:
break;
}
Expand Down
75 changes: 75 additions & 0 deletions ...parser/src/test/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,18 @@
import java.util.List;

import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
import org.apache.maven.plugins.surefire.report.ReportTestCase.FlakyError;
import org.apache.maven.plugins.surefire.report.ReportTestCase.FlakyFailure;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.startsWith;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
Expand Down Expand Up @@ -708,4 +712,75 @@ public void shouldParseLargeFile() throws Exception {
// Delete test file
Files.delete(tempFile);
}

@Test
public void shouldParseFlakes() throws Exception {
// Parse test file
TestSuiteXmlParser parser = new TestSuiteXmlParser(consoleLogger);

List<ReportTestSuite> testSuites =
parser.parse("src/test/resources/fixture/testsuitexmlparser/TEST-org.acme.FlakyTest.xml");
assertTrue(parser.isValid());
assertThat(testSuites.size(), is(1));

assertThat(testSuites.size(), is(1));
ReportTestSuite report = testSuites.get(0);
assertThat(report.getFullClassName(), is("org.acme.FlakyTest"));
assertThat(report.getName(), is("FlakyTest"));
assertThat(report.getPackageName(), is("org.acme"));
assertThat(report.getNumberOfTests(), is(2));
assertThat(report.getNumberOfSkipped(), is(0));
assertThat(report.getNumberOfErrors(), is(0));
assertThat(report.getNumberOfFailures(), is(0));
assertThat(report.getNumberOfFlakes(), is(4));
assertThat(report.getTimeElapsed(), is(1.324f));
assertThat(report.getTestCases().size(), is(2));

List<ReportTestCase> tests = report.getTestCases();
assertThat(tests.get(0).getFullClassName(), is("org.acme.FlakyTest"));
assertThat(tests.get(0).getName(), is("testFlaky"));
assertNull(tests.get(0).getFailureDetail());
assertThat(tests.get(0).getClassName(), is("FlakyTest"));
assertThat(tests.get(0).getTime(), is(0.034f));
assertThat(tests.get(0).getFullName(), is("org.acme.FlakyTest.testFlaky"));
assertThat(tests.get(0).hasError(), is(false));
assertThat(tests.get(0).hasFlakes(), is(true));

List<FlakyFailure> flakyFailures = tests.get(0).getFlakyFailures();
assertThat(flakyFailures.size(), is(3));

assertThat(flakyFailures.get(0).getMessage(), startsWith("expected: <1> but was: <0>"));
assertThat(flakyFailures.get(0).getType(), is("org.opentest4j.AssertionFailedError"));
assertThat(
flakyFailures.get(0).getStackTrace(),
containsString("at org.acme.FlakyTest.testFlaky(FlakyTest.java:18)"));

assertThat(flakyFailures.get(1).getMessage(), startsWith("expected: <1> but was: <3>"));
assertThat(flakyFailures.get(1).getType(), is("org.opentest4j.AssertionFailedError"));
assertThat(
flakyFailures.get(1).getStackTrace(),
containsString("at org.acme.FlakyTest.testFlaky(FlakyTest.java:18)"));

assertThat(flakyFailures.get(2).getMessage(), startsWith("expected: <1> but was: <4>"));
assertThat(flakyFailures.get(2).getType(), is("org.opentest4j.AssertionFailedError"));
assertThat(
flakyFailures.get(2).getStackTrace(),
containsString("at org.acme.FlakyTest.testFlaky(FlakyTest.java:18)"));

List<FlakyError> flakyErrors = tests.get(0).getFlakyErrors();
assertThat(flakyErrors.size(), is(1));
assertThat(flakyErrors.get(0).getMessage(), startsWith("expected: <1> but was: <0>"));
assertThat(flakyErrors.get(0).getType(), is("org.opentest4j.AssertionFailedError"));
assertThat(
flakyErrors.get(0).getStackTrace(),
containsString("at org.acme.FlakyTest.testFlaky(FlakyTest.java:18)"));

assertThat(tests.get(1).getFullClassName(), is("org.acme.FlakyTest"));
assertThat(tests.get(1).getName(), is("testStable"));
assertThat(tests.get(1).getClassName(), is("FlakyTest"));
assertThat(tests.get(1).getTime(), is(0.001f));
assertThat(tests.get(1).getFullName(), is("org.acme.FlakyTest.testStable"));
assertThat(tests.get(1).hasError(), is(false));
assertThat(tests.get(1).hasFlakes(), is(false));
}
}

0 comments on commit ef4f2bf

Please sign in to comment.