[SUREFIRE-2065] Fix parameterized JUnit4 test reporting

surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/TestFile.java

@@ -155,6 +155,25 @@ public TestFile assertContainsText( String text )
         return assertContainsText( containsString( text ) );
     }
 
+    public TestFile assertNotContainsText( Matcher<String> matcher )
+    {
+        final List<String> list = surefireVerifier.loadFile( file, encoding );
+        for ( String line : list )
+        {
+            if ( matcher.matches( line ) )
+            {
+                Assert.fail( "Found unexpected message in log" );
+                return null;
+            }
+        }
+        return this;
+    }
+
+    public TestFile assertNotContainsText( String text )
+    {
+        return assertNotContainsText( containsString( text ) );
+    }
+
     public URI toURI()
     {
         return file.toURI();

surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2065IT.java

@@ -0,0 +1,103 @@
+package org.apache.maven.surefire.its.jiras;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import org.apache.maven.it.VerificationException;
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.hamcrest.Matchers;
+import org.junit.Test;
+
+/**
+ * Integration Tests for SUREFIRE-2065
+ */
+@SuppressWarnings( "checkstyle:magicnumber" )
+public class Surefire2065IT extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void shouldNotDetectFlakyTestsWhenCombiningJunit4And5Tests() throws VerificationException
+    {
+        OutputValidator validator = unpack( "surefire-2065-common" )
+            .mavenTestFailureIgnore( true )
+            .executeTest()
+            .assertTestSuiteResults( 8, 0, 4, 0, 0 );
+
+        assertJunit4( validator );
+        assertJunit5( validator );
+    }
+
+    @Test
+    public void shouldNotDetectFlakyTestsWhenRunningOnlyJunit4() throws VerificationException
+    {
+        OutputValidator validator = unpack( "surefire-2065-junit4" )
+            .mavenTestFailureIgnore( true )
+            .executeTest()
+            .assertTestSuiteResults( 4, 0, 2, 0, 0 );
+
+        assertJunit4( validator );
+    }
+
+    @Test
+    public void shouldNotDetectFlakyTestsWhenRunningOnlyJunit5() throws VerificationException
+    {
+        OutputValidator validator = unpack( "surefire-2065-junit5" )
+            .mavenTestFailureIgnore( true )
+            .executeTest()
+            .assertTestSuiteResults( 4, 0, 2, 0, 0 );
+
+        assertJunit5( validator );
+    }
+
+    private static void assertJunit4( OutputValidator validator )
+    {
+        validator.getSurefireReportsFile( "TEST-pkg.junit4.ParameterizedTest.xml", UTF_8 )
+            .assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[0]\".*/>$" ) )
+            .assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[1]\".*[^/]>$" ) )
+            .assertContainsText( "<failure message=" )
+            .assertContainsText( "<rerunFailure message=" )
+            .assertNotContainsText( "<flakyFailure message=" );
+
+        validator.getSurefireReportsFile( "TEST-pkg.junit4.ParameterizedWithDisplayNameTest.xml", UTF_8 )
+            .assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[value=0]\".*/>$" ) )
+            .assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[value=1]\".*[^/]>$" ) )
+            .assertContainsText( "<failure message=" )
+            .assertContainsText( "<rerunFailure message=" )
+            .assertNotContainsText( "<flakyFailure message=" );
+    }
+
+    private static void assertJunit5( OutputValidator validator )
+    {
+        validator.getSurefireReportsFile( "TEST-pkg.junit5.ParameterizedTest.xml", UTF_8 )
+            .assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[1]\".*/>$" ) )
+            .assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[2]\".*[^/]>$" ) )
+            .assertContainsText( "<failure message=" )
+            .assertContainsText( "<rerunFailure message=" )
+            .assertNotContainsText( "<flakyFailure message=" );
+
+        validator.getSurefireReportsFile( "TEST-pkg.junit5.ParameterizedWithDisplayNameTest.xml", UTF_8 )
+            .assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[1]\".*/>$" ) )
+            .assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[2]\".*[^/]>$" ) )
+            .assertContainsText( "<failure message=" )
+            .assertContainsText( "<rerunFailure message=" )
+            .assertNotContainsText( "<flakyFailure message=" );
+    }
+}

surefire-its/src/test/resources/surefire-2065-common/pom.xml

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.example</groupId>
+  <artifactId>surefire-2065-common</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.source>${java.specification.version}</maven.compiler.source>
+    <maven.compiler.target>${java.specification.version}</maven.compiler.target>
+    <junit4.version>4.12</junit4.version>
+    <junit5.version>5.3.2</junit5.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>${junit4.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <version>${junit5.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <version>${junit5.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>${surefire.version}</version>
+        <configuration>
+          <rerunFailingTestsCount>1</rerunFailingTestsCount>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

surefire-its/src/test/resources/surefire-2065-common/src/test/java/pkg/junit4/ParameterizedTest.java

@@ -0,0 +1,29 @@
+package pkg.junit4;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class ParameterizedTest
+{
+    @Parameterized.Parameters
+    public static List<Integer> parameters() throws Exception
+    {
+        return Arrays.asList( 0, 1 );
+    }
+
+    @Parameterized.Parameter(0)
+    public int expected;
+
+    @Test
+    public void notFlaky()
+    {
+        assertEquals( expected, 0 );
+    }
+}

surefire-its/src/test/resources/surefire-2065-common/src/test/java/pkg/junit4/ParameterizedWithDisplayNameTest.java

@@ -0,0 +1,31 @@
+package pkg.junit4;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class ParameterizedWithDisplayNameTest
+{
+    private static int count = 0;
+
+    @Parameterized.Parameters(name = "value={0}")
+    public static List<Integer> parameters() throws Exception
+    {
+        return Arrays.asList( 0, 1 );
+    }
+
+    @Parameterized.Parameter(0)
+    public int expected;
+
+    @Test
+    public void notFlaky()
+    {
+        assertEquals( expected, 0 );
+    }
+}

surefire-its/src/test/resources/surefire-2065-common/src/test/java/pkg/junit5/ParameterizedTest.java

@@ -0,0 +1,30 @@
+package pkg.junit5;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.Arrays;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.ArgumentsProvider;
+import org.junit.jupiter.params.provider.ArgumentsSource;
+
+class ParameterizedTest
+{
+    static class ParameterSource implements ArgumentsProvider
+    {
+        @Override
+        public Stream<? extends Arguments> provideArguments( ExtensionContext context ) throws Exception
+        {
+            return Arrays.asList(0, 1).stream().map( Arguments::of );
+        }
+    }
+
+    @org.junit.jupiter.params.ParameterizedTest
+    @ArgumentsSource(value = ParameterSource.class)
+    public void notFlaky(int expected)
+    {
+        assertEquals( expected, 0 );
+    }
+}

surefire-its/src/test/resources/surefire-2065-common/src/test/java/pkg/junit5/ParameterizedWithDisplayNameTest.java

@@ -0,0 +1,30 @@
+package pkg.junit5;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.Arrays;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.ArgumentsProvider;
+import org.junit.jupiter.params.provider.ArgumentsSource;
+
+class ParameterizedWithDisplayNameTest
+{
+    static class ParameterSource implements ArgumentsProvider
+    {
+        @Override
+        public Stream<? extends Arguments> provideArguments( ExtensionContext context ) throws Exception
+        {
+            return Arrays.asList(0, 1).stream().map( Arguments::of );
+        }
+    }
+
+    @org.junit.jupiter.params.ParameterizedTest(name = "value={0}")
+    @ArgumentsSource(value = ParameterSource.class)
+    public void notFlaky(int expected)
+    {
+        assertEquals( expected, 0 );
+    }
+}

surefire-its/src/test/resources/surefire-2065-junit4/pom.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.example</groupId>
+  <artifactId>surefire-2065-junit4</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.source>${java.specification.version}</maven.compiler.source>
+    <maven.compiler.target>${java.specification.version}</maven.compiler.target>
+    <junit4.version>4.12</junit4.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>${junit4.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>${surefire.version}</version>
+        <configuration>
+          <rerunFailingTestsCount>1</rerunFailingTestsCount>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>