xunit/visualstudio.xunit#317: Slow performance on discovery / running…

… due to discovering TestReporters

src/xunit.console/ConsoleRunner.cs

@@ -126,50 +126,23 @@ public int EntryPoint(string[] args)
 
         List<IRunnerReporter> GetAvailableRunnerReporters()
         {
-            var result = new List<IRunnerReporter>();
+            var result = RunnerReporterUtility.GetAvailableRunnerReporters(Path.GetDirectoryName(typeof(Program).GetTypeInfo().Assembly.Location), out var messages);
 
-            var runnerPath = Path.GetDirectoryName(typeof(Program).GetTypeInfo().Assembly.Location);
-
-            foreach (var dllFile in Directory.GetFiles(runnerPath, "*.dll").Select(f => Path.Combine(runnerPath, f)))
-            {
-                Type[] types = new Type[0];
-
-                try
-                {
-#if NETFRAMEWORK
-                    var assembly = Assembly.LoadFile(dllFile);
-#else
-                    var assembly = Assembly.Load(new AssemblyName(Path.GetFileNameWithoutExtension(dllFile)));
-#endif
-                    types = assembly.GetTypes();
-                }
-                catch (ReflectionTypeLoadException ex)
-                {
-                    types = ex.Types;
-                }
-                catch
+            if (messages.Count > 0)
+                lock (consoleLock)
                 {
-                    continue;
-                }
+                    if (!commandLine.NoColor)
+                        ConsoleHelper.SetForegroundColor(ConsoleColor.Yellow);
 
-                foreach (var type in types)
-                {
-#pragma warning disable CS0618
-                    if (type == null || type.GetTypeInfo().IsAbstract || type == typeof(DefaultRunnerReporter) || type == typeof(DefaultRunnerReporterWithTypes) || !type.GetInterfaces().Any(t => t == typeof(IRunnerReporter)))
-                        continue;
-#pragma warning restore CS0618
-                    var ctor = type.GetConstructor(new Type[0]);
-                    if (ctor == null)
+                    foreach (var message in messages)
                     {
-                        ConsoleHelper.SetForegroundColor(ConsoleColor.Yellow);
-                        Console.WriteLine($"Type {type.FullName} in assembly {dllFile} appears to be a runner reporter, but does not have an empty constructor.");
-                        ConsoleHelper.ResetColor();
-                        continue;
+                        Console.WriteLine(message);
+                        Console.WriteLine();
                     }
 
-                    result.Add((IRunnerReporter)ctor.Invoke(new object[0]));
+                    if (!commandLine.NoColor)
+                        ConsoleHelper.ResetColor();
                 }
-            }
 
             return result;
         }

src/xunit.runner.msbuild/xunit.cs

@@ -355,44 +355,9 @@ protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly, AppDom
 
         protected virtual List<IRunnerReporter> GetAvailableRunnerReporters()
         {
-            var result = new List<IRunnerReporter>();
-            var runnerPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().GetLocalCodeBase());
-
-            foreach (var dllFile in Directory.GetFiles(runnerPath, "*.dll").Select(f => Path.Combine(runnerPath, f)))
-            {
-                Type[] types;
-
-                try
-                {
-                    var assembly = CrossPlatform.LoadAssembly(dllFile);
-                    types = assembly.GetTypes();
-                }
-                catch (ReflectionTypeLoadException ex)
-                {
-                    types = ex.Types;
-                }
-                catch
-                {
-                    continue;
-                }
-
-                foreach (var type in types)
-                {
-#pragma warning disable CS0618
-                    if (type == null || type.GetTypeInfo().IsAbstract || type == typeof(DefaultRunnerReporter) || type == typeof(DefaultRunnerReporterWithTypes) || !type.GetInterfaces().Any(t => t == typeof(IRunnerReporter)))
-                        continue;
-#pragma warning restore CS0618
-
-                    var ctor = type.GetConstructor(new Type[0]);
-                    if (ctor == null)
-                    {
-                        Log.LogWarning("Type {0} in assembly {1} appears to be a runner reporter, but does not have an empty constructor.", type.FullName, dllFile);
-                        continue;
-                    }
-
-                    result.Add((IRunnerReporter)ctor.Invoke(new object[0]));
-                }
-            }
+            var result = RunnerReporterUtility.GetAvailableRunnerReporters(Path.GetDirectoryName(Assembly.GetExecutingAssembly().GetLocalCodeBase()), out var messages);
+            foreach (var message in messages)
+                Log.LogWarning(message);
 
             return result;
         }

src/xunit.runner.utility/Reporters/RunnerReporterUtility.cs

@@ -0,0 +1,93 @@
+#if NETFRAMEWORK || NETCOREAPP || NETSTANDARD1_5_OR_GREATER
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+
+namespace Xunit
+{
+    /// <summary>
+    /// A utility class for finding runner reporters.
+    /// </summary>
+    public static class RunnerReporterUtility
+    {
+        /// <summary>
+        /// Gets a list of runner reporters from DLLs in the given folder. The only DLLs that are searched are those
+        /// named "*reporters*.dll"
+        /// </summary>
+        /// <param name="folder">The folder to search for reporters in</param>
+        /// <param name="messages">Messages that were generated during discovery</param>
+        /// <returns>List of available reporters</returns>
+        public static List<IRunnerReporter> GetAvailableRunnerReporters(string folder, out List<string> messages)
+        {
+            var result = new List<IRunnerReporter>();
+            messages = new List<string>();
+            string[] dllFiles;
+
+            try
+            {
+                dllFiles = Directory.GetFiles(folder, "*reporters*.dll").Select(f => Path.Combine(folder, f)).ToArray();
+            }
+            catch (Exception ex)
+            {
+                messages.Add($"Exception thrown looking for reporters in folder '{folder}':{Environment.NewLine}{ex}");
+                return result;
+            }
+
+            foreach (var dllFile in dllFiles)
+            {
+                Type[] types = new Type[0];
+
+                try
+                {
+#if NETFRAMEWORK
+                    var assembly = Assembly.LoadFile(dllFile);
+#else
+                    var assembly = Assembly.Load(new AssemblyName(Path.GetFileNameWithoutExtension(dllFile)));
+#endif
+                    types = assembly.GetTypes();
+                }
+                catch (ReflectionTypeLoadException ex)
+                {
+                    types = ex.Types;
+                }
+                catch
+                {
+                    continue;
+                }
+
+                foreach (var type in types)
+                {
+#if NETFRAMEWORK
+                    if (type == null || type.IsAbstract || !type.GetInterfaces().Any(t => t == typeof(IRunnerReporter)))
+#else
+                    if (type == null || type.GetTypeInfo().IsAbstract || !type.GetInterfaces().Any(t => t == typeof(IRunnerReporter)))
+#endif
+                        continue;
+
+                    try
+                    {
+                        var ctor = type.GetConstructor(new Type[0]);
+                        if (ctor == null)
+                        {
+                            messages.Add($"Type '{type.FullName ?? type.Name}' in assembly '{dllFile}' appears to be a runner reporter, but does not have an empty constructor.");
+                            continue;
+                        }
+
+                        result.Add((IRunnerReporter)ctor.Invoke(new object[0]));
+                    }
+                    catch (Exception ex)
+                    {
+                        messages.Add($"Exception thrown while inspecting type '{type.FullName ?? type.Name}' in assembly '{dllFile}':{Environment.NewLine}{ex}");
+                    }
+                }
+            }
+
+            return result;
+        }
+    }
+}
+
+#endif