CHANGELOG.md

@@ -1,3 +1,15 @@
+## 2.23.2
+
+🎉🎉🎉
+
+At long last, some long-standing performance gaps between `ginkgo` and `go test` have been resolved!
+
+Ginkgo operates by running `go test -c` to generate test binaries, and then running those binaries.  It turns out that the compilation step of `go test -c` is slower than `go test`'s compilation step because `go test` strips out debug symbols (`ldflags=-w`) whereas `go test -c` does not.
+
+Ginkgo now passes the appropriate `ldflags` to `go test -c` when running specs to strip out symbols.  This is only done when it is safe to do so and symbols are preferred when profiling is enabled and when `ginkgo build` is called explicitly.
+
+This, coupled, with the [instructions for disabling XProtect on MacOS](https://onsi.github.io/ginkgo/#if-you-are-running-on-macos) yields a much better performance experience with Ginkgo.
+
 ## 2.23.1
 
 ## 🚨 For users on MacOS 🚨

ginkgo/build/build_command.go

@@ -44,7 +44,7 @@ func buildSpecs(args []string, cliConfig types.CLIConfig, goFlagsConfig types.Go
 	internal.VerifyCLIAndFrameworkVersion(suites)
 
 	opc := internal.NewOrderedParallelCompiler(cliConfig.ComputedNumCompilers())
-	opc.StartCompiling(suites, goFlagsConfig)
+	opc.StartCompiling(suites, goFlagsConfig, true)
 
 	for {
 		suiteIdx, suite := opc.Next()

ginkgo/command/command.go

@@ -26,7 +26,7 @@ func (c Command) Run(args []string, additionalArgs []string) {
 	}
 	for _, arg := range args {
 		if strings.HasPrefix(arg, "-") {
-			AbortWith("Malformed arguments - make sure all flags appear {{bold}}after{{/}} the Ginkgo subcommand and {{bold}}before{{/}} your list of packages.\n{{gray}}e.g. 'ginkgo run -p my_package' is valid `ginkgo -p run my_package` is not.{{/}}")
+			AbortWith(types.GinkgoErrors.FlagAfterPositionalParameter().Error())
 		}
 	}
 	c.Command(args, additionalArgs)

ginkgo/internal/compile.go

@@ -11,7 +11,7 @@ import (
 	"github.com/onsi/ginkgo/v2/types"
 )
 
-func CompileSuite(suite TestSuite, goFlagsConfig types.GoFlagsConfig) TestSuite {
+func CompileSuite(suite TestSuite, goFlagsConfig types.GoFlagsConfig, preserveSymbols bool) TestSuite {
 	if suite.PathToCompiledTest != "" {
 		return suite
 	}
@@ -46,7 +46,7 @@ func CompileSuite(suite TestSuite, goFlagsConfig types.GoFlagsConfig) TestSuite
 		suite.CompilationError = fmt.Errorf("Failed to get relative path from package to the current working directory:\n%s", err.Error())
 		return suite
 	}
-	args, err := types.GenerateGoTestCompileArgs(goFlagsConfig, "./", pathToInvocationPath)
+	args, err := types.GenerateGoTestCompileArgs(goFlagsConfig, "./", pathToInvocationPath, preserveSymbols)
 	if err != nil {
 		suite.State = TestSuiteStateFailedToCompile
 		suite.CompilationError = fmt.Errorf("Failed to generate go test compile flags:\n%s", err.Error())
@@ -120,7 +120,7 @@ func NewOrderedParallelCompiler(numCompilers int) *OrderedParallelCompiler {
 	}
 }
 
-func (opc *OrderedParallelCompiler) StartCompiling(suites TestSuites, goFlagsConfig types.GoFlagsConfig) {
+func (opc *OrderedParallelCompiler) StartCompiling(suites TestSuites, goFlagsConfig types.GoFlagsConfig, preserveSymbols bool) {
 	opc.stopped = false
 	opc.idx = 0
 	opc.numSuites = len(suites)
@@ -135,7 +135,7 @@ func (opc *OrderedParallelCompiler) StartCompiling(suites TestSuites, goFlagsCon
 				stopped := opc.stopped
 				opc.mutex.Unlock()
 				if !stopped {
-					suite = CompileSuite(suite, goFlagsConfig)
+					suite = CompileSuite(suite, goFlagsConfig, preserveSymbols)
 				}
 				c <- suite
 			}

ginkgo/performance/performance_suite_test.go

@@ -51,8 +51,8 @@ var _ = SynchronizedAfterSuite(func() {}, func() {
 })
 
 /*
-	GoModCacheManager sets up a new GOMODCACHE and knows how to clear it
-	This allows us to bust the go mod cache.
+GoModCacheManager sets up a new GOMODCACHE and knows how to clear it
+This allows us to bust the go mod cache.
 */
 type GoModCacheManager struct {
 	Path         string
@@ -302,7 +302,7 @@ func RunScenarioWithGinkgoInternals(stopwatch *gmeasure.Stopwatch, settings Scen
 			for suite := range compile {
 				if !suite.State.Is(internal.TestSuiteStateCompiled) {
 					subStopwatch := stopwatch.NewStopwatch()
-					suite = internal.CompileSuite(suite, goFlagsConfig)
+					suite = internal.CompileSuite(suite, goFlagsConfig, false)
 					subStopwatch.Record("compile-test: "+suite.PackageName, annotation)
 					Ω(suite.CompilationError).Should(BeNil())
 				}

ginkgo/run/run_command.go

@@ -107,7 +107,7 @@ OUTER_LOOP:
 		}
 
 		opc := internal.NewOrderedParallelCompiler(r.cliConfig.ComputedNumCompilers())
-		opc.StartCompiling(suites, r.goFlagsConfig)
+		opc.StartCompiling(suites, r.goFlagsConfig, false)
 
 	SUITE_LOOP:
 		for {

ginkgo/watch/watch_command.go

@@ -153,7 +153,7 @@ func (w *SpecWatcher) WatchSpecs(args []string, additionalArgs []string) {
 }
 
 func (w *SpecWatcher) compileAndRun(suite internal.TestSuite, additionalArgs []string) internal.TestSuite {
-	suite = internal.CompileSuite(suite, w.goFlagsConfig)
+	suite = internal.CompileSuite(suite, w.goFlagsConfig, false)
 	if suite.State.Is(internal.TestSuiteStateFailedToCompile) {
 		fmt.Println(suite.CompilationError.Error())
 		return suite

integration/_fixtures/symbol_fixture/symbol_fixture_suite_test.go

@@ -0,0 +1,22 @@
+package symbol_fixture_test
+
+import (
+	"fmt"
+	"os/exec"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestSymbolFixture(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "SymbolFixture Suite")
+}
+
+var _ = It("prints out its symbols", func() {
+	cmd := exec.Command("go", "tool", "nm", "symbol_fixture.test")
+	output, err := cmd.CombinedOutput()
+	Expect(err).NotTo(HaveOccurred())
+	fmt.Println(string(output))
+})

integration/precompiled_test.go

@@ -25,6 +25,14 @@ var _ = Describe("ginkgo build", func() {
 		Ω(fm.PathTo("passing_ginkgo_tests", "passing_ginkgo_tests.test")).Should(BeAnExistingFile())
 	})
 
+	It("should have the symbols in the compiled binary", func() {
+		cmd := exec.Command("go", "tool", "nm", fm.PathTo("passing_ginkgo_tests", "passing_ginkgo_tests.test"))
+		session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
+		Ω(err).ShouldNot(HaveOccurred())
+		Eventually(session).Should(gexec.Exit(0))
+		Ω(session).Should(gbytes.Say("github.com/onsi/ginkgo/v2.It")) // a symbol from ginkgo
+	})
+
 	It("should be possible to run the test binary directly", func() {
 		cmd := exec.Command("./passing_ginkgo_tests.test")
 		cmd.Dir = fm.PathTo("passing_ginkgo_tests")

integration/run_test.go

@@ -458,6 +458,27 @@ var _ = Describe("Running Specs", func() {
 		})
 	})
 
+	Describe("optimizing build times by stripping symbols", func() {
+		BeforeEach(func() {
+			fm.MountFixture("symbol")
+		})
+		Context("when no symbols are required (i.e. no profiling)", func() {
+			It("should not have symbols", func() {
+				session := startGinkgo(fm.PathTo("symbol"), "--no-color")
+				Eventually(session).Should(gexec.Exit(0))
+				Ω(session).ShouldNot(gbytes.Say("github.com/onsi/ginkgo/v2.It")) // a symbol from ginkgo
+			})
+		})
+
+		Context("when symbols are required (i.e. with profiling)", func() {
+			It("should have symbols", func() {
+				session := startGinkgo(fm.PathTo("symbol"), "--no-color", "--cpuprofile=cpu.out")
+				Eventually(session).Should(gexec.Exit(0))
+				Ω(session).Should(gbytes.Say("github.com/onsi/ginkgo/v2.It")) // a symbol from ginkgo
+			})
+		})
+	})
+
 	Context("when there is a version mismatch between the cli and the test package", func() {
 		It("emits a useful error and tries running", func() {
 			fm.MountFixture(("version_mismatch"))

types/config.go

@@ -231,6 +231,10 @@ func (g GoFlagsConfig) BinaryMustBePreserved() bool {
 	return g.BlockProfile != "" || g.CPUProfile != "" || g.MemProfile != "" || g.MutexProfile != ""
 }
 
+func (g GoFlagsConfig) NeedsSymbols() bool {
+	return g.BinaryMustBePreserved()
+}
+
 // Configuration that were deprecated in 2.0
 type deprecatedConfig struct {
 	DebugParallel                   bool
@@ -640,7 +644,7 @@ func VetAndInitializeCLIAndGoConfig(cliConfig CLIConfig, goFlagsConfig GoFlagsCo
 }
 
 // GenerateGoTestCompileArgs is used by the Ginkgo CLI to generate command line arguments to pass to the go test -c command when compiling the test
-func GenerateGoTestCompileArgs(goFlagsConfig GoFlagsConfig, packageToBuild string, pathToInvocationPath string) ([]string, error) {
+func GenerateGoTestCompileArgs(goFlagsConfig GoFlagsConfig, packageToBuild string, pathToInvocationPath string, preserveSymbols bool) ([]string, error) {
 	// if the user has set the CoverProfile run-time flag make sure to set the build-time cover flag to make sure
 	// the built test binary can generate a coverprofile
 	if goFlagsConfig.CoverProfile != "" {
@@ -663,6 +667,10 @@ func GenerateGoTestCompileArgs(goFlagsConfig GoFlagsConfig, packageToBuild strin
 		goFlagsConfig.CoverPkg = strings.Join(adjustedCoverPkgs, ",")
 	}
 
+	if !goFlagsConfig.NeedsSymbols() && goFlagsConfig.LDFlags == "" && !preserveSymbols {
+		goFlagsConfig.LDFlags = "-w -s"
+	}
+
 	args := []string{"test", "-c", packageToBuild}
 	goArgs, err := GenerateFlagArgs(
 		GoBuildFlags,

types/errors.go

@@ -636,6 +636,13 @@ func (g ginkgoErrors) ExpectFilenameNotPath(flag string, path string) error {
 	}
 }
 
+func (g ginkgoErrors) FlagAfterPositionalParameter() error {
+	return GinkgoError{
+		Heading: "Malformed arguments - detected a flag after the package liste",
+		Message: "Make sure all flags appear {{bold}}after{{/}} the Ginkgo subcommand and {{bold}}before{{/}} your list of packages (or './...').\n{{gray}}e.g. 'ginkgo run -p my_package' is valid but `ginkgo -p run my_package` is not.\n{{gray}}e.g. 'ginkgo -p -vet ./...' is valid but 'ginkgo -p ./... -vet' is not{{/}}",
+	}
+}
+
 /* Stack-Trace parsing errors */
 
 func (g ginkgoErrors) FailedToParseStackTrace(message string) error {

types/version.go

@@ -1,3 +1,3 @@
 package types
 
-const VERSION = "2.23.1"
+const VERSION = "2.23.2"