Skip to content

Commit

Permalink
馃尡 Add RepoClient re-use E2E tests. (#2625)
Browse files Browse the repository at this point in the history
* Add e2e test for re-used repoclient.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Improve diff logging

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Skip scorecard e2e test during unit tests.

Signed-off-by: Spencer Schrock <sschrock@google.com>

* Fix linter.

Signed-off-by: Spencer Schrock <sschrock@google.com>

---------

Signed-off-by: Spencer Schrock <sschrock@google.com>
  • Loading branch information
spencerschrock committed Mar 11, 2023
1 parent a7e81bb commit d591e38
Show file tree
Hide file tree
Showing 2 changed files with 170 additions and 0 deletions.
31 changes: 31 additions & 0 deletions pkg/pkg_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright 2023 OpenSSF Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pkg_test

import (
"os"
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

func TestPkg(t *testing.T) {
if val, exists := os.LookupEnv("SKIP_GINKGO"); exists && val == "1" {
t.Skip()
}
RegisterFailHandler(Fail)
RunSpecs(t, "Pkg Suite")
}
139 changes: 139 additions & 0 deletions pkg/scorecard_e2e_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
// Copyright 2023 OpenSSF Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pkg

import (
"context"
"fmt"
"sort"
"time"

"github.com/google/go-cmp/cmp"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/checks"
"github.com/ossf/scorecard/v4/clients"
"github.com/ossf/scorecard/v4/clients/githubrepo"
sclog "github.com/ossf/scorecard/v4/log"
)

func (r *ScorecardResult) normalize() {
r.Date = time.Time{}
sort.Slice(r.Checks, func(i, j int) bool {
return r.Checks[i].Name < r.Checks[j].Name
})
}

func countDetails(c []checker.CheckDetail) (debug, info, warn int) {
for i := range c {
switch c[i].Type {
case checker.DetailDebug:
debug++
case checker.DetailInfo:
info++
case checker.DetailWarn:
warn++
}
}
return debug, info, warn
}

//nolint:lll,gocritic // comparison was failing with pointer types
func compareScorecardResults(a, b ScorecardResult) bool {
if a.Repo != b.Repo {
fmt.Fprintf(GinkgoWriter, "Unequal repo details in results: %v vs %v\n", a.Repo, b.Repo)
return false
}
if a.Scorecard != b.Scorecard {
fmt.Fprintf(GinkgoWriter, "Unequal scorecard details in results: %v vs %v\n", a.Scorecard, b.Scorecard)
return false
}
if len(a.Checks) != len(b.Checks) {
fmt.Fprintf(GinkgoWriter, "Unequal number of checks in results: %d vs %d\n", len(a.Checks), len(b.Checks))
return false
}
for i := range a.Checks {
if a.Checks[i].Name != b.Checks[i].Name {
fmt.Fprintf(GinkgoWriter, "Check name mismatch: %q vs %q\n", a.Checks[i].Name, b.Checks[i].Name)
return false
}
if a.Checks[i].Version != b.Checks[i].Version {
fmt.Fprintf(GinkgoWriter, "%q version mismatch: %d vs %d\n", a.Checks[i].Name, a.Checks[i].Version, b.Checks[i].Version)
return false
}
if a.Checks[i].Score != b.Checks[i].Score {
fmt.Fprintf(GinkgoWriter, "%q score mismatch: %d vs %d\n", a.Checks[i].Name, a.Checks[i].Score, b.Checks[i].Score)
return false
}
if a.Checks[i].Reason != b.Checks[i].Reason {
fmt.Fprintf(GinkgoWriter, "%q reason mismatch: %q vs %q\n", a.Checks[i].Name, a.Checks[i].Reason, b.Checks[i].Reason)
return false
}
// details are only compared using the number of debug, info and warn
aDebug, aInfo, aWarn := countDetails(a.Checks[i].Details)
bDebug, bInfo, bWarn := countDetails(b.Checks[i].Details)
if aDebug != bDebug || aInfo != bInfo || aWarn != bWarn {
fmt.Fprintf(GinkgoWriter, "%q details mismatch:\n", a.Checks[i].Name)
fmt.Fprintf(GinkgoWriter, "\tdebug: %d-%d\n", aDebug, bDebug)
fmt.Fprintf(GinkgoWriter, "\tinfo: %d-%d\n", aInfo, bInfo)
fmt.Fprintf(GinkgoWriter, "\twarn: %d-%d\n", aWarn, bWarn)
return false
}
}
return true
}

var _ = Describe("E2E TEST: RunScorecard with re-used repoClient", func() {
Context("E2E TEST: Validate results are identical regardless of order", func() {
assertLastResultsIdentical := func(repos []string) {
if len(repos) < 2 {
return
}
ctx := context.Background()
allChecks := checks.GetAll()

isolatedLogger := sclog.NewLogger(sclog.DebugLevel)
lastRepo := repos[len(repos)-1]
repo, rc, ofrc, cc, vc, err := checker.GetClients(ctx, lastRepo, "", isolatedLogger)
Expect(err).Should(BeNil())
isolatedResult, err := RunScorecard(ctx, repo, clients.HeadSHA, 0, allChecks, rc, ofrc, cc, vc)
Expect(err).Should(BeNil())

logger := sclog.NewLogger(sclog.DebugLevel)
_, rc2, ofrc2, cc2, vc2, err := checker.GetClients(ctx, repos[0], "", logger)
Expect(err).Should(BeNil())

var sharedResult ScorecardResult
for i := range repos {
repo, err = githubrepo.MakeGithubRepo(repos[i])
Expect(err).Should(BeNil())
sharedResult, err = RunScorecard(ctx, repo, clients.HeadSHA, 0, allChecks, rc2, ofrc2, cc2, vc2)
Expect(err).Should(BeNil())
}

isolatedResult.normalize()
sharedResult.normalize()
Expect(isolatedResult).To(BeComparableTo(sharedResult, cmp.Comparer(compareScorecardResults)))
}
It("A then B results should be produce the same distribution of details as the isolated B results", func() {
assertLastResultsIdentical([]string{
"https://github.com/ossf-tests/scorecard",
"https://github.com/ossf-tests/scorecard-action",
})
})
})
})

0 comments on commit d591e38

Please sign in to comment.