Detect GitLab repos including self-hosted

Signed-off-by: Raghav Kaul <raghavkaul@google.com>

checker/client.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"strings"
 
 	"github.com/ossf/scorecard/v4/clients"
 	ghrepo "github.com/ossf/scorecard/v4/clients/githubrepo"
@@ -56,15 +55,18 @@ func GetClients(ctx context.Context, repoURI, localURI string, logger *log.Logge
 
 	_, experimental := os.LookupEnv("SCORECARD_EXPERIMENTAL")
 
-	if strings.Contains(repoURI, "gitlab.") && experimental {
-		repo, makeRepoError = glrepo.MakeGitlabRepo(repoURI)
-		if makeRepoError != nil {
-			return repo,
-				nil,
-				nil,
-				nil,
-				nil,
-				fmt.Errorf("getting local directory client: %w", makeRepoError)
+	//nolint:nestif
+	if experimental {
+		if isGl := glrepo.DetectGitLab(repoURI); isGl {
+			repo, makeRepoError = glrepo.MakeGitlabRepo(repoURI)
+			if makeRepoError != nil {
+				return repo,
+					nil,
+					nil,
+					nil,
+					nil,
+					fmt.Errorf("getting local directory client: %w", makeRepoError)
+			}
 		}
 	} else {
 		repo, makeRepoError = ghrepo.MakeGithubRepo(repoURI)
@@ -84,7 +86,7 @@ func GetClients(ctx context.Context, repoURI, localURI string, logger *log.Logge
 		retErr = fmt.Errorf("getting OSS-Fuzz repo client: %w", errOssFuzz)
 	}
 	// TODO(repo): Should we be handling the OSS-Fuzz client error like this?
-	if strings.Contains(repoURI, "gitlab.") && experimental {
+	if glrepo.DetectGitLab(repoURI) && experimental {
 		glClient, err := glrepo.CreateGitlabClientWithToken(ctx, os.Getenv("GITLAB_AUTH_TOKEN"), repo)
 		if err != nil {
 			return repo,

checks/raw/ci_tests.go

@@ -57,7 +57,6 @@ func CITests(c clients.RepoClient) (checker.CITestData, error) {
 				fmt.Sprintf("Client.Repositories.ListCheckRunsForRef: %v", err),
 			)
 		}
-		fmt.Printf("crs: %v\n", crs)
 
 		runs[pr.HeadSHA] = append(runs[pr.HeadSHA], crs...)
 

clients/gitlabrepo/client.go

@@ -280,3 +280,14 @@ func CreateGitlabClientWithToken(ctx context.Context, token string, repo clients
 func CreateOssFuzzRepoClient(ctx context.Context, logger *log.Logger) (clients.RepoClient, error) {
 	return nil, fmt.Errorf("%w, oss fuzz currently only supported for github repos", clients.ErrUnsupportedFeature)
 }
+
+// DetectGitLab: check whether the repoURI is a GitLab URI
+// Makes HTTP request to GitLab API.
+func DetectGitLab(repoURI string) bool {
+	var repo repoURL
+	if err := repo.parse(repoURI); err != nil {
+		return false
+	}
+
+	return repo.IsValid() == nil
+}

clients/gitlabrepo/repo.go

@@ -19,9 +19,10 @@ package gitlabrepo
 import (
 	"fmt"
 	"net/url"
-	"regexp"
 	"strings"
 
+	"github.com/xanzy/go-gitlab"
+
 	"github.com/ossf/scorecard/v4/clients"
 	sce "github.com/ossf/scorecard/v4/errors"
 )
@@ -105,12 +106,26 @@ func (r *repoURL) Org() clients.Repo {
 
 // IsValid implements Repo.IsValid.
 func (r *repoURL) IsValid() error {
-	hostMatched, err := regexp.MatchString("gitlab.*com", r.host)
+	if strings.Contains(r.host, "gitlab.") {
+		return nil
+	}
+
+	client, err := gitlab.NewClient("", gitlab.WithBaseURL(fmt.Sprintf("%s://%s", r.scheme, r.host)))
 	if err != nil {
-		return fmt.Errorf("error processing regex: %w", err)
+		return sce.WithMessage(err,
+			fmt.Sprintf("couldn't create gitlab client for %s", r.host),
+		)
 	}
-	if !hostMatched {
-		return sce.WithMessage(sce.ErrorInvalidURL, "non gitlab repository found")
+	_, resp, err := client.Projects.ListProjects(&gitlab.ListProjectsOptions{})
+	if resp == nil || resp.StatusCode != 200 {
+		return sce.WithMessage(sce.ErrRepoUnreachable,
+			fmt.Sprintf("couldn't reach gitlab instance at %s", r.host),
+		)
+	}
+	if err != nil {
+		return sce.WithMessage(err,
+			fmt.Sprintf("error when connecting to gitlab instance at %s", r.host),
+		)
 	}
 
 	if strings.TrimSpace(r.owner) == "" || strings.TrimSpace(r.project) == "" {

clients/gitlabrepo/repo_test.go

@@ -31,90 +31,58 @@ func TestRepoURL_IsValid(t *testing.T) {
 		wantErr  bool
 	}{
 		{
-			name: "valid http address",
-			expected: repoURL{
-				scheme:  "http",
-				host:    "gitlab.example.com",
-				owner:   "foo",
-				project: "1234",
-			},
-			inputURL: "http://gitlab.example.com/foo/1234",
-			wantErr:  false,
-		},
-		{
-			name: "valid https address",
+			name: "github repository",
 			expected: repoURL{
 				scheme:  "https",
-				host:    "gitlab.example.com",
-				owner:   "foo",
-				project: "1234",
+				host:    "github.com",
+				owner:   "ossf",
+				project: "scorecard",
 			},
-			inputURL: "https://gitlab.example.com/foo/1234",
-			wantErr:  false,
+			inputURL: "https://github.com/ossf/scorecard",
+			wantErr:  true,
 		},
 		{
-			name: "valid http address with trailing slash",
+			name: "GitHub project with 'gitlab.' in the title",
 			expected: repoURL{
 				scheme:  "http",
-				host:    "gitlab.example.com",
-				owner:   "foo",
-				project: "1234",
-			},
-			inputURL: "http://gitlab.example.com/foo/1234/",
-			wantErr:  false,
-		},
-		{
-			name: "valid https address with trailing slash",
-			expected: repoURL{
-				scheme:  "https",
-				host:    "gitlab.example.com",
-				owner:   "foo",
-				project: "1234",
-			},
-			inputURL: "https://gitlab.example.com/foo/1234/",
-			wantErr:  false,
-		},
-		{
-			name: "non gitlab repository",
-			expected: repoURL{
-				scheme:  "https",
 				host:    "github.com",
 				owner:   "foo",
-				project: "1234",
+				project: "gitlab.test",
 			},
-			inputURL: "https://github.com/foo/1234",
+			inputURL: "http://github.com/foo/gitlab.test",
 			wantErr:  true,
 		},
 		{
-			name: "GitLab project with wrong projectID",
+			name: "valid gitlab project",
 			expected: repoURL{
-				scheme:  "https",
-				host:    "gitlab.example.com",
-				owner:   "foo",
-				project: "bar",
+				host:    "gitlab.com",
+				owner:   "ossf-test",
+				project: "scorecard-check-binary-artifacts-e2e",
 			},
-			inputURL: "https://gitlab.example.com/foo/bar",
+			inputURL: "gitlab.com/ossf-test/scorecard-check-binary-artifacts-e2e",
 			wantErr:  false,
 		},
 		{
-			name: "GitHub project with 'gitlab.' in the title",
+			name: "valid https address with trailing slash",
 			expected: repoURL{
-				scheme:  "http",
-				host:    "github.com",
-				owner:   "foo",
-				project: "gitlab.test",
+				scheme:  "https",
+				host:    "gitlab.com",
+				owner:   "ossf-test",
+				project: "scorecard-check-binary-artifacts-e2e",
 			},
-			inputURL: "http://github.com/foo/gitlab.test",
-			wantErr:  true,
+			inputURL: "https://gitlab.com/ossf-test/scorecard-check-binary-artifacts-e2e/",
+			wantErr:  false,
 		},
+
 		{
-			name: "valid gitlab project without http or https",
+			name: "valid hosted gitlab project",
 			expected: repoURL{
-				host:    "gitlab.example.com",
-				owner:   "foo",
-				project: "1234",
+				scheme:  "https",
+				host:    "salsa.debian.org",
+				owner:   "webmaster-team",
+				project: "webml",
 			},
-			inputURL: "gitlab.example.com/foo/1234",
+			inputURL: "https://salsa.debian.org/webmaster-team/webwml",
 			wantErr:  false,
 		},
 	}