Feature: add search support for sha512 (#1142)

* Feature: add search support for sha512

This commit introduces the possibility to *search* via sha512 digests.
Example of how this could be useful is for the npm ecosystem. npm often relies
on sha512 digests, and provenance attestations in intoto format may use
sha512 digests as the subject. Subjects are extract during uploads and added
as index keys, which are then stored in Redis. This feature lets the cli
accept sha512 digests, and the server to accept them to use when searching in
the Redis cache.

Signed-off-by: Fredrik Skogman <kommendorkapten@github.com>

* Added SHA512 to the cli help

Signed-off-by: Fredrik Skogman <kommendorkapten@github.com>

* Re-ran code generation.

Signed-off-by: Fredrik Skogman <kommendorkapten@github.com>

* Added e2e tests that uploads an intoto attestation, with a SHA512 subject.
The tests then verifies that the entry can be found via the SHA512 digest.

Signed-off-by: Fredrik Skogman <kommendorkapten@github.com>

Signed-off-by: Fredrik Skogman <kommendorkapten@github.com>

.gitignore

@@ -20,3 +20,4 @@ trillianSignerImagerefs
 cosign.*
 signature
 rekor.pub
+*~

cmd/rekor-cli/app/pflags.go

@@ -219,6 +219,7 @@ func isURL(v string) bool {
 }
 
 // validateSHAValue ensures that the supplied string matches the following formats:
+// [sha512:]<128 hexadecimal characters>
 // [sha256:]<64 hexadecimal characters>
 // [sha1:]<40 hexadecimal characters>
 // where [sha256:] and [sha1:] are optional
@@ -228,11 +229,17 @@ func validateSHAValue(v string) error {
 		return nil
 	}
 
-	if err := util.ValidateSHA256Value(v); err != nil {
-		return fmt.Errorf("error parsing %v flag: %w", shaFlag, err)
+	err = util.ValidateSHA256Value(v)
+	if err == nil {
+		return nil
 	}
 
-	return nil
+	err = util.ValidateSHA512Value(v)
+	if err == nil {
+		return nil
+	}
+
+	return fmt.Errorf("error parsing %v flag: %w", shaFlag, err)
 }
 
 // validateFileOrURL ensures the provided string is either a valid file path that can be opened or a valid URL

cmd/rekor-cli/app/pflags_test.go

@@ -638,17 +638,59 @@ func TestSearchPFlags(t *testing.T) {
 			expectValidateSuccess: true,
 		},
 		{
-			caseDesc:              "valid SHA",
+			caseDesc:              "valid SHA1",
+			sha:                   "84374135959eacf60cf3fed7520a01b336332efe",
+			expectParseSuccess:    true,
+			expectValidateSuccess: true,
+		},
+		{
+			caseDesc:              "valid SHA1 with prefix",
+			sha:                   "sha1:84374135959eacf60cf3fed7520a01b336332efe",
+			expectParseSuccess:    true,
+			expectValidateSuccess: true,
+		},
+		{
+			caseDesc:              "valid SHA256",
 			sha:                   "45c7b11fcbf07dec1694adecd8c5b85770a12a6c8dfdcf2580a2db0c47c31779",
 			expectParseSuccess:    true,
 			expectValidateSuccess: true,
 		},
+		{
+			caseDesc:              "valid SHA256 with prefix",
+			sha:                   "sha256:45c7b11fcbf07dec1694adecd8c5b85770a12a6c8dfdcf2580a2db0c47c31779",
+			expectParseSuccess:    true,
+			expectValidateSuccess: true,
+		},
+		{
+			caseDesc:              "valid SHA512",
+			sha:                   "a5d575f245588b64bcec78a1bb9d92a66bfb4d68d7de1aea4162ad0b232753860cb764fd0645ada1f5d935163522987359e515e0594068d7bc108f0584d6da29",
+			expectParseSuccess:    true,
+			expectValidateSuccess: true,
+		},
+		{
+			caseDesc:              "valid SHA512 with prefix",
+			sha:                   "sha512:a5d575f245588b64bcec78a1bb9d92a66bfb4d68d7de1aea4162ad0b232753860cb764fd0645ada1f5d935163522987359e515e0594068d7bc108f0584d6da29",
+			expectParseSuccess:    true,
+			expectValidateSuccess: true,
+		},
+		{
+			caseDesc:              "invalid SHA prefix",
+			sha:                   "sha257:45c7b11fcbf07dec1694adecd8c5b85770a12a6c8dfdcf2580a2db0c47c31779",
+			expectParseSuccess:    false,
+			expectValidateSuccess: false,
+		},
 		{
 			caseDesc:              "invalid SHA",
 			sha:                   "45c7b11fcbf",
 			expectParseSuccess:    false,
 			expectValidateSuccess: false,
 		},
+		{
+			caseDesc:              "invalid hash alg",
+			sha:                   "md5:d408f34c27cf5930be6394a455f23d40",
+			expectParseSuccess:    false,
+			expectValidateSuccess: false,
+		},
 		{
 			caseDesc:              "valid email",
 			email:                 "cat@foo.com",

cmd/rekor-cli/app/search.go

@@ -54,7 +54,7 @@ func addSearchPFlags(cmd *cobra.Command) error {
 
 	cmd.Flags().Var(NewFlagValue(fileOrURLFlag, ""), "artifact", "path or URL to artifact file")
 
-	cmd.Flags().Var(NewFlagValue(shaFlag, ""), "sha", "the SHA256 or SHA1 sum of the artifact")
+	cmd.Flags().Var(NewFlagValue(shaFlag, ""), "sha", "the SHA512, SHA256 or SHA1 sum of the artifact")
 
 	cmd.Flags().Var(NewFlagValue(emailFlag, ""), "email", "email associated with the public key's subject")
 

openapi.yaml

@@ -238,7 +238,7 @@ paths:
           $ref: '#/responses/BadContent'
         default:
           $ref: '#/responses/InternalServerError'
-  
+
 definitions:
   ProposedEntry:
     type: object
@@ -439,7 +439,7 @@ definitions:
           type: object
           properties:
             data:
-              format: byte       
+              format: byte
 
           format: byte
         verification:
@@ -483,7 +483,7 @@ definitions:
           - "format"
       hash:
         type: string
-        pattern: '^(sha256:)?[0-9a-fA-F]{64}$|^(sha1:)?[0-9a-fA-F]{40}$'
+        pattern: '^(sha512:)?[0-9a-fA-F]{128}$|^(sha256:)?[0-9a-fA-F]{64}$|^(sha1:)?[0-9a-fA-F]{40}$'
       operator:
         type: string
         enum: ['and','or']

pkg/api/index.go

@@ -43,7 +43,7 @@ func SearchIndexHandler(params index.SearchIndexParams) middleware.Responder {
 	var result = NewCollection(queryOperator)
 
 	if params.Query.Hash != "" {
-		// This must be a valid sha256 hash
+		// This must be a valid hash
 		sha := util.PrefixSHA(params.Query.Hash)
 		var resultUUIDs []string
 		if err := redisClient.Do(httpReqCtx, radix.Cmd(&resultUUIDs, "LRANGE", strings.ToLower(sha), "0", "-1")); err != nil {

pkg/util/sha.go

@@ -22,12 +22,20 @@ import (
 // PrefixSHA sets the prefix of a sha hash to match how it is stored based on the length.
 func PrefixSHA(sha string) string {
 	var prefix string
-	if !strings.HasPrefix(sha, "sha256:") && !strings.HasPrefix(sha, "sha1:") {
-		if len(sha) == 40 {
-			prefix = "sha1:"
-		} else {
-			prefix = "sha256:"
-		}
+	var components = strings.Split(sha, ":")
+
+	if len(components) == 2 {
+		return sha
+	}
+
+	switch len(sha) {
+	case 40:
+		prefix = "sha1:"
+	case 64:
+		prefix = "sha256:"
+	case 128:
+		prefix = "sha512:"
 	}
+
 	return fmt.Sprintf("%v%v", prefix, sha)
 }

pkg/util/sha_test.go

@@ -0,0 +1,54 @@
+// Copyright 2022 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"testing"
+)
+
+func TestPrefixSHA(t *testing.T) {
+	var testCases = []struct {
+		input string
+		want  string
+	}{
+		{
+			input: "123",
+			want:  "123",
+		},
+		{
+			input: "sha512:abc",
+			want:  "sha512:abc",
+		},
+		{
+			input: "09b80428c53912d4174162fd5b7c7d485bdcc3ab",
+			want:  "sha1:09b80428c53912d4174162fd5b7c7d485bdcc3ab",
+		},
+		{
+			input: "b9869be95b24001702120dd5dd673a9bd8447446fb57220388d8d0a48c738808",
+			want:  "sha256:b9869be95b24001702120dd5dd673a9bd8447446fb57220388d8d0a48c738808",
+		},
+		{
+			input: "cfd356237e261871e8f92ae6710a75a65a925ae121d94d28533f008bd3e00b5472d261b5d0e1ab4082e3078dd1ad2af57876ed3c1c797c4097dbed870f458408",
+			want:  "sha512:cfd356237e261871e8f92ae6710a75a65a925ae121d94d28533f008bd3e00b5472d261b5d0e1ab4082e3078dd1ad2af57876ed3c1c797c4097dbed870f458408",
+		},
+	}
+
+	for _, tr := range testCases {
+		got := PrefixSHA(tr.input)
+		if got != tr.want {
+			t.Errorf("Got '%s' expected '%s'", got, tr.want)
+		}
+	}
+}

pkg/util/validate.go

@@ -21,6 +21,30 @@ import (
 	validator "github.com/go-playground/validator/v10"
 )
 
+// validateSHA512Value ensures that the supplied string matches the
+// following format: [sha512:]<128 hexadecimal characters>
+// where [sha512:] is optional
+func ValidateSHA512Value(v string) error {
+	var prefix, hash string
+
+	split := strings.SplitN(v, ":", 2)
+	switch len(split) {
+	case 1:
+		hash = split[0]
+	case 2:
+		prefix = split[0]
+		hash = split[1]
+	}
+
+	s := struct {
+		Prefix string `validate:"omitempty,oneof=sha512"`
+		Hash   string `validate:"required,len=128,hexadecimal"`
+	}{prefix, hash}
+
+	validate := validator.New()
+	return validate.Struct(s)
+}
+
 // validateSHA256Value ensures that the supplied string matches the following format:
 // [sha256:]<64 hexadecimal characters>
 // where [sha256:] is optional