Fix RSA signature verification issue

Both verifyKeySignature() and verifyCertificateVerify() has a bug
when handling RSA signature as it looks at the signature algorithm
of the certificate to determine whether to verify with RSA PKCSv1.5.
This will cause issues if the certificate's issuing CA uses something
other than RSA (e.g. ECDSA) to sign the certificate.

Since DTLS v1.2 does not support RSA-PSS [1], we can just use RSA
PKCSv1.5 verification directly if the public key of the certificate
is RSA.

[1] https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml

Author: hoihochan

crypto.go

@@ -89,13 +89,11 @@ func verifyKeySignature(message, remoteKeySignature []byte, hashAlgorithm hash.A
 		}
 		return nil
 	case *rsa.PublicKey:
-		switch certificate.SignatureAlgorithm {
-		case x509.SHA1WithRSA, x509.SHA256WithRSA, x509.SHA384WithRSA, x509.SHA512WithRSA:
-			hashed := hashAlgorithm.Digest(message)
-			return rsa.VerifyPKCS1v15(p, hashAlgorithm.CryptoHash(), hashed, remoteKeySignature)
-		default:
-			return errKeySignatureVerifyUnimplemented
+		hashed := hashAlgorithm.Digest(message)
+		if rsa.VerifyPKCS1v15(p, hashAlgorithm.CryptoHash(), hashed, remoteKeySignature) != nil {
+			return errKeySignatureMismatch
 		}
+		return nil
 	}
 
 	return errKeySignatureVerifyUnimplemented
@@ -158,13 +156,11 @@ func verifyCertificateVerify(handshakeBodies []byte, hashAlgorithm hash.Algorith
 		}
 		return nil
 	case *rsa.PublicKey:
-		switch certificate.SignatureAlgorithm {
-		case x509.SHA1WithRSA, x509.SHA256WithRSA, x509.SHA384WithRSA, x509.SHA512WithRSA:
-			hash := hashAlgorithm.Digest(handshakeBodies)
-			return rsa.VerifyPKCS1v15(p, hashAlgorithm.CryptoHash(), hash, remoteKeySignature)
-		default:
-			return errKeySignatureVerifyUnimplemented
+		hash := hashAlgorithm.Digest(handshakeBodies)
+		if rsa.VerifyPKCS1v15(p, hashAlgorithm.CryptoHash(), hash, remoteKeySignature) != nil {
+			return errKeySignatureMismatch
 		}
+		return nil
 	}
 
 	return errKeySignatureVerifyUnimplemented