Fix the RSA OID used for signing PKCS#7/SMIME

The current implementation computes the algorithm identifier used
in the `digest_encryption_algorithm` PKCS#7 field
(or `SignatureAlgorithmIdentifier` in S/MIME) based on both the
algorithm used to sign (e.g. RSA) and the digest algorithm (e.g. SHA512).

This is correct for ECDSA signatures, where the OIDs used include the
digest algorithm (e.g: ecdsa-with-SHA512). However, due to historical
reasons, when signing with RSA the OID specified should be the one
corresponding to just RSA ("1.2.840.113549.1.1.1" rsaEncryption),
rather than OIDs which also include the digest algorithm (such as
"1.2.840.113549.1.1.13", sha512WithRSAEncryption).

This means that the logic to compute the algorithm identifier is the
same except when signing with RSA, in which case the OID will always
be `rsaEncryption`. This is consistent with the OpenSSL implementation,
and the RFCs that define PKCS#7 and S/MIME.

See RFC 3851 (section 2.2), and RFC 3370 (section 3.2) for more details.

src/rust/src/pkcs7.rs

@@ -205,7 +205,7 @@ fn sign_and_serialize<'p>(
             },
             digest_algorithm: digest_alg,
             authenticated_attributes: authenticated_attrs,
-            digest_encryption_algorithm: x509::sign::compute_signature_algorithm(
+            digest_encryption_algorithm: compute_pkcs7_signature_algorithm(
                 py,
                 py_private_key,
                 py_hash_alg,
@@ -262,6 +262,26 @@ fn sign_and_serialize<'p>(
     }
 }
 
+fn compute_pkcs7_signature_algorithm<'p>(
+    py: pyo3::Python<'p>,
+    private_key: &'p pyo3::PyAny,
+    hash_algorithm: &'p pyo3::PyAny,
+    rsa_padding: &'p pyo3::PyAny,
+) -> pyo3::PyResult<common::AlgorithmIdentifier<'static>> {
+    let key_type = x509::sign::identify_key_type(py, private_key)?;
+    let has_pss_padding = !rsa_padding.is_none() && rsa_padding.is_instance(types::PSS.get(py)?)?;
+    // For RSA signatures (with no PSS padding), the OID is always the same no matter the
+    // digest algorithm. See RFC 3370 (section 3.2).
+    if key_type == x509::sign::KeyType::Rsa && !has_pss_padding {
+        Ok(common::AlgorithmIdentifier {
+            oid: asn1::DefinedByMarker::marker(),
+            params: common::AlgorithmParameters::Rsa(None),
+        })
+    } else {
+        x509::sign::compute_signature_algorithm(py, private_key, hash_algorithm, rsa_padding)
+    }
+}
+
 fn smime_canonicalize(data: &[u8], text_mode: bool) -> (Cow<'_, [u8]>, Cow<'_, [u8]>) {
     let mut new_data_with_header = vec![];
     let mut new_data_without_header = vec![];

src/rust/src/x509/sign.rs

@@ -48,7 +48,10 @@ enum HashType {
     Sha3_512,
 }
 
-fn identify_key_type(py: pyo3::Python<'_>, private_key: &pyo3::PyAny) -> pyo3::PyResult<KeyType> {
+pub(crate) fn identify_key_type(
+    py: pyo3::Python<'_>,
+    private_key: &pyo3::PyAny,
+) -> pyo3::PyResult<KeyType> {
     if private_key.is_instance(types::RSA_PRIVATE_KEY.get(py)?)? {
         Ok(KeyType::Rsa)
     } else if private_key.is_instance(types::DSA_PRIVATE_KEY.get(py)?)? {