crypto: fix default MGF1 hash for OpenSSL 3

Refs: #39999

PR-URL: #40031
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Filip Skokan <panva.ip@gmail.com>

Author: tniessen

src/crypto/crypto_rsa.cc

@@ -63,10 +63,19 @@ EVPKeyCtxPointer RsaKeyGenTraits::Setup(RsaKeyPairGenConfig* params) {
       return EVPKeyCtxPointer();
     }
 
-    if (params->params.mgf1_md != nullptr &&
+    // TODO(tniessen): This appears to only be necessary in OpenSSL 3, while
+    // OpenSSL 1.1.1 behaves as recommended by RFC 8017 and defaults the MGF1
+    // hash algorithm to the RSA-PSS hashAlgorithm. Remove this code if the
+    // behavior of OpenSSL 3 changes.
+    const EVP_MD* mgf1_md = params->params.mgf1_md;
+    if (mgf1_md == nullptr && params->params.md != nullptr) {
+      mgf1_md = params->params.md;
+    }
+
+    if (mgf1_md != nullptr &&
         EVP_PKEY_CTX_set_rsa_pss_keygen_mgf1_md(
             ctx.get(),
-            params->params.mgf1_md) <= 0) {
+            mgf1_md) <= 0) {
       return EVPKeyCtxPointer();
     }
 

test/parallel/test-crypto-keygen.js

@@ -369,6 +369,28 @@ const sec1EncExp = (cipher) => getRegExpForPEM('EC PRIVATE KEY', cipher);
   }));
 }
 
+{
+  // RFC 8017, 9.1.: "Assuming that the mask generation function is based on a
+  // hash function, it is RECOMMENDED that the hash function be the same as the
+  // one that is applied to the message."
+
+  generateKeyPair('rsa-pss', {
+    modulusLength: 512,
+    hashAlgorithm: 'sha256',
+    saltLength: 16
+  }, common.mustSucceed((publicKey, privateKey) => {
+    const expectedKeyDetails = {
+      modulusLength: 512,
+      publicExponent: 65537n,
+      hashAlgorithm: 'sha256',
+      mgf1HashAlgorithm: 'sha256',
+      saltLength: 16
+    };
+    assert.deepStrictEqual(publicKey.asymmetricKeyDetails, expectedKeyDetails);
+    assert.deepStrictEqual(privateKey.asymmetricKeyDetails, expectedKeyDetails);
+  }));
+}
+
 {
   const privateKeyEncoding = {
     type: 'pkcs8',