crypto: fix RSA-PSS default saltLength

src/crypto/crypto_rsa.cc

@@ -79,10 +79,15 @@ EVPKeyCtxPointer RsaKeyGenTraits::Setup(RsaKeyPairGenConfig* params) {
       return EVPKeyCtxPointer();
     }
 
-    if (params->params.saltlen >= 0 &&
+    int saltlen = params->params.saltlen;
+    if (saltlen < 0 && params->params.md != nullptr) {
+      saltlen = EVP_MD_size(params->params.md);
+    }
+
+    if (saltlen >= 0 &&
         EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen(
             ctx.get(),
-            params->params.saltlen) <= 0) {
+            saltlen) <= 0) {
       return EVPKeyCtxPointer();
     }
   }

test/parallel/test-crypto-keygen.js

@@ -391,6 +391,43 @@ const sec1EncExp = (cipher) => getRegExpForPEM('EC PRIVATE KEY', cipher);
   }));
 }
 
+{
+  // RFC 8017, A.2.3.: "For a given hashAlgorithm, the default value of
+  // saltLength is the octet length of the hash value."
+
+  generateKeyPair('rsa-pss', {
+    modulusLength: 512,
+    hashAlgorithm: 'sha512'
+  }, common.mustSucceed((publicKey, privateKey) => {
+    const expectedKeyDetails = {
+      modulusLength: 512,
+      publicExponent: 65537n,
+      hashAlgorithm: 'sha512',
+      mgf1HashAlgorithm: 'sha512',
+      saltLength: 64
+    };
+    assert.deepStrictEqual(publicKey.asymmetricKeyDetails, expectedKeyDetails);
+    assert.deepStrictEqual(privateKey.asymmetricKeyDetails, expectedKeyDetails);
+  }));
+
+  // It is still possible to explicitly set saltLength to 0.
+  generateKeyPair('rsa-pss', {
+    modulusLength: 512,
+    hashAlgorithm: 'sha512',
+    saltLength: 0
+  }, common.mustSucceed((publicKey, privateKey) => {
+    const expectedKeyDetails = {
+      modulusLength: 512,
+      publicExponent: 65537n,
+      hashAlgorithm: 'sha512',
+      mgf1HashAlgorithm: 'sha512',
+      saltLength: 0
+    };
+    assert.deepStrictEqual(publicKey.asymmetricKeyDetails, expectedKeyDetails);
+    assert.deepStrictEqual(privateKey.asymmetricKeyDetails, expectedKeyDetails);
+  }));
+}
+
 {
   const privateKeyEncoding = {
     type: 'pkcs8',