test: add coverage for invalid RSA-PSS digests

PR-URL: #44271
Reviewed-By: Richard Lau <rlau@redhat.com>
Reviewed-By: Filip Skokan <panva.ip@gmail.com>
Reviewed-By: Darshan Sen <raisinten@gmail.com>
Reviewed-By: Luigi Pinca <luigipinca@gmail.com>
Reviewed-By: Anna Henningsen <anna@addaleax.net>

test/parallel/test-crypto-keygen.js

@@ -1542,6 +1542,24 @@ const sec1EncExp = (cipher) => getRegExpForPEM('EC PRIVATE KEY', cipher);
       }
     );
   }
+
+  assert.throws(() => generateKeyPair('rsa-pss', {
+    modulusLength: 512,
+    hashAlgorithm: 'sha2',
+  }, common.mustNotCall()), {
+    name: 'TypeError',
+    code: 'ERR_CRYPTO_INVALID_DIGEST',
+    message: 'md specifies an invalid digest'
+  });
+
+  assert.throws(() => generateKeyPair('rsa-pss', {
+    modulusLength: 512,
+    mgf1HashAlgorithm: 'sha2',
+  }, common.mustNotCall()), {
+    name: 'TypeError',
+    code: 'ERR_CRYPTO_INVALID_DIGEST',
+    message: 'mgf1_md specifies an invalid digest'
+  });
 }
 
 // Passing an empty passphrase string should not cause OpenSSL's default