crypto: avoid hang when no algorithm available

Author: richardlau

src/crypto/crypto_util.cc

@@ -62,6 +62,12 @@ int VerifyCallback(int preverify_ok, X509_STORE_CTX* ctx) {
 
 MUST_USE_RESULT CSPRNGResult CSPRNG(void* buffer, size_t length) {
   do {
+#if OPENSSL_VERSION_MAJOR >= 3
+    const uint32_t err = ERR_peek_error();
+    if (err == ERR_PACK(ERR_LIB_RAND, 0, RAND_R_UNABLE_TO_FETCH_DRBG)) {
+      return {false};
+    }
+#endif
     if (1 == RAND_status())
       if (1 == RAND_bytes(static_cast<unsigned char*>(buffer), length))
         return {true};

test/fixtures/openssl3-conf/base_only.cnf

@@ -0,0 +1,12 @@
+nodejs_conf = nodejs_init
+
+[nodejs_init]
+providers = provider_sect
+
+# List of providers to load
+[provider_sect]
+base = base_sect
+
+[base_sect]
+activate = 1
+

test/parallel/test-crypto-no-algorithm.js

@@ -0,0 +1,41 @@
+// Flags: --expose-internals
+'use strict';
+
+const common = require('../common');
+if (!common.hasCrypto)
+  common.skip('missing crypto');
+
+if (!common.hasOpenSSL3)
+  common.skip('this test requires OpenSSL 3.x');
+
+const { internalBinding } = require('node:internal/test/binding');
+const { testFipsCrypto } = internalBinding('crypto');
+const assert = require('node:assert/strict');
+const crypto = require('node:crypto');
+
+{
+  // TODO(richardlau): Decide if `crypto.setFips` should error if the
+  // provider namd "fips" is not available.
+  crypto.setFips(1);
+  crypto.randomBytes(20, testFipsCrypto() ?
+    common.mustSucceed() :
+    common.expectsError((err) => {
+      const expected = /digital envelope routines::unsupported/;
+      assert(err.opensslErrorStack.some((msg) => expected.test(msg)),
+             `did not find ${expected} in ${err.opensslErrorStack}`);
+      return true;
+    })
+  );
+}
+
+{
+  // Startup test. Should not hang.
+  const { path } = require('../common/fixtures');
+  const { spawnSync } = require('node:child_process');
+  const baseConf = path('openssl3-conf', 'base_only.cnf');
+  const cp = spawnSync(process.execPath,
+                       [ `--openssl-config=${baseConf}`, '-p', '"hello"' ],
+                       { encoding: 'utf8' });
+  assert(common.nodeProcessAborted(cp.status, cp.signal),
+         `process did not abort, code:${cp.status} signal:${cp.signal}`);
+}