crypto: fix checkPrime crash with large buffers

Fixes: #56512

Author: santigimeno

src/crypto/crypto_random.cc

@@ -176,6 +176,12 @@ Maybe<void> CheckPrimeTraits::AdditionalConfig(
   ArrayBufferOrViewContents<unsigned char> candidate(args[offset]);
 
   params->candidate = BignumPointer(candidate.data(), candidate.size());
+  if (params->candidate.get() == nullptr) {
+    ThrowCryptoError(Environment::GetCurrent(args),
+                     ERR_get_error(),
+                     "BignumPointer");
+    return Nothing<void>();
+  }
 
   CHECK(args[offset + 1]->IsInt32());  // Checks
   params->checks = args[offset + 1].As<Int32>()->Value();

test/parallel/test-crypto-prime.js

@@ -12,6 +12,7 @@ const {
   generatePrimeSync,
   checkPrime,
   checkPrimeSync,
+  randomBytes,
 } = require('crypto');
 
 const { Worker } = require('worker_threads');
@@ -254,6 +255,18 @@ for (const checks of [-(2 ** 31), -1, 2 ** 31, 2 ** 32 - 1, 2 ** 32, 2 ** 50]) {
   });
 }
 
+{
+  const bytes = randomBytes(67108864);
+  assert.throws(() => checkPrime(bytes, common.mustNotCall()), {
+    code: 'ERR_OSSL_BN_BIGNUM_TOO_LONG',
+    message: /bignum too long/
+  });
+  assert.throws(() => checkPrimeSync(bytes), {
+    code: 'ERR_OSSL_BN_BIGNUM_TOO_LONG',
+    message: /bignum too long/
+  });
+}
+
 assert(!checkPrimeSync(Buffer.from([0x1])));
 assert(checkPrimeSync(Buffer.from([0x2])));
 assert(checkPrimeSync(Buffer.from([0x3])));