crypto: throw proper errors if out enc is UTF-16

Throw `Error`s instead of hard crashing when the `.digest()` output
encoding is UTF-16.

Fixes: https://github.com/nodejs/node/issues/9817
PR-URL: https://github.com/nodejs/node/pull/12752
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: Refael Ackermann <refack@gmail.com>
Reviewed-By: Gibson Fahnestock <gibfahn@gmail.com>

src/node_crypto.cc

@@ -3792,6 +3792,10 @@ void Hmac::HmacDigest(const FunctionCallbackInfo<Value>& args) {
     encoding = ParseEncoding(env->isolate(), args[0], BUFFER);
   }
 
+  if (encoding == UCS2) {
+    return env->ThrowError("hmac.digest() does not support UTF-16");
+  }
+
   unsigned char* md_value = nullptr;
   unsigned int md_len = 0;
 
@@ -3915,6 +3919,10 @@ void Hash::HashDigest(const FunctionCallbackInfo<Value>& args) {
     encoding = ParseEncoding(env->isolate(), args[0], BUFFER);
   }
 
+  if (encoding == UCS2) {
+    return env->ThrowError("hash.digest() does not support UTF-16");
+  }
+
   unsigned char md_value[EVP_MAX_MD_SIZE];
   unsigned int md_len;
 

test/parallel/test-crypto-hash.js

@@ -108,10 +108,12 @@ const h3 = crypto.createHash('sha256');
 h3.digest();
 assert.throws(function() {
   h3.digest();
-},
-              /Digest already called/);
+}, /Digest already called/);
 
 assert.throws(function() {
   h3.update('foo');
-},
-              /Digest already called/);
+}, /Digest already called/);
+
+assert.throws(function() {
+  crypto.createHash('sha256').digest('ucs2');
+}, /^Error: hash\.digest\(\) does not support UTF-16$/);

test/parallel/test-crypto-hmac.js

@@ -377,3 +377,7 @@ for (let i = 0, l = rfc2202_sha1.length; i < l; i++) {
     `Test HMAC-SHA1 : Test case ${i + 1} rfc 2202`
   );
 }
+
+assert.throws(function() {
+  crypto.createHmac('sha256', 'w00t').digest('ucs2');
+}, /^Error: hmac\.digest\(\) does not support UTF-16$/);