vnc: Clean up vncws_send_handshake_response()

Use appropriate types, drop superfluous casts, use sizeof, don't
exploit that this particular call of gnutls_fingerprint() doesn't
change its last argument.

Signed-off-by: Markus Armbruster <armbru@redhat.com>
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

diff --git a/ui/vnc-ws.c b/ui/vnc-ws.c
index 9ccdc1971cd562f8f507142f23d4eaebaad5051d..3e3020916ce70d7102b4bfb81c52a34e864a1388 100644 (file)
--- a/ui/vnc-ws.c
+++ b/ui/vnc-ws.c
@@ -120,10 +120,11 @@ static char *vncws_extract_handshake_entry(const char *handshake,
 static void vncws_send_handshake_response(VncState *vs, const char* key)
 {
     char combined_key[WS_CLIENT_KEY_LEN + WS_GUID_LEN + 1];
-    char hash[SHA1_DIGEST_LEN];
-    size_t hash_size = SHA1_DIGEST_LEN;
+    unsigned char hash[SHA1_DIGEST_LEN];
+    size_t hash_size = sizeof(hash);
     char *accept = NULL, *response = NULL;
     gnutls_datum_t in;
+    int ret;
 
     g_strlcpy(combined_key, key, WS_CLIENT_KEY_LEN + 1);
     g_strlcat(combined_key, WS_GUID, WS_CLIENT_KEY_LEN + WS_GUID_LEN + 1);
@@ -131,9 +132,9 @@ static void vncws_send_handshake_response(VncState *vs, const char* key)
     /* hash and encode it */
     in.data = (void *)combined_key;
     in.size = WS_CLIENT_KEY_LEN + WS_GUID_LEN;
-    if (gnutls_fingerprint(GNUTLS_DIG_SHA1, &in, hash, &hash_size)
-            == GNUTLS_E_SUCCESS) {
-        accept = g_base64_encode((guchar *)hash, SHA1_DIGEST_LEN);
+    ret = gnutls_fingerprint(GNUTLS_DIG_SHA1, &in, hash, &hash_size);
+    if (ret == GNUTLS_E_SUCCESS && hash_size <= SHA1_DIGEST_LEN) {
+        accept = g_base64_encode(hash, hash_size);
     }
     if (accept == NULL) {
         VNC_DEBUG("Hashing Websocket combined key failed\n");