add key setup flag for malicious or misbehaving handshake messages with new memory system

cyassl/internal.h

@@ -1004,6 +1004,7 @@ typedef struct Ciphers {
 #ifdef BUILD_RABBIT
     Rabbit* rabbit;
 #endif
+    byte    setup;       /* have we set it up flag for detection */
 } Ciphers;
 
 

src/internal.c

@@ -458,6 +458,8 @@ void InitCiphers(CYASSL* ssl)
     ssl->encrypt.rabbit = NULL;
     ssl->decrypt.rabbit = NULL;
 #endif
+    ssl->encrypt.setup = 0;
+    ssl->decrypt.setup = 0;
 }
 
 
@@ -2657,8 +2659,13 @@ static INLINE word32 GetSEQIncrement(CYASSL* ssl, int verify)
 }
 
 
-static INLINE void Encrypt(CYASSL* ssl, byte* out, const byte* input, word32 sz)
+static INLINE int Encrypt(CYASSL* ssl, byte* out, const byte* input, word32 sz)
 {
+    if (ssl->encrypt.setup == 0) {
+        CYASSL_MSG("Encrypt ciphers not setup");
+        return ENCRYPT_ERROR;
+    }
+
     switch (ssl->specs.bulk_cipher_algorithm) {
         #ifdef BUILD_ARC4
             case rc4:
@@ -2730,13 +2737,21 @@ static INLINE void Encrypt(CYASSL* ssl, byte* out, const byte* input, word32 sz)
 
             default:
                 CYASSL_MSG("CyaSSL Encrypt programming error");
+                return ENCRYPT_ERROR;
     }
+
+    return 0;
 }
 
 
 static INLINE int Decrypt(CYASSL* ssl, byte* plain, const byte* input,
                            word32 sz)
 {
+    if (ssl->decrypt.setup == 0) {
+        CYASSL_MSG("Decrypt ciphers not setup");
+        return DECRYPT_ERROR;
+    }
+
     switch (ssl->specs.bulk_cipher_algorithm) {
         #ifdef BUILD_ARC4
             case rc4:
@@ -2800,6 +2815,7 @@ static INLINE int Decrypt(CYASSL* ssl, byte* plain, const byte* input,
 
             default:
                 CYASSL_MSG("CyaSSL Decrypt programming error");
+                return DECRYPT_ERROR;
     }
     return 0;
 }
@@ -3483,6 +3499,7 @@ static int BuildMessage(CYASSL* ssl, byte* output, const byte* input, int inSz,
     word32 headerSz = RECORD_HEADER_SZ;
     word16 size;
     byte               iv[AES_BLOCK_SIZE];                  /* max size */
+    int ret  = 0;
 
 #ifdef CYASSL_DTLS
     if (ssl->options.dtls) {
@@ -3526,7 +3543,6 @@ static int BuildMessage(CYASSL* ssl, byte* output, const byte* input, int inSz,
     if (type == handshake) {
 #ifdef CYASSL_DTLS
         if (ssl->options.dtls) {
-            int ret;
             if ((ret = DtlsPoolSave(ssl, output, headerSz+inSz)) != 0)
                 return ret;
         }
@@ -3542,7 +3558,8 @@ static int BuildMessage(CYASSL* ssl, byte* output, const byte* input, int inSz,
         for (i = 0; i <= pad; i++)
             output[idx++] = (byte)pad; /* pad byte gets pad value too */
 
-    Encrypt(ssl, output + headerSz, output + headerSz, size);
+    if ( (ret = Encrypt(ssl, output + headerSz, output + headerSz, size)) != 0)
+        return ret;
 
     return sz;
 }

src/keys.c

@@ -937,6 +937,8 @@ static int SetKeys(Ciphers* enc, Ciphers* dec, Keys* keys, CipherSpecs* specs,
             Arc4SetKey(enc->arc4, keys->server_write_key, sz);
             Arc4SetKey(dec->arc4, keys->client_write_key, sz);
         }
+        enc->setup = 1;
+        dec->setup = 1;
     }
 #endif
     
@@ -960,6 +962,8 @@ static int SetKeys(Ciphers* enc, Ciphers* dec, Keys* keys, CipherSpecs* specs,
             Hc128_SetKey(dec->hc128, keys->client_write_key,
                                          keys->client_write_IV);
         }
+        enc->setup = 1;
+        dec->setup = 1;
     }
 #endif
     
@@ -983,6 +987,8 @@ static int SetKeys(Ciphers* enc, Ciphers* dec, Keys* keys, CipherSpecs* specs,
             RabbitSetKey(dec->rabbit, keys->client_write_key,
                                            keys->client_write_IV);
         }
+        enc->setup = 1;
+        dec->setup = 1;
     }
 #endif
     
@@ -1006,6 +1012,8 @@ static int SetKeys(Ciphers* enc, Ciphers* dec, Keys* keys, CipherSpecs* specs,
             Des3_SetKey(dec->des3, keys->client_write_key,
                 keys->client_write_IV, DES_DECRYPTION);
         }
+        enc->setup = 1;
+        dec->setup = 1;
     }
 #endif
 
@@ -1033,6 +1041,8 @@ static int SetKeys(Ciphers* enc, Ciphers* dec, Keys* keys, CipherSpecs* specs,
                       specs->key_size, keys->client_write_IV,
                       AES_DECRYPTION);
         }
+        enc->setup = 1;
+        dec->setup = 1;
     }
 #endif
 
@@ -1062,6 +1072,8 @@ static int SetKeys(Ciphers* enc, Ciphers* dec, Keys* keys, CipherSpecs* specs,
             AesGcmSetKey(dec->aes, keys->client_write_key, specs->key_size,
                         keys->client_write_IV);
         }
+        enc->setup = 1;
+        dec->setup = 1;
     }
 #endif