wolfSSL/wolfssl
1
0
Fork 1
mirror of https://github.com/wolfSSL/wolfssl.git synced 2025-07-31 19:24:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Refactor memory BIO

Browse Source 
- use the `WOLFSSL_BUF_MEM` struct to resize the internal memory buffer
- add a `WOLFSSL_BIO_RESIZE_THRESHOLD` define that will be used to determine how often to shrink the internal buffer. This should cut down on the number of free/malloc calls made significantly. This should help with our inefficient 1 byte reads in `loadX509orX509REQFromPemBio`.
- implement `wolfSSL_BUF_MEM_resize` which allows bi-directional buffer size manipulation
This commit is contained in:
Juliusz Sosinowicz
2022-03-16 21:17:40 +01:00
parent ae9b01c5b8
commit 98bc8402db
4 changed files with 154 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
src/bio.c
View File

@@ -36,6 +36,13 @@
#endif #endif
#else #else
/*
* WOLFSSL_BIO_RESIZE_THRESHOLD:
* The amount of data to return before we attempt to resize the internal
* buffers. After we have returned more than this define amount of bytes of
* data, we will resize the buffers to get rid of excess memory.
*/
/* Helper function to decode a base64 input /* Helper function to decode a base64 input
* *
@@ -75,6 +82,9 @@ static int wolfSSL_BIO_BIO_read(WOLFSSL_BIO* bio, void* buf, int len)
return sz; return sz;
} }
#ifndef WOLFSSL_BIO_RESIZE_THRESHOLD
#define WOLFSSL_BIO_RESIZE_THRESHOLD 100
#endif
/* Handles reading from a memory type BIO and advancing the state. /* Handles reading from a memory type BIO and advancing the state.
* *
@@ -97,39 +107,56 @@ static int wolfSSL_BIO_MEMORY_read(WOLFSSL_BIO* bio, void* buf, int len)
sz = wolfSSL_BIO_pending(bio); sz = wolfSSL_BIO_pending(bio);
if (sz > 0) { if (sz > 0) {
const unsigned char* pt = NULL;
int memSz; int memSz;
if (bio->mem_buf == NULL) {
WOLFSSL_MSG("bio->mem_buf is null");
return WOLFSSL_BIO_ERROR;
}
if (sz > len) { if (sz > len) {
sz = len; sz = len;
} }
memSz = wolfSSL_BIO_get_mem_data(bio, (void*)&pt);
if (memSz >= sz && pt != NULL) {
byte* tmp;
XMEMCPY(buf, (void*)pt, sz); memSz = bio->mem_buf->length - bio->rdIdx;
if (memSz - sz > 0) { if (memSz < sz) {
tmp = (byte*)XMALLOC(memSz-sz, bio->heap, DYNAMIC_TYPE_OPENSSL); WOLFSSL_MSG("Not enough memory for reading");
if (tmp == NULL) {
WOLFSSL_MSG("Memory error");
return WOLFSSL_BIO_ERROR; return WOLFSSL_BIO_ERROR;
} }
XMEMCPY(tmp, (void*)(pt + sz), memSz - sz);
/* reset internal bio->mem */ XMEMCPY(buf, bio->mem_buf->data + bio->rdIdx, sz);
XFREE(bio->ptr, bio->heap, DYNAMIC_TYPE_OPENSSL); bio->rdIdx += sz;
bio->ptr = tmp;
bio->num = memSz-sz; if (bio->rdIdx >= bio->wrSz) {
if (bio->mem_buf != NULL) { /* All data read resize down to WOLFSSL_BIO_RESIZE_THRESHOLD */
bio->mem_buf->data = (char*)bio->ptr; if (bio->mem_buf->max > WOLFSSL_BIO_RESIZE_THRESHOLD &&
bio->mem_buf->length = bio->num; wolfSSL_BUF_MEM_resize(bio->mem_buf,
WOLFSSL_BIO_RESIZE_THRESHOLD) == 0) {
WOLFSSL_MSG("wolfSSL_BUF_MEM_resize error");
return WOLFSSL_BIO_ERROR;
} }
bio->wrSz = 0;
bio->rdIdx = 0;
bio->mem_buf->length = 0;
bio->ptr = bio->mem_buf->data;
} }
bio->wrSz -= sz; else if (bio->rdIdx >= WOLFSSL_BIO_RESIZE_THRESHOLD) {
/* Resize the memory so we are not taking up more than necessary.
* memmove reverts internally to memcpy if areas don't overlap */
XMEMMOVE(bio->mem_buf->data, bio->mem_buf->data + bio->rdIdx,
bio->wrSz - bio->rdIdx);
bio->wrSz -= bio->rdIdx;
bio->rdIdx = 0;
/* Resize down to WOLFSSL_BIO_RESIZE_THRESHOLD for fewer
* allocations. */
if (wolfSSL_BUF_MEM_resize(bio->mem_buf,
bio->wrSz > WOLFSSL_BIO_RESIZE_THRESHOLD ? bio->wrSz :
WOLFSSL_BIO_RESIZE_THRESHOLD) == 0) {
WOLFSSL_MSG("wolfSSL_BUF_MEM_resize error");
return WOLFSSL_BIO_ERROR;
} }
else { bio->mem_buf->length = bio->wrSz;
WOLFSSL_MSG("Issue with getting bio mem pointer"); bio->ptr = bio->mem_buf->data;
return 0;
} }
} }
else { else {
@@ -483,53 +510,25 @@ static int wolfSSL_BIO_BIO_write(WOLFSSL_BIO* bio, const void* data,
static int wolfSSL_BIO_MEMORY_write(WOLFSSL_BIO* bio, const void* data, static int wolfSSL_BIO_MEMORY_write(WOLFSSL_BIO* bio, const void* data,
int len) int len)
{ {
int sz;
const unsigned char* buf;
WOLFSSL_ENTER("wolfSSL_BIO_MEMORY_write"); WOLFSSL_ENTER("wolfSSL_BIO_MEMORY_write");
if (bio == NULL || data == NULL) { if (bio == NULL || bio->mem_buf == NULL || data == NULL) {
return BAD_FUNC_ARG; WOLFSSL_MSG("one of input parameters is null");
}
sz = wolfSSL_BIO_pending(bio);
if (sz < 0) {
WOLFSSL_MSG("Error getting memory data");
return sz;
}
if (bio->ptr == NULL) {
bio->ptr = (byte*)XMALLOC(len, bio->heap, DYNAMIC_TYPE_OPENSSL);
if (bio->ptr == NULL) {
WOLFSSL_MSG("Error on malloc");
return WOLFSSL_FAILURE; return WOLFSSL_FAILURE;
} }
bio->num = len;
if (bio->mem_buf != NULL) {
bio->mem_buf->data = (char*)bio->ptr;
bio->mem_buf->length = bio->num;
}
}
/* check if will fit in current buffer size */ if (len == 0)
if (wolfSSL_BIO_get_mem_data(bio, (void*)&buf) < 0) { return WOLFSSL_SUCCESS; /* Return early to make logic simpler */
return WOLFSSL_BIO_ERROR;
} if (wolfSSL_BUF_MEM_grow_ex(bio->mem_buf, bio->wrSz + len, 0)
if (bio->num < sz + len) { == 0) {
bio->ptr = (byte*)XREALLOC(bio->ptr, sz + len, bio->heap, WOLFSSL_MSG("Error growing memory area");
DYNAMIC_TYPE_OPENSSL);
if (bio->ptr == NULL) {
WOLFSSL_MSG("Error on realloc");
return WOLFSSL_FAILURE; return WOLFSSL_FAILURE;
} }
bio->num = sz + len;
if (bio->mem_buf != NULL) {
bio->mem_buf->data = (char*)bio->ptr;
bio->mem_buf->length = bio->num;
}
}
XMEMCPY((byte*)bio->ptr + sz, data, len); XMEMCPY(bio->mem_buf->data + bio->wrSz, data, len);
bio->ptr = bio->mem_buf->data;
bio->num = bio->mem_buf->max;
bio->wrSz += len; bio->wrSz += len;
return len; return len;
@@ -1090,7 +1089,7 @@ size_t wolfSSL_BIO_ctrl_pending(WOLFSSL_BIO *bio)
#endif #endif
if (bio->type == WOLFSSL_BIO_MEMORY) { if (bio->type == WOLFSSL_BIO_MEMORY) {
return bio->wrSz; return bio->wrSz - bio->rdIdx;
} }
/* type BIO_BIO then check paired buffer */ /* type BIO_BIO then check paired buffer */
@@ -1157,7 +1156,7 @@ int wolfSSL_BIO_set_write_buf_size(WOLFSSL_BIO *bio, long size)
{ {
WOLFSSL_ENTER("wolfSSL_BIO_set_write_buf_size"); WOLFSSL_ENTER("wolfSSL_BIO_set_write_buf_size");
if (bio == NULL || bio->type != WOLFSSL_BIO_BIO || size < 0) { if (bio == NULL || bio->type != WOLFSSL_BIO_BIO || (int)size < 0) {
return WOLFSSL_FAILURE; return WOLFSSL_FAILURE;
} }
@@ -1167,27 +1166,32 @@ int wolfSSL_BIO_set_write_buf_size(WOLFSSL_BIO *bio, long size)
return WOLFSSL_FAILURE; return WOLFSSL_FAILURE;
} }
bio->wrSz = (int)size;
if (bio->wrSz < 0) {
WOLFSSL_MSG("Unexpected negative size value");
return WOLFSSL_FAILURE;
}
if (bio->ptr != NULL) { if (bio->ptr != NULL) {
XFREE(bio->ptr, bio->heap, DYNAMIC_TYPE_OPENSSL); XFREE(bio->ptr, bio->heap, DYNAMIC_TYPE_OPENSSL);
} }
bio->ptr = (byte*)XMALLOC(bio->wrSz, bio->heap, DYNAMIC_TYPE_OPENSSL); bio->ptr = (byte*)XMALLOC(size, bio->heap, DYNAMIC_TYPE_OPENSSL);
if (bio->ptr == NULL) { if (bio->ptr == NULL) {
WOLFSSL_MSG("Memory allocation error"); WOLFSSL_MSG("Memory allocation error");
bio->wrSz = 0;
bio->num = 0;
bio->wrIdx = 0;
bio->rdIdx = 0;
if (bio->mem_buf != NULL) {
bio->mem_buf->data = NULL;
bio->mem_buf->length = 0;
bio->mem_buf->max = 0;
}
return WOLFSSL_FAILURE; return WOLFSSL_FAILURE;
} }
bio->num = bio->wrSz; bio->wrSz = (int)size;
bio->num = (int)size;
bio->wrIdx = 0; bio->wrIdx = 0;
bio->rdIdx = 0; bio->rdIdx = 0;
if (bio->mem_buf != NULL) { if (bio->mem_buf != NULL) {
bio->mem_buf->data = (char*)bio->ptr; bio->mem_buf->data = (char*)bio->ptr;
bio->mem_buf->length = bio->num; bio->mem_buf->length = bio->num;
bio->mem_buf->max = bio->num;
} }
return WOLFSSL_SUCCESS; return WOLFSSL_SUCCESS;
@@ -1428,8 +1432,9 @@ int wolfSSL_BIO_reset(WOLFSSL_BIO *bio)
bio->ptr = NULL; bio->ptr = NULL;
bio->num = 0; bio->num = 0;
if (bio->mem_buf != NULL) { if (bio->mem_buf != NULL) {
bio->mem_buf->data = (char*)bio->ptr; bio->mem_buf->data = NULL;
bio->mem_buf->length = bio->num; bio->mem_buf->length = 0;
bio->mem_buf->max = 0;
} }
return 0; return 0;
@@ -1590,12 +1595,13 @@ long wolfSSL_BIO_set_mem_eof_return(WOLFSSL_BIO *bio, int v)
{ {
WOLFSSL_ENTER("wolfSSL_BIO_set_mem_eof_return"); WOLFSSL_ENTER("wolfSSL_BIO_set_mem_eof_return");
if (bio != NULL) { if (bio != NULL && bio->type == WOLFSSL_BIO_MEMORY) {
bio->eof = v; bio->eof = v;
}
return WOLFSSL_SUCCESS; return WOLFSSL_SUCCESS;
} }
else
return WOLFSSL_FAILURE;
}
int wolfSSL_BIO_get_len(WOLFSSL_BIO *bio) int wolfSSL_BIO_get_len(WOLFSSL_BIO *bio)
{ {
@@ -1889,10 +1895,10 @@ int wolfSSL_BIO_get_mem_data(WOLFSSL_BIO* bio, void* p)
} }
if (p) { if (p) {
*(byte**)p = (byte*)mem_bio->ptr; *(byte**)p = (byte*)mem_bio->ptr + mem_bio->rdIdx;
} }
return mem_bio->num; return mem_bio->wrSz - mem_bio->rdIdx;
} }
int wolfSSL_BIO_pending(WOLFSSL_BIO* bio) int wolfSSL_BIO_pending(WOLFSSL_BIO* bio)
@@ -2476,14 +2482,12 @@ int wolfSSL_BIO_flush(WOLFSSL_BIO* bio)
bio->eof = WOLFSSL_BIO_ERROR; /* Return value for empty buffer */ bio->eof = WOLFSSL_BIO_ERROR; /* Return value for empty buffer */
if (method->type == WOLFSSL_BIO_MEMORY || if (method->type == WOLFSSL_BIO_MEMORY ||
method->type == WOLFSSL_BIO_BIO) { method->type == WOLFSSL_BIO_BIO) {
bio->mem_buf =(WOLFSSL_BUF_MEM*)XMALLOC(sizeof(WOLFSSL_BUF_MEM), bio->mem_buf = wolfSSL_BUF_MEM_new();
0, DYNAMIC_TYPE_OPENSSL);
if (bio->mem_buf == NULL) { if (bio->mem_buf == NULL) {
WOLFSSL_MSG("Memory error"); WOLFSSL_MSG("Memory error");
wolfSSL_BIO_free(bio); wolfSSL_BIO_free(bio);
return NULL; return NULL;
} }
bio->mem_buf->data = (char*)bio->ptr;
} }
if (method->type == WOLFSSL_BIO_MD) { if (method->type == WOLFSSL_BIO_MD) {
@@ -2532,17 +2536,15 @@ int wolfSSL_BIO_flush(WOLFSSL_BIO* bio)
/* The length of the string including terminating null. */ /* The length of the string including terminating null. */
len = (int)XSTRLEN((const char*)buf) + 1; len = (int)XSTRLEN((const char*)buf) + 1;
} }
bio->num = bio->wrSz = len;
bio->ptr = (byte*)XMALLOC(len, 0, DYNAMIC_TYPE_OPENSSL); if (wolfSSL_BUF_MEM_resize(bio->mem_buf, len) == 0) {
if (bio->ptr == NULL) {
wolfSSL_BIO_free(bio); wolfSSL_BIO_free(bio);
return NULL; return NULL;
} }
if (bio->mem_buf != NULL) {
bio->mem_buf->data = (char*)bio->ptr;
bio->mem_buf->length = bio->num;
}
bio->num = bio->mem_buf->max;
bio->wrSz = len;
bio->ptr = bio->mem_buf->data;
XMEMCPY(bio->ptr, buf, len); XMEMCPY(bio->ptr, buf, len);
return bio; return bio;

57
src/ssl.c
View File

@@ -58506,12 +58506,14 @@ WOLFSSL_BUF_MEM* wolfSSL_BUF_MEM_new(void)
return buf; return buf;
} }
/* non-compat API returns length of buffer on success */
/* returns length of buffer on success */ int wolfSSL_BUF_MEM_grow_ex(WOLFSSL_BUF_MEM* buf, size_t len,
int wolfSSL_BUF_MEM_grow(WOLFSSL_BUF_MEM* buf, size_t len) char zeroFill)
{ {
int len_int = (int)len; int len_int = (int)len;
int mx; int mx;
char* tmp;
/* verify provided arguments */ /* verify provided arguments */
if (buf == NULL || len_int < 0) { if (buf == NULL || len_int < 0) {
@@ -58526,7 +58528,7 @@ int wolfSSL_BUF_MEM_grow(WOLFSSL_BUF_MEM* buf, size_t len)
/* check to see if fits in max buffer */ /* check to see if fits in max buffer */
if (buf->max >= len) { if (buf->max >= len) {
if (buf->data != NULL) { if (buf->data != NULL && zeroFill) {
XMEMSET(&buf->data[buf->length], 0, len - buf->length); XMEMSET(&buf->data[buf->length], 0, len - buf->length);
} }
buf->length = len; buf->length = len;
@@ -58537,23 +58539,64 @@ int wolfSSL_BUF_MEM_grow(WOLFSSL_BUF_MEM* buf, size_t len)
mx = (len_int + 3) / 3 * 4; mx = (len_int + 3) / 3 * 4;
/* use realloc */ /* use realloc */
buf->data = (char*)XREALLOC(buf->data, mx, NULL, DYNAMIC_TYPE_TMP_BUFFER); tmp = (char*)XREALLOC(buf->data, mx, NULL, DYNAMIC_TYPE_OPENSSL);
if (buf->data == NULL) { if (tmp == NULL) {
return 0; /* ERR_R_MALLOC_FAILURE; */ return 0; /* ERR_R_MALLOC_FAILURE; */
} }
buf->data = tmp;
buf->max = mx; buf->max = mx;
if (zeroFill)
XMEMSET(&buf->data[buf->length], 0, len - buf->length); XMEMSET(&buf->data[buf->length], 0, len - buf->length);
buf->length = len; buf->length = len;
return len_int; return len_int;
}
/* returns length of buffer on success */
int wolfSSL_BUF_MEM_grow(WOLFSSL_BUF_MEM* buf, size_t len)
{
return wolfSSL_BUF_MEM_grow_ex(buf, len, 1);
}
/* non-compat API returns length of buffer on success */
int wolfSSL_BUF_MEM_resize(WOLFSSL_BUF_MEM* buf, size_t len)
{
char* tmp;
int mx;
/* verify provided arguments */
if (buf == NULL || len == 0) {
return 0; /* BAD_FUNC_ARG; */
}
if (len == buf->length)
return (int)len;
if (len > buf->length)
return wolfSSL_BUF_MEM_grow_ex(buf, len, 0);
/* expand size, to handle growth */
mx = (len + 3) / 3 * 4;
/* We want to shrink the internal buffer */
tmp = (char*)XREALLOC(buf->data, mx, NULL, DYNAMIC_TYPE_OPENSSL);
if (tmp == NULL)
return 0;
buf->data = tmp;
buf->length = len;
buf->max = mx;
return (int)len;
} }
void wolfSSL_BUF_MEM_free(WOLFSSL_BUF_MEM* buf) void wolfSSL_BUF_MEM_free(WOLFSSL_BUF_MEM* buf)
{ {
if (buf) { if (buf) {
if (buf->data) { if (buf->data) {
XFREE(buf->data, NULL, DYNAMIC_TYPE_TMP_BUFFER); XFREE(buf->data, NULL, DYNAMIC_TYPE_OPENSSL);
buf->data = NULL; buf->data = NULL;
} }
buf->max = 0; buf->max = 0;

4
tests/api.c
View File

@@ -35341,8 +35341,11 @@ static void test_wolfSSL_PEM_read_bio(void)
AssertNull(x509 = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL)); AssertNull(x509 = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL));
AssertNotNull(bio = BIO_new_mem_buf((void*)buff, bytes)); AssertNotNull(bio = BIO_new_mem_buf((void*)buff, bytes));
AssertIntEQ(BIO_set_mem_eof_return(bio, -0xDEAD), 1);
AssertNotNull(x509 = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL)); AssertNotNull(x509 = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL));
AssertIntEQ((int)BIO_set_fd(bio, 0, BIO_CLOSE), 1); AssertIntEQ((int)BIO_set_fd(bio, 0, BIO_CLOSE), 1);
/* BIO should return the set EOF value */
AssertIntEQ(BIO_read(bio, buff, sizeof(buff)), -0xDEAD);
AssertIntEQ(BIO_set_close(bio, BIO_NOCLOSE), 1); AssertIntEQ(BIO_set_close(bio, BIO_NOCLOSE), 1);
AssertIntEQ(BIO_set_close(NULL, BIO_NOCLOSE), 1); AssertIntEQ(BIO_set_close(NULL, BIO_NOCLOSE), 1);
AssertIntEQ(SSL_SUCCESS, BIO_get_mem_ptr(bio, &buf)); AssertIntEQ(SSL_SUCCESS, BIO_get_mem_ptr(bio, &buf));
@@ -35542,6 +35545,7 @@ static void test_wolfSSL_BIO(void)
AssertNotNull(f_bio1 = BIO_new(BIO_s_file())); AssertNotNull(f_bio1 = BIO_new(BIO_s_file()));
AssertNotNull(f_bio2 = BIO_new(BIO_s_file())); AssertNotNull(f_bio2 = BIO_new(BIO_s_file()));
/* Failure due to wrong BIO type */
AssertIntEQ((int)BIO_set_mem_eof_return(f_bio1, -1), 0); AssertIntEQ((int)BIO_set_mem_eof_return(f_bio1, -1), 0);
AssertIntEQ((int)BIO_set_mem_eof_return(NULL, -1), 0); AssertIntEQ((int)BIO_set_mem_eof_return(NULL, -1), 0);

3
wolfssl/openssl/buffer.h
View File

@@ -33,6 +33,9 @@
WOLFSSL_API WOLFSSL_BUF_MEM* wolfSSL_BUF_MEM_new(void); WOLFSSL_API WOLFSSL_BUF_MEM* wolfSSL_BUF_MEM_new(void);
WOLFSSL_API int wolfSSL_BUF_MEM_grow(WOLFSSL_BUF_MEM* buf, size_t len); WOLFSSL_API int wolfSSL_BUF_MEM_grow(WOLFSSL_BUF_MEM* buf, size_t len);
WOLFSSL_API int wolfSSL_BUF_MEM_grow_ex(WOLFSSL_BUF_MEM* buf, size_t len,
char zeroFill);
WOLFSSL_API int wolfSSL_BUF_MEM_resize(WOLFSSL_BUF_MEM* buf, size_t len);
WOLFSSL_API void wolfSSL_BUF_MEM_free(WOLFSSL_BUF_MEM* buf); WOLFSSL_API void wolfSSL_BUF_MEM_free(WOLFSSL_BUF_MEM* buf);
WOLFSSL_API size_t wolfSSL_strlcpy(char *dst, const char *src, size_t dstSize); WOLFSSL_API size_t wolfSSL_strlcpy(char *dst, const char *src, size_t dstSize);
WOLFSSL_API size_t wolfSSL_strlcat(char *dst, const char *src, size_t dstSize); WOLFSSL_API size_t wolfSSL_strlcat(char *dst, const char *src, size_t dstSize);