libkmipclient: Fix integer overflow in kmip_format_hex()

If length is >= 0x80000000 (2 GB), length * 2 wraps around to a small
value, calloc allocates a too small buffer, then the loop writes
length * 2 bytes into it causing a heap buffer overflow.

Fix this by using a size_t for size calculation, and also checking
the length before multiplication (needed on 32 bit platforms).

Assisted-by: IBM Bob:2.0.0
Signed-off-by: Ingo Franzki <ifranzki@linux.ibm.com>
Reviewed-by: Finn Callies <fcallies@linux.ibm.com>
Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>
This commit is contained in:
Ingo Franzki
2026-06-30 12:03:15 +02:00
committed by Jan Höppner
parent d7648875cc
commit c7d0d3c1b9
2 changed files with 6 additions and 3 deletions

View File

@@ -166,13 +166,16 @@ int kmip_parse_hex(const char *str, bool has_prefix, unsigned char **val,
* Format a hex string from the byte array specified in val. The caller must
* free the returned str.
*/
int kmip_format_hex(const unsigned char *val, uint32_t length, bool prefix,
int kmip_format_hex(const unsigned char *val, size_t length, bool prefix,
char **str)
{
uint32_t str_len, i;
size_t str_len, i;
char tmp[4];
char *ret;
if (length > (SIZE_MAX - ((prefix ? 2 : 0) + 1)) / 2)
return -EINVAL;
str_len = length * 2 + (prefix ? 2 : 0) + 1;
ret = calloc(1, str_len);
if (ret == NULL)

View File

@@ -38,7 +38,7 @@ int kmip_parse_decimal_uint(const char *str, uint64_t *val);
int kmip_parse_hex_int(const char *str, int64_t *val);
int kmip_parse_hex(const char *str, bool has_prefix, unsigned char **val,
uint32_t *length);
int kmip_format_hex(const unsigned char *val, uint32_t length, bool prefix,
int kmip_format_hex(const unsigned char *val, size_t length, bool prefix,
char **str);
int kmip_parse_bignum(const char *str, bool has_prefix, BIGNUM **bn);