libkmipclient: Fix integer overflow in kmip_format_hex()

If length is >= 0x80000000 (2 GB), length * 2 wraps around to a small
value, calloc allocates a too small buffer, then the loop writes
length * 2 bytes into it causing a heap buffer overflow.

Fix this by using a size_t for size calculation, and also checking
the length before multiplication (needed on 32 bit platforms).

Assisted-by: IBM Bob:2.0.0
Signed-off-by: Ingo Franzki <ifranzki@linux.ibm.com>
Reviewed-by: Finn Callies <fcallies@linux.ibm.com>
Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>
This commit is contained in:
Ingo Franzki
2026-06-30 12:03:15 +02:00
committed by Jan Höppner
parent d7648875cc
commit c7d0d3c1b9
2 changed files with 6 additions and 3 deletions

View File

@@ -166,13 +166,16 @@ int kmip_parse_hex(const char *str, bool has_prefix, unsigned char **val,
* Format a hex string from the byte array specified in val. The caller must * Format a hex string from the byte array specified in val. The caller must
* free the returned str. * free the returned str.
*/ */
int kmip_format_hex(const unsigned char *val, uint32_t length, bool prefix, int kmip_format_hex(const unsigned char *val, size_t length, bool prefix,
char **str) char **str)
{ {
uint32_t str_len, i; size_t str_len, i;
char tmp[4]; char tmp[4];
char *ret; char *ret;
if (length > (SIZE_MAX - ((prefix ? 2 : 0) + 1)) / 2)
return -EINVAL;
str_len = length * 2 + (prefix ? 2 : 0) + 1; str_len = length * 2 + (prefix ? 2 : 0) + 1;
ret = calloc(1, str_len); ret = calloc(1, str_len);
if (ret == NULL) if (ret == NULL)

View File

@@ -38,7 +38,7 @@ int kmip_parse_decimal_uint(const char *str, uint64_t *val);
int kmip_parse_hex_int(const char *str, int64_t *val); int kmip_parse_hex_int(const char *str, int64_t *val);
int kmip_parse_hex(const char *str, bool has_prefix, unsigned char **val, int kmip_parse_hex(const char *str, bool has_prefix, unsigned char **val,
uint32_t *length); uint32_t *length);
int kmip_format_hex(const unsigned char *val, uint32_t length, bool prefix, int kmip_format_hex(const unsigned char *val, size_t length, bool prefix,
char **str); char **str);
int kmip_parse_bignum(const char *str, bool has_prefix, BIGNUM **bn); int kmip_parse_bignum(const char *str, bool has_prefix, BIGNUM **bn);