libekmfweb: Fix integer overflow errors in base64 encode/decode functions

On large data sizes the calculation may cause an overflow, if done with
int type. Do the calculation with size_t instead to avoid an overflow.

Assisted-by: IBM Bob:2.0.0
Signed-off-by: Ingo Franzki <ifranzki@linux.ibm.com>
Reviewed-by: Finn Callies <fcallies@linux.ibm.com>
Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>
This commit is contained in:
Ingo Franzki
2026-06-30 15:58:36 +02:00
committed by Jan Höppner
parent dfaa1791b7
commit 35cc35894b
2 changed files with 4 additions and 3 deletions

View File

@@ -4548,8 +4548,9 @@ out:
*/
static char *_ekmf_base64_encode(const unsigned char *data, size_t data_size)
{
int outlen, len;
size_t outlen;
char *out;
int len;
outlen = (data_size / 3) * 4;
if (data_size % 3 > 0)
@@ -4560,7 +4561,7 @@ static char *_ekmf_base64_encode(const unsigned char *data, size_t data_size)
return NULL;
len = EVP_EncodeBlock((unsigned char *)out, data, data_size);
if (len != outlen) {
if (len < 0 || (size_t)len != outlen) {
free(out);
return NULL;
}

View File

@@ -106,7 +106,7 @@ int decode_base64url(unsigned char *output, size_t *outlen,
len = EVP_DecodeBlock((unsigned char *)padded_output,
(unsigned char *)padded_input, padded_inlen);
if (len != (int)padded_inlen * 3 / 4) {
if ((size_t)len != padded_inlen * 3 / 4) {
rc = -EIO;
goto out;
}