diff --git a/include/libpv/crypto.h b/include/libpv/crypto.h
index 8535c7ad..44b253bf 100644
--- a/include/libpv/crypto.h
+++ b/include/libpv/crypto.h
@@ -42,11 +42,14 @@ enum PvCryptoMode {
 	PV_DECRYPT,
 };
 
-/** pv_get_openssl_error:
+/** pv_get_openssl_errors:
  *
- * Returns: (transfer full): String representing the error.
+ * Returns the last OpenSSL error messages.
+ * Caller is responsible to free the returned value.
+ *
+ * Returns: String representing the error.
  */
-const char *pv_get_openssl_error(void);
+char *pv_get_openssl_errors(void);
 
 /**
  * pv_BIO_reset:
diff --git a/libpv/cert.c b/libpv/cert.c
index a7a3bd68..c8bb8cc3 100644
--- a/libpv/cert.c
+++ b/libpv/cert.c
@@ -290,7 +290,9 @@ static char *pv_X509_NAME_oneline(const X509_NAME *name)
 	long len;
 
 	if (X509_NAME_print_ex(key_bio, name, 0, XN_FLAG_RFC2253) == -1) {
-		g_warning(_("Cannot receive X509-NAME from CRL: %s"), pv_get_openssl_error());
+		g_autofree char *openssl_err_msg = pv_get_openssl_errors();
+
+		g_warning(_("Cannot receive X509-NAME from CRL: %s"), openssl_err_msg);
 		return NULL;
 	}
 
diff --git a/libpv/crypto.c b/libpv/crypto.c
index bb5acefb..ae7f1f29 100644
--- a/libpv/crypto.c
+++ b/libpv/crypto.c
@@ -19,18 +19,18 @@
 #include "libpv/glib-helper.h"
 #include "libpv/hash.h"
 
-const char *pv_get_openssl_error(void)
+char *pv_get_openssl_errors(void)
 {
-	const char *ret;
-	BIO *bio;
+	char *ret;
 	char *buf;
+	BIO *bio;
 	long len;
 
 	bio = BIO_new(BIO_s_mem());
 	ERR_print_errors(bio);
 	len = BIO_get_mem_data(bio, &buf);
-	if (len < 0)
-		ret = "Cannot receive OpenSSL error message.";
+	if (len <= 0 || !buf)
+		ret = g_strdup("Cannot receive OpenSSL error message.");
 	else
 		ret = g_strndup(buf, (size_t)len);
 	BIO_free(bio);
@@ -354,7 +354,6 @@ GBytes *pv_hkdf_extract_and_expand(size_t derived_key_len, GBytes *key, GBytes *
 	if (EVP_PKEY_derive(ctx, derived_key, &derived_key_len) != 1) {
 		g_set_error(error, PV_CRYPTO_ERROR, PV_CRYPTO_ERROR_HKDF_FAIL,
 			    "FAILED to derive key via HKDF");
-		printf("%s\n", pv_get_openssl_error());
 		return NULL;
 	}
 
diff --git a/libpv/hash.c b/libpv/hash.c
index 5d58147d..f104960a 100644
--- a/libpv/hash.c
+++ b/libpv/hash.c
@@ -97,8 +97,10 @@ HMAC_CTX *pv_hmac_ctx_new(GBytes *key, const EVP_MD *md, GError **error)
 	key_data = g_bytes_get_data(key, &key_size);
 
 	if (HMAC_Init_ex(ctx, key_data, (int)key_size, md, NULL) != 1) {
+		g_autofree char *openssl_err_msg = pv_get_openssl_errors();
+
 		g_set_error(error, PV_HASH_ERROR, PV_HASH_ERROR_INTERNAL,
-			    "unable to create HMAC context: %s", pv_get_openssl_error());
+			    "unable to create HMAC context: %s", openssl_err_msg);
 		return NULL;
 	}
 	return g_steal_pointer(&ctx);
@@ -110,8 +112,10 @@ int pv_hmac_ctx_update_raw(HMAC_CTX *ctx, const void *buf, size_t size, GError *
 		return 0;
 
 	if (HMAC_Update(ctx, buf, size) != 1) {
+		g_autofree char *openssl_err_msg = pv_get_openssl_errors();
+
 		g_set_error(error, PV_HASH_ERROR, PV_HASH_ERROR_INTERNAL,
-			    "unable to add data to HMAC context: %s", pv_get_openssl_error());
+			    "unable to add data to HMAC context: %s", openssl_err_msg);
 		return -1;
 	}
 	return 0;
@@ -139,8 +143,10 @@ GBytes *pv_hamc_ctx_finalize(HMAC_CTX *ctx, GError **error)
 	hmac = g_malloc0((unsigned int)md_size);
 
 	if (HMAC_Final(ctx, hmac, &hmac_size) != 1) {
+		g_autofree char *openssl_err_msg = pv_get_openssl_errors();
+
 		g_set_error(error, PV_HASH_ERROR, PV_HASH_ERROR_INTERNAL,
-			    "unable to calculate HMAC: %s", pv_get_openssl_error());
+			    "unable to calculate HMAC: %s", openssl_err_msg);
 		return NULL;
 	}
 	return g_bytes_new_take(g_steal_pointer(&hmac), hmac_size);