From 4899247bf2cf1cae088e0c11a36f09166561875b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 12 Oct 2018 19:19:12 +0200 Subject: [PATCH] Fix undefined behavior in unsigned-to-signed conversion The code assumed that `int x = - (unsigned) u` with 0 <= u < INT_MAX sets `x` to the negative of u, but actually this calculates (UINT_MAX - u) and then converts this value to int, which overflows. Cast to int before applying the unary minus operator to guarantee the desired behavior. --- library/rsa.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library/rsa.c b/library/rsa.c index b401189d2..b9708646f 100644 --- a/library/rsa.c +++ b/library/rsa.c @@ -1578,9 +1578,9 @@ int mbedtls_rsa_rsaes_pkcs1_v15_decrypt( mbedtls_rsa_context *ctx, * - OUTPUT_TOO_LARGE if the padding is good but the decrypted * plaintext does not fit in the output buffer. * - 0 if the padding is correct. */ - ret = - if_int( bad, - MBEDTLS_ERR_RSA_INVALID_PADDING, - if_int( output_too_large, - MBEDTLS_ERR_RSA_OUTPUT_TOO_LARGE, - 0 ) ); + ret = - (int) if_int( bad, - MBEDTLS_ERR_RSA_INVALID_PADDING, + if_int( output_too_large, - MBEDTLS_ERR_RSA_OUTPUT_TOO_LARGE, + 0 ) ); /* If the padding is bad or the plaintext is too large, zero the * data that we're about to copy to the output buffer.