Fix data loss in unsigned int cast in PK

This patch introduces some additional checks in the PK module for 64-bit
systems only. The problem is that the API functions in the PK
abstraction accept a size_t value for the hashlen, while the RSA module
accepts an unsigned int for the hashlen. Instead of silently casting
size_t to unsigned int, this change checks whether the hashlen overflows
an unsigned int and returns an error.

ChangeLog

@@ -2,6 +2,13 @@ mbed TLS ChangeLog (Sorted per branch, date)
 
 = mbed TLS 2.1.x branch released xxxx-xx-xx
 
+Security
+    * Add checks to prevent signature forgeries for very large messages while
+      using RSA through the PK module in 64-bit systems. The issue was caused by
+      some data loss when casting a size_t to an unsigned int value in the
+      functions rsa_verify_wrap(), rsa_sign_wrap(), rsa_alt_sign_wrap() and
+      mbedtls_pk_sign(). Found by Jean-Philippe Aumasson.
+
 Bugfix
    * Fix the redefinition of macro ssl_set_bio to an undefined symbol
      mbedtls_ssl_set_bio_timeout in compat-1.3.h, by removing it.

library/pk.c

@@ -29,6 +29,8 @@
 #include "mbedtls/pk.h"
 #include "mbedtls/pk_internal.h"
 
+#include "mbedtls/bignum.h"
+
 #if defined(MBEDTLS_RSA_C)
 #include "mbedtls/rsa.h"
 #endif
@@ -39,6 +41,8 @@
 #include "mbedtls/ecdsa.h"
 #endif
 
+#include <limits.h>
+
 /* Implementation that should never be optimized out by the compiler */
 static void mbedtls_zeroize( void *v, size_t n ) {
     volatile unsigned char *p = v; while( n-- ) *p++ = 0;
@@ -209,6 +213,11 @@ int mbedtls_pk_verify_ext( mbedtls_pk_type_t type, const void *options,
         int ret;
         const mbedtls_pk_rsassa_pss_options *pss_opts;
 
+#if defined(MBEDTLS_HAVE_INT64)
+        if( md_alg == MBEDTLS_MD_NONE && UINT_MAX < hash_len )
+            return( MBEDTLS_ERR_PK_BAD_INPUT_DATA );
+#endif /* MBEDTLS_HAVE_INT64 */
+
         if( options == NULL )
             return( MBEDTLS_ERR_PK_BAD_INPUT_DATA );
 
@@ -232,7 +241,7 @@ int mbedtls_pk_verify_ext( mbedtls_pk_type_t type, const void *options,
         return( 0 );
 #else
         return( MBEDTLS_ERR_PK_FEATURE_UNAVAILABLE );
-#endif
+#endif /* MBEDTLS_RSA_C && MBEDTLS_PKCS1_V21 */
     }
 
     /* General case: no options */

library/pk_wrap.c

@@ -30,6 +30,7 @@
 
 /* Even if RSA not activated, for the sake of RSA-alt */
 #include "mbedtls/rsa.h"
+#include "mbedtls/bignum.h"
 
 #include <string.h>
 
@@ -49,6 +50,8 @@
 #define mbedtls_free       free
 #endif
 
+#include <limits.h>
+
 #if defined(MBEDTLS_PK_RSA_ALT_SUPPORT)
 /* Implementation that should never be optimized out by the compiler */
 static void mbedtls_zeroize( void *v, size_t n ) {
@@ -74,6 +77,11 @@ static int rsa_verify_wrap( void *ctx, mbedtls_md_type_t md_alg,
 {
     int ret;
 
+#if defined(MBEDTLS_HAVE_INT64)
+    if( md_alg == MBEDTLS_MD_NONE && UINT_MAX < hash_len )
+        return( MBEDTLS_ERR_PK_BAD_INPUT_DATA );
+#endif /* MBEDTLS_HAVE_INT64 */
+
     if( sig_len < ((mbedtls_rsa_context *) ctx)->len )
         return( MBEDTLS_ERR_RSA_VERIFY_FAILED );
 
@@ -93,6 +101,11 @@ static int rsa_sign_wrap( void *ctx, mbedtls_md_type_t md_alg,
                    unsigned char *sig, size_t *sig_len,
                    int (*f_rng)(void *, unsigned char *, size_t), void *p_rng )
 {
+#if defined(MBEDTLS_HAVE_INT64)
+    if( md_alg == MBEDTLS_MD_NONE && UINT_MAX < hash_len )
+        return( MBEDTLS_ERR_PK_BAD_INPUT_DATA );
+#endif /* MBEDTLS_HAVE_INT64 */
+
     *sig_len = ((mbedtls_rsa_context *) ctx)->len;
 
     return( mbedtls_rsa_pkcs1_sign( (mbedtls_rsa_context *) ctx, f_rng, p_rng, MBEDTLS_RSA_PRIVATE,
@@ -402,6 +415,11 @@ static int rsa_alt_sign_wrap( void *ctx, mbedtls_md_type_t md_alg,
 {
     mbedtls_rsa_alt_context *rsa_alt = (mbedtls_rsa_alt_context *) ctx;
 
+#if defined(MBEDTLS_HAVE_INT64)
+    if( UINT_MAX < hash_len )
+        return( MBEDTLS_ERR_PK_BAD_INPUT_DATA );
+#endif /* MBEDTLS_HAVE_INT64 */
+
     *sig_len = rsa_alt->key_len_func( rsa_alt->key );
 
     return( rsa_alt->sign_func( rsa_alt->key, f_rng, p_rng, MBEDTLS_RSA_PRIVATE,