Fix data loss in unsigned int cast in PK

This patch introduces some additional checks in the PK module for 64-bit
systems only. The problem is that the API functions in the PK
abstraction accept a size_t value for the hashlen, while the RSA module
accepts an unsigned int for the hashlen. Instead of silently casting
size_t to unsigned int, this change checks whether the hashlen overflows
an unsigned int and returns an error.

ChangeLog

@@ -1,6 +1,13 @@
 mbed TLS ChangeLog (Sorted per branch, date)
 
-= mbed TLS x.x.x branch xxxx-xx-xx
+= mbed TLS x.x.x branch released xxxx-xx-xx
+
+Security
+    * Add checks to prevent signature forgeries for very large messages while
+      using RSA through the PK module in 64-bit systems. The issue was caused by
+      some data loss when casting a size_t to an unsigned int value in the
+      functions rsa_verify_wrap(), rsa_sign_wrap(), rsa_alt_sign_wrap() and
+      pk_sign(). Found by Jean-Philippe Aumasson.
 
 Bugfix
    * Fix unused variable/function compilation warnings in pem.c and x509_csr.c

library/pk.c

@@ -30,6 +30,8 @@
 #include "polarssl/pk.h"
 #include "polarssl/pk_wrap.h"
 
+#include "polarssl/bignum.h"
+
 #if defined(POLARSSL_RSA_C)
 #include "polarssl/rsa.h"
 #endif
@@ -40,6 +42,8 @@
 #include "polarssl/ecdsa.h"
 #endif
 
+#include <limits.h>
+
 /* Implementation that should never be optimized out by the compiler */
 static void polarssl_zeroize( void *v, size_t n ) {
     volatile unsigned char *p = v; while( n-- ) *p++ = 0;
@@ -208,6 +212,11 @@ int pk_verify_ext( pk_type_t type, const void *options,
         int ret;
         const pk_rsassa_pss_options *pss_opts;
 
+#if defined(POLARSSL_HAVE_INT64)
+        if( md_alg == POLARSSL_MD_NONE && UINT_MAX < hash_len )
+            return( POLARSSL_ERR_PK_BAD_INPUT_DATA );
+#endif /* POLARSSL_HAVE_INT64 */
+
         if( options == NULL )
             return( POLARSSL_ERR_PK_BAD_INPUT_DATA );
 
@@ -231,7 +240,7 @@ int pk_verify_ext( pk_type_t type, const void *options,
         return( 0 );
 #else
         return( POLARSSL_ERR_PK_FEATURE_UNAVAILABLE );
-#endif
+#endif /* POLARSSL_RSA_C && POLARSSL_PKCS1_V21 */
     }
 
     /* General case: no options */

library/pk_wrap.c

@@ -31,6 +31,7 @@
 
 /* Even if RSA not activated, for the sake of RSA-alt */
 #include "polarssl/rsa.h"
+#include "polarssl/bignum.h"
 
 #include <string.h>
 
@@ -50,6 +51,8 @@
 #define polarssl_free       free
 #endif
 
+#include <limits.h>
+
 /* Implementation that should never be optimized out by the compiler */
 static void polarssl_zeroize( void *v, size_t n ) {
     volatile unsigned char *p = v; while( n-- ) *p++ = 0;
@@ -73,6 +76,11 @@ static int rsa_verify_wrap( void *ctx, md_type_t md_alg,
 {
     int ret;
 
+#if defined(POLARSSL_HAVE_INT64)
+    if( md_alg == POLARSSL_MD_NONE && UINT_MAX < hash_len )
+        return( POLARSSL_ERR_PK_BAD_INPUT_DATA );
+#endif /* POLARSSL_HAVE_INT64 */
+
     if( sig_len < ((rsa_context *) ctx)->len )
         return( POLARSSL_ERR_RSA_VERIFY_FAILED );
 
@@ -92,6 +100,11 @@ static int rsa_sign_wrap( void *ctx, md_type_t md_alg,
                    unsigned char *sig, size_t *sig_len,
                    int (*f_rng)(void *, unsigned char *, size_t), void *p_rng )
 {
+#if defined(POLARSSL_HAVE_INT64)
+    if( md_alg == POLARSSL_MD_NONE && UINT_MAX < hash_len )
+        return( POLARSSL_ERR_PK_BAD_INPUT_DATA );
+#endif /* POLARSSL_HAVE_INT64 */
+
     *sig_len = ((rsa_context *) ctx)->len;
 
     return( rsa_pkcs1_sign( (rsa_context *) ctx, f_rng, p_rng, RSA_PRIVATE,
@@ -411,6 +424,11 @@ static int rsa_alt_sign_wrap( void *ctx, md_type_t md_alg,
 {
     rsa_alt_context *rsa_alt = (rsa_alt_context *) ctx;
 
+#if defined(POLARSSL_HAVE_INT64)
+    if( UINT_MAX < hash_len )
+        return( POLARSSL_ERR_PK_BAD_INPUT_DATA );
+#endif /* POLARSSL_HAVE_INT64 */
+
     *sig_len = rsa_alt->key_len_func( rsa_alt->key );
 
     return( rsa_alt->sign_func( rsa_alt->key, f_rng, p_rng, RSA_PRIVATE,