rsa: pss: Enable use of big hashes with small keys

It should be valid to RSASSA-PSS sign a SHA-512 hash with a 1024-bit or
1032-bit RSA key, but with the salt size being always equal to the hash
size, this isn't possible: the key is too small.

To enable use of hashes that are relatively large compared to the key
size, allow reducing the salt size to no less than the hash size minus 2
bytes. We don't allow salt sizes smaller than the hash size minus 2
bytes because that too significantly changes the security guarantees the
library provides compared to the previous implementation which always
used a salt size equal to the hash size. The new calculated salt size
remains compliant with FIPS 186-4.

We also need to update the "hash too large" test, since we now reduce
the salt size when certain key sizes are used. We used to not support
1024-bit keys with SHA-512, but now we support this by reducing the salt
size to 62. Update the "hash too large" test to use a 1016-bit RSA key
with SHA-512, which still has too large of a hash because we will not
reduce the salt size further than 2 bytes shorter than the hash size.

The RSA private key used for the test was generated using "openssl
genrsa 1016" using OpenSSL 1.1.1-pre8.

    $ openssl genrsa 1016
    Generating RSA private key, 1016 bit long modulus (2 primes)
    ..............++++++
    ....++++++
    e is 65537 (0x010001)
    -----BEGIN RSA PRIVATE KEY-----
    MIICVwIBAAKBgACu54dKTbLxUQBEQF2ynxTfDze7z2H8vMmUo9McqvhYp0zI8qQK
    yanOeqmgaA9iz52NS4JxFFM/2/hvFvyd/ly/hX2GE1UZpGEf/FnLdHOGFhmnjj7D
    FHFegEz/gtbzLp9X3fOQVjYpiDvTT0Do20EyCbFRzul9gXpdZcfaVHNLAgMBAAEC
    gYAAiWht2ksmnP01B2nF8tGV1RQghhUL90Hd4D/AWFJdX1C4O1qc07jRBd1KLDH0
    fH19WocLCImeSZooGCZn+jveTuaEH14w6I0EfnpKDcpWVAoIP6I8eSdAttrnTyTn
    Y7VgPrcobyq4WkCVCD/jLUbn97CneF7EHNspXGMTvorMeQJADjy2hF5SginhnPsk
    YR5oWawc6n01mStuLnloI8Uq/6A0AOQoMPkGl/CESZw+NYfe/BnnSeckM917cMKL
    DIKAtwJADEj55Frjj9tKUUO+N9eaEM1PH5eC7yakhIpESccs/XEsaDUIGHNjhctK
    mrbbWu+OlsVRA5z8yJFYIa7gae1mDQJABjtQ8JOQreTDGkFbZR84MbgCWClCIq89
    5R3DFZUiAw4OdS1o4ja+Shc+8DFxkWDNm6+C63g/Amy5sVuWHX2p9QI/a69Cxmns
    TxHoXm1w9Azublk7N7DgB26yqxlTfWJo+ysOFmLEk47g0ekoCwLPxkwXlYIEoad2
    JqPh418DwYExAkACcqrd9+rfxtrbCbTXHEizW7aHR+fVOr9lpXXDEZTlDJ57sRkS
    SpjXbAmylqQuKLqH8h/72RbiP36kEm5ptmw2
    -----END RSA PRIVATE KEY-----

ChangeLog

@@ -62,6 +62,12 @@ Changes
      mbedtls_ssl_get_record_expansion() in case of ChachaPoly ciphersuites,
      or CBC ciphersuites in (D)TLS versions 1.1 or higher. Fixes #1913, #1914.
    * Add support for buffering of out-of-order handshake messages.
+   * Extend RSASSA-PSS signature to allow slightly a smaller salt size.
+     Previously, PSS signature always used a salt with the same length as the
+     hash, and returned an error if this was not possible. Now the salt size
+     may be up to two bytes shorter. This allows the library to support all
+     hash and signature sizes that comply with FIPS 186-4, including SHA-512
+     with a 1024-bit key.
 
 INTERNAL NOTE: need to bump soversion of libmbedtls:
 - added new member 'mtu' to public 'mbedtls_ssl_conf' structure

include/mbedtls/rsa.h

@@ -894,6 +894,16 @@ int mbedtls_rsa_rsassa_pkcs1_v15_sign( mbedtls_rsa_context *ctx,
  *                 Specifications</em> it is advised to keep both hashes the
  *                 same.
  *
+ * \note           This function always uses the maximum possible salt size,
+ *                 up to the length of the payload hash. This choice of salt
+ *                 size complies with FIPS 186-4 §5.5 (e) and RFC 8017 (PKCS#1
+ *                 v2.2) §9.1.1 step 3. Furthermore this function enforces a
+ *                 minimum salt size which is the hash size minus 2 bytes. If
+ *                 this minimum size is too large given the key size (the salt
+ *                 size, plus the hash size, plus 2 bytes must be no more than
+ *                 the key size in bytes), this function returns
+ *                 #MBEDTLS_ERR_RSA_BAD_INPUT_DATA.
+ *
  * \deprecated     It is deprecated and discouraged to call this function
  *                 in #MBEDTLS_RSA_PUBLIC mode. Future versions of the library
  *                 are likely to remove the \p mode argument and have it

library/rsa.c

@@ -1521,7 +1521,7 @@ int mbedtls_rsa_rsassa_pss_sign( mbedtls_rsa_context *ctx,
     size_t olen;
     unsigned char *p = sig;
     unsigned char salt[MBEDTLS_MD_MAX_SIZE];
-    size_t slen, hlen, offset = 0;
+    size_t slen, min_slen, hlen, offset = 0;
     int ret;
     size_t msb;
     const mbedtls_md_info_t *md_info;
@@ -1550,10 +1550,20 @@ int mbedtls_rsa_rsassa_pss_sign( mbedtls_rsa_context *ctx,
         return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
 
     hlen = mbedtls_md_get_size( md_info );
-    slen = hlen;
 
-    if( olen < hlen + slen + 2 )
+    /* Calculate the largest possible salt length. Normally this is the hash
+     * length, which is the maximum length the salt can have. If there is not
+     * enough room, use the maximum salt length that fits. The constraint is
+     * that the hash length plus the salt length plus 2 bytes must be at most
+     * the key length. This complies with FIPS 186-4 §5.5 (e) and RFC 8017
+     * (PKCS#1 v2.2) §9.1.1 step 3. */
+    min_slen = hlen - 2;
+    if( olen < hlen + min_slen + 2 )
         return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
+    else if( olen >= hlen + hlen + 2 )
+        slen = hlen;
+    else
+        slen = olen - hlen - 2;
 
     memset( sig, 0, olen );
 
@@ -1563,7 +1573,7 @@ int mbedtls_rsa_rsassa_pss_sign( mbedtls_rsa_context *ctx,
 
     /* Note: EMSA-PSS encoding is over the length of N - 1 bits */
     msb = mbedtls_mpi_bitlen( &ctx->N ) - 1;
-    p += olen - hlen * 2 - 2;
+    p += olen - hlen - slen - 2;
     *p++ = 0x01;
     memcpy( p, salt, slen );
     p += slen;

tests/suites/test_suite_pkcs1_v21.data

@@ -376,8 +376,9 @@ pkcs1_rsassa_pss_sign:1024:16:"d17f655bf27c8b16d35462c905cc04a26f37e2a67fa9c0ce0
 RSASSA-PSS Verification Test Vector Int
 pkcs1_rsassa_pss_verify:1024:16:"a2ba40ee07e3b2bd2f02ce227f36a195024486e49c19cb41bbbdfbba98b22b0e577c2eeaffa20d883a76e65e394c69d4b3c05a1e8fadda27edb2a42bc000fe888b9b32c22d15add0cd76b3e7936e19955b220dd17d4ea904b1ec102b2e4de7751222aa99151024c7cb41cc5ea21d00eeb41f7c800834d2c6e06bce3bce7ea9a5":16:"010001":MBEDTLS_MD_SHA1:MBEDTLS_MD_SHA1:"859eef2fd78aca00308bdc471193bf55bf9d78db8f8a672b484634f3c9c26e6478ae10260fe0dd8c082e53a5293af2173cd50c6d5d354febf78b26021c25c02712e78cd4694c9f469777e451e7f8e9e04cd3739c6bbfedae487fb55644e9ca74ff77a53cb729802f6ed4a5ffa8ba159890fc":"e3b5d5d002c1bce50c2b65ef88a188d83bce7e61":"8daa627d3de7595d63056c7ec659e54406f10610128baae821c8b2a0f3936d54dc3bdce46689f6b7951bb18e840542769718d5715d210d85efbb596192032c42be4c29972c856275eb6d5a45f05f51876fc6743deddd28caec9bb30ea99e02c3488269604fe497f74ccd7c7fca1671897123cbd30def5d54a2b5536ad90a747e":0
 
-RSASSA-PSS Signing Test Vector Hash too large
+RSASSA-PSS Signature RSA-1016, SHA-512: minimum salt size not met
-pkcs1_rsassa_pss_sign:1024:16:"d17f655bf27c8b16d35462c905cc04a26f37e2a67fa9c0ce0dced472394a0df743fe7f929e378efdb368eddff453cf007af6d948e0ade757371f8a711e278f6b":16:"c6d92b6fee7414d1358ce1546fb62987530b90bd15e0f14963a5e2635adb69347ec0c01b2ab1763fd8ac1a592fb22757463a982425bb97a3a437c5bf86d03f2f":16:"a2ba40ee07e3b2bd2f02ce227f36a195024486e49c19cb41bbbdfbba98b22b0e577c2eeaffa20d883a76e65e394c69d4b3c05a1e8fadda27edb2a42bc000fe888b9b32c22d15add0cd76b3e7936e19955b220dd17d4ea904b1ec102b2e4de7751222aa99151024c7cb41cc5ea21d00eeb41f7c800834d2c6e06bce3bce7ea9a5":16:"010001":MBEDTLS_MD_SHA1:MBEDTLS_MD_SHA512:"d436e99569fd32a7c8a05bbc90d32c49d436e99569fd32a7c8a05bbc90d32c49d436e99569fd32a7c8a05bbc90d32c49d436e99569fd32a7c8a05bbc90d32c49d436e99569fd32a7c8a05bbc90d32c49d436e99569fd00":"e3b5d5d002c1bce50c2b65ef88a188d83bce7e61":"":MBEDTLS_ERR_RSA_BAD_INPUT_DATA
+depends_on:MBEDTLS_SHA512_C
+pkcs1_rsassa_pss_sign:1016:16:"0e3cb6845e528229e19cfb24611e6859ac1cea7d35992b6e2e796823c52affa03400e42830f90697f084499c3e3587defc19e749e72433dd7b70c28b0c8280b7":16:"0c48f9e45ae38fdb4a5143be37d79a10cd4f1f9782ef26a4848a4449c72cfd712c68350818736385cb4a9ab6db5aef8e96c551039cfcc8915821aee069ed660d":16:"00aee7874a4db2f1510044405db29f14df0f37bbcf61fcbcc994a3d31caaf858a74cc8f2a40ac9a9ce7aa9a0680f62cf9d8d4b827114533fdbf86f16fc9dfe5cbf857d86135519a4611ffc59cb7473861619a78e3ec314715e804cff82d6f32e9f57ddf390563629883bd34f40e8db413209b151cee97d817a5d65c7da54734b":16:"010001":MBEDTLS_MD_SHA512:MBEDTLS_MD_SHA512:"d436e99569fd32a7c8a05bbc90d32c49d436e99569fd32a7c8a05bbc90d32c49d436e99569fd32a7c8a05bbc90d32c49d436e99569fd32a7c8a05bbc90d32c49d436e99569fd32a7c8a05bbc90d32c49d436e99569fd00":"e3b5d5d002c1bce50c2b65ef88a188d83bce7e61":"":MBEDTLS_ERR_RSA_BAD_INPUT_DATA
 
 RSASSA-PSS Signature Example 1_1
 pkcs1_rsassa_pss_sign:1024:16:"e7e8942720a877517273a356053ea2a1bc0c94aa72d55c6e86296b2dfc967948c0a72cbccca7eacb35706e09a1df55a1535bd9b3cc34160b3b6dcd3eda8e6443":16:"b69dca1cf7d4d7ec81e75b90fcca874abcde123fd2700180aa90479b6e48de8d67ed24f9f19d85ba275874f542cd20dc723e6963364a1f9425452b269a6799fd":16:"a56e4a0e701017589a5187dc7ea841d156f2ec0e36ad52a44dfeb1e61f7ad991d8c51056ffedb162b4c0f283a12a88a394dff526ab7291cbb307ceabfce0b1dfd5cd9508096d5b2b8b6df5d671ef6377c0921cb23c270a70e2598e6ff89d19f105acc2d3f0cb35f29280e1386b6f64c4ef22e1e1f20d0ce8cffb2249bd9a2137":16:"010001":MBEDTLS_MD_SHA1:MBEDTLS_MD_SHA1:"cdc87da223d786df3b45e0bbbc721326d1ee2af806cc315475cc6f0d9c66e1b62371d45ce2392e1ac92844c310102f156a0d8d52c1f4c40ba3aa65095786cb769757a6563ba958fed0bcc984e8b517a3d5f515b23b8a41e74aa867693f90dfb061a6e86dfaaee64472c00e5f20945729cbebe77f06ce78e08f4098fba41f9d6193c0317e8b60d4b6084acb42d29e3808a3bc372d85e331170fcbf7cc72d0b71c296648b3a4d10f416295d0807aa625cab2744fd9ea8fd223c42537029828bd16be02546f130fd2e33b936d2676e08aed1b73318b750a0167d0":"dee959c7e06411361420ff80185ed57f3e6776af":"9074308fb598e9701b2294388e52f971faac2b60a5145af185df5287b5ed2887e57ce7fd44dc8634e407c8e0e4360bc226f3ec227f9d9e54638e8d31f5051215df6ebb9c2f9579aa77598a38f914b5b9c1bd83c4e2f9f382a0d0aa3542ffee65984a601bc69eb28deb27dca12c82c2d4c3f66cd500f1ff2b994d8a4e30cbb33c":0