Fix errors in the definition of MBEDTLS_PK_SIGNATURE_MAX_SIZE

The initial value for the max calculation needs to be 0. The fallback
needs to come last. With the old code, the value was never smaller
than the fallback.

For RSA_ALT, use MPI_MAX_SIZE. Only use this if RSA_ALT is enabled.

For PSA, check PSA_ASYMMETRIC_SIGNATURE_MAX_SIZE, and separately check
the special case of ECDSA where PSA and mbedtls have different
representations for the signature.

include/mbedtls/pk.h

@@ -104,37 +104,54 @@ typedef struct mbedtls_pk_rsassa_pss_options
 /**
  * \brief           Maximum size of a signature made by mbedtls_pk_sign().
  */
-/* This fallback value is used if there is no software signature support.
+/* We need to set MBEDTLS_PK_SIGNATURE_MAX_SIZE to the maximum signature
- * This is possible even if check_config.h is included, for example if
+ * size among the supported signature types. Do it by starting at 0,
- * MBEDTLS_ECDH_C is enabled but neither MBEDTLS_ECDSA_C  nor MBEDTLS_RSA_C.
+ * then incrementally increasing to be large enough for each supported
- * Use MBEDTLS_MPI_MAX_SIZE which is the maximum size than an RSA-alt
+ * signature mechanism.
- * implementation can produce, assuming that MBEDTLS_MPI_MAX_SIZE is set
+ *
- * correctly. This is not necessarily the best choice of size and it may
+ * The resulting value can be 0, for example if MBEDTLS_ECDH_C is enabled
- * change in future versions. */
+ * (which allows the pk module to be included) but neither MBEDTLS_ECDSA_C
-#define MBEDTLS_PK_SIGNATURE_MAX_SIZE MBEDTLS_MPI_MAX_SIZE
+ * nor MBEDTLS_RSA_C nor any opaque signature mechanism (PSA or RSA_ALT).
-#if defined(MBEDTLS_RSA_C) &&                                   \
+ */
+#define MBEDTLS_PK_SIGNATURE_MAX_SIZE 0
+
+#if ( defined(MBEDTLS_RSA_C) || defined(MBEDTLS_PK_RSA_ALT_SUPPORT) ) && \
     MBEDTLS_MPI_MAX_SIZE > MBEDTLS_PK_SIGNATURE_MAX_SIZE
+/* For RSA, the signature can be as large as the bignum module allows.
+ * For RSA_ALT, the signature size is not necessarily tied to what the
+ * bignum module can do, but in the absence of any specific setting,
+ * we use that (rsa_alt_sign_wrap in pk_wrap will check). */
 #undef MBEDTLS_PK_SIGNATURE_MAX_SIZE
 #define MBEDTLS_PK_SIGNATURE_MAX_SIZE MBEDTLS_MPI_MAX_SIZE
 #endif
+
 #if defined(MBEDTLS_ECDSA_C) &&                                 \
     MBEDTLS_ECDSA_MAX_LEN > MBEDTLS_PK_SIGNATURE_MAX_SIZE
+/* For ECDSA, the ecdsa module exports a constant for the maximum
+ * signature size. */
 #undef MBEDTLS_PK_SIGNATURE_MAX_SIZE
 #define MBEDTLS_PK_SIGNATURE_MAX_SIZE MBEDTLS_ECDSA_MAX_LEN
 #endif
-#if defined(MBEDTLS_USE_PSA_CRYPTO) && \
+
-    PSA_ASYMMETRIC_SIGNATURE_MAX_SIZE + 11 > MBEDTLS_PK_SIGNATURE_MAX_SIZE
+#if defined(MBEDTLS_USE_PSA_CRYPTO)
+#if PSA_ASYMMETRIC_SIGNATURE_MAX_SIZE > MBEDTLS_PK_SIGNATURE_MAX_SIZE
 /* PSA_ASYMMETRIC_SIGNATURE_MAX_SIZE is the maximum size of a signature made
- * through the PSA API in the PSA representation.
+ * through the PSA API in the PSA representation. */
- * The Mbed TLS representation is different for ECDSA signatures:
+#undef MBEDTLS_PK_SIGNATURE_MAX_SIZE
+#define MBEDTLS_PK_SIGNATURE_MAX_SIZE PSA_ASYMMETRIC_SIGNATURE_MAX_SIZE
+#endif
+
+#if PSA_VENDOR_ECDSA_SIGNATURE_MAX_SIZE + 11 > MBEDTLS_PK_SIGNATURE_MAX_SIZE
+/* The Mbed TLS representation is different for ECDSA signatures:
  * PSA uses the raw concatenation of r and s,
  * whereas Mbed TLS uses the ASN.1 representation (SEQUENCE of two INTEGERs).
  * Add the overhead of ASN.1: up to (1+2) + 2 * (1+2+1) for the
  * types, lengths (represented by up to 2 bytes), and potential leading
  * zeros of the INTEGERs and the SEQUENCE. */
 #undef MBEDTLS_PK_SIGNATURE_MAX_SIZE
-#define MBEDTLS_PK_SIGNATURE_MAX_SIZE ( PSA_ASYMMETRIC_SIGNATURE_MAX_SIZE + 11 )
+#define MBEDTLS_PK_SIGNATURE_MAX_SIZE ( PSA_VENDOR_ECDSA_SIGNATURE_MAX_SIZE + 11 )
 #endif
+#endif /* defined(MBEDTLS_USE_PSA_CRYPTO) */
 
 /**
  * \brief           Types for interfacing with the debug module