account for rsa_pss_rsae vs rsa_pss_pss type

scripts/openssl.test

@@ -1228,6 +1228,27 @@ do
 done
 IFS="$OIFS" #restore separator
 
+# Test for RSA-PSS certs
+echo -e "Doing interop RSA-PSS test"
+
+key_file=${CERT_DIR}/rsapss/server-rsapss-priv.pem
+cert_file=${CERT_DIR}/rsapss/server-rsapss.pem
+ca_file=${CERT_DIR}/client-cert.pem
+openssl_suite="RSAPSS"
+start_openssl_server
+
+cert="${CERT_DIR}/client-cert.pem"
+key="${CERT_DIR}/client-key.pem"
+caCert="${CERT_DIR}/rsapss/ca-rsapss.pem"
+crl="-C"
+wolfSuite="ALL"
+version="4"
+port=$server_port
+do_wolfssl_client
+
+version="3"
+do_wolfssl_client
+
 do_cleanup
 
 echo -e "wolfSSL total cases   $wolf_cases_total"

src/internal.c

@@ -5142,7 +5142,7 @@ int RsaVerify(WOLFSSL* ssl, byte* in, word32 inSz, byte** out, int sigAlgo,
 #endif
 
 #if defined(WC_RSA_PSS)
-    if (sigAlgo == rsa_pss_sa_algo) {
+    if (sigAlgo == rsa_pss_sa_algo || sigAlgo == rsa_pss_pss_algo) {
         enum wc_HashType hashType = WC_HASH_TYPE_NONE;
         int mgf = 0;
 
@@ -32292,6 +32292,13 @@ static int DoServerKeyExchange(WOLFSSL* ssl, const byte* input,
                         }
                         else
                     #endif
+                    #ifdef WC_RSA_PSS
+                        if (sigAlgo == rsa_pss_pss_algo &&
+                                ssl->options.peerSigAlgo == rsa_sa_algo) {
+                            ssl->options.peerSigAlgo = sigAlgo;
+                        }
+                        else
+                    #endif
                     #if defined(WOLFSSL_SM2) && defined(WOLFSSL_SM3)
                         if (sigAlgo == sm2_sa_algo &&
                                              ssl->options.peerSigAlgo == ecc_dsa_sa_algo) {
@@ -32358,6 +32365,7 @@ static int DoServerKeyExchange(WOLFSSL* ssl, const byte* input,
                     #ifndef NO_RSA
                     #ifdef WC_RSA_PSS
                         case rsa_pss_sa_algo:
+                        case rsa_pss_pss_algo:
                     #endif
                         case rsa_sa_algo:
                         {
@@ -32458,6 +32466,7 @@ static int DoServerKeyExchange(WOLFSSL* ssl, const byte* input,
                     #ifndef NO_RSA
                     #ifdef WC_RSA_PSS
                         case rsa_pss_sa_algo:
+                        case rsa_pss_pss_algo:
                     #endif
                         case rsa_sa_algo:
                         {
@@ -32669,6 +32678,7 @@ static int DoServerKeyExchange(WOLFSSL* ssl, const byte* input,
                     #ifndef NO_RSA
                     #ifdef WC_RSA_PSS
                         case rsa_pss_sa_algo:
+                        case rsa_pss_pss_algo:
                         #ifdef HAVE_SELFTEST
                             ret = wc_RsaPSS_CheckPadding(
                                              ssl->buffers.digest.buffer,

src/tls13.c

@@ -7938,6 +7938,27 @@ static void EncodeDualSigAlg(byte sigAlg, byte altSigAlg, byte* output)
 }
 #endif /* WOLFSSL_DUAL_ALG_CERTS */
 
+static enum wc_MACAlgorithm GetNewSAHashAlgo(int typeIn)
+{
+    switch (typeIn) {
+        case RSA_PSS_RSAE_SHA256_MINOR:
+        case RSA_PSS_PSS_SHA256_MINOR:
+            return sha256_mac;
+
+        case RSA_PSS_RSAE_SHA384_MINOR:
+        case RSA_PSS_PSS_SHA384_MINOR:
+            return sha384_mac;
+
+        case RSA_PSS_RSAE_SHA512_MINOR:
+        case RSA_PSS_PSS_SHA512_MINOR:
+        case ED25519_SA_MINOR:
+        case ED448_SA_MINOR:
+            return sha512_mac;
+        default:
+            return no_mac;
+    }
+}
+
 /* Decode the signature algorithm.
  *
  * input     The encoded signature algorithm.
@@ -7962,17 +7983,23 @@ static WC_INLINE int DecodeTls13SigAlg(byte* input, byte* hashAlgo,
             break;
     #endif
         case NEW_SA_MAJOR:
-            /* PSS signatures: 0x080[4-6] */
-            if (input[1] >= sha256_mac && input[1] <= sha512_mac) {
+            *hashAlgo = GetNewSAHashAlgo(input[1]);
+
+            /* PSS encryption: 0x080[4-6] */
+            if (input[1] >= RSA_PSS_RSAE_SHA256_MINOR &&
+                    input[1] <= RSA_PSS_RSAE_SHA512_MINOR) {
+                *hsType   = input[0];
+            }
+            /* PSS signature: 0x080[9-B] */
+            else if (input[1] >= RSA_PSS_PSS_SHA256_MINOR &&
+                    input[1] <= RSA_PSS_PSS_SHA512_MINOR) {
                 *hsType   = input[0];
-                *hashAlgo = input[1];
             }
     #ifdef HAVE_ED25519
             /* ED25519: 0x0807 */
             else if (input[1] == ED25519_SA_MINOR) {
                 *hsType = ed25519_sa_algo;
                 /* Hash performed as part of sign/verify operation. */
-                *hashAlgo = sha512_mac;
             }
     #endif
     #ifdef HAVE_ED448
@@ -7980,7 +8007,6 @@ static WC_INLINE int DecodeTls13SigAlg(byte* input, byte* hashAlgo,
             else if (input[1] == ED448_SA_MINOR) {
                 *hsType = ed448_sa_algo;
                 /* Hash performed as part of sign/verify operation. */
-                *hashAlgo = sha512_mac;
             }
     #endif
             else

wolfssl/internal.h

@@ -1813,6 +1813,13 @@ enum Misc {
     MAX_CURVE_NAME_SZ   = 18,  /* Maximum size of curve name string */
 
     NEW_SA_MAJOR        = 8,   /* Most significant byte used with new sig algos */
+    RSA_PSS_RSAE_SHA256_MINOR = 0x04,
+    RSA_PSS_RSAE_SHA384_MINOR = 0x05,
+    RSA_PSS_RSAE_SHA512_MINOR = 0x06,
+    RSA_PSS_PSS_SHA256_MINOR = 0x09,
+    RSA_PSS_PSS_SHA384_MINOR = 0x0A,
+    RSA_PSS_PSS_SHA512_MINOR = 0x0B,
+
     ED25519_SA_MAJOR    = 8,   /* Most significant byte for ED25519 */
     ED25519_SA_MINOR    = 7,   /* Least significant byte for ED25519 */
     ED448_SA_MAJOR      = 8,   /* Most significant byte for ED448 */