Add support for separate MGF1 hashes

Update PKCS#1-PSS and RSA APIs that allow passing a separate hash index for
the MGF1 hash.

Signed-off-by: Steffen Jaeckel <[email protected]>

src/headers/tomcrypt_pk.h

@@ -103,10 +103,10 @@ void rsa_free(rsa_key *key);
   rsa_decrypt_key_ex(in, inlen, out, outlen, lparam, lparamlen, hash_idx, -1, LTC_PKCS_1_OAEP, stat, key)
 
 #define rsa_sign_hash(in, inlen, out, outlen, prng, prng_idx, hash_idx, saltlen, key) \
-  rsa_sign_hash_ex(in, inlen, out, outlen, LTC_PKCS_1_PSS, prng, prng_idx, hash_idx, saltlen, key)
+  rsa_sign_hash_ex(in, inlen, out, outlen, LTC_PKCS_1_PSS, prng, prng_idx, hash_idx, hash_idx, saltlen, key)
 
 #define rsa_verify_hash(sig, siglen, hash, hashlen, hash_idx, saltlen, stat, key) \
-  rsa_verify_hash_ex(sig, siglen, hash, hashlen, LTC_PKCS_1_PSS, hash_idx, saltlen, stat, key)
+  rsa_verify_hash_ex(sig, siglen, hash, hashlen, LTC_PKCS_1_PSS, hash_idx, hash_idx, saltlen, stat, key)
 
 #define rsa_sign_saltlen_get_max(hash_idx, key) \
   rsa_sign_saltlen_get_max_ex(LTC_PKCS_1_PSS, hash_idx, key)
@@ -130,14 +130,16 @@ int rsa_decrypt_key_ex(const unsigned char *in,             unsigned long  inlen
 int rsa_sign_hash_ex(const unsigned char *in,       unsigned long  inlen,
                            unsigned char *out,      unsigned long *outlen,
                            int            padding,
-                           prng_state    *prng,     int            prng_idx,
-                           int            hash_idx, unsigned long  saltlen,
+                           prng_state    *prng,               int  prng_idx,
+                           int            hash_idx,           int  mgf_hash_idx,
+                           unsigned long  saltlen,
                      const rsa_key       *key);
 
 int rsa_verify_hash_ex(const unsigned char *sig,            unsigned long  siglen,
                        const unsigned char *hash,           unsigned long  hashlen,
                              int            padding,
-                             int            hash_idx,       unsigned long  saltlen,
+                             int            hash_idx,                 int  mgf_hash_idx,
+                             unsigned long  saltlen,
                              int           *stat,     const rsa_key       *key);
 
 int rsa_sign_saltlen_get_max_ex(int padding, int hash_idx, const rsa_key *key);

src/headers/tomcrypt_private.h

@@ -750,6 +750,22 @@ int pk_oid_cmp_with_asn1(const char *o1, const ltc_asn1_list *o2);
 
 /* tomcrypt_pkcs.h */
 
+#ifdef LTC_PKCS_1
+
+int pkcs_1_pss_encode_mgf1(const unsigned char *msghash,       unsigned long  msghashlen,
+                                 unsigned long saltlen,
+                                 prng_state    *prng,                    int  prng_idx,
+                                 int           hash_idx,                 int  mgf_hash_idx,
+                                 unsigned long modulus_bitlen,
+                                 unsigned char *out,           unsigned long *outlen);
+int pkcs_1_pss_decode_mgf1(const unsigned char *msghash, unsigned long msghashlen,
+                           const unsigned char *sig,     unsigned long siglen,
+                                 unsigned long saltlen,
+                                          int  hash_idx,           int mgf_hash_idx,
+                                 unsigned long modulus_bitlen,     int *res);
+
+#endif /* LTC_PKCS_1 */
+
 #ifdef LTC_PKCS_8
 
 /* Public-Key Cryptography Standards (PKCS) #8:

src/pk/pkcs1/pkcs_1_pss_decode.c

@@ -17,14 +17,16 @@
    @param  siglen          The length of the signature data (octets)
    @param  saltlen         The length of the salt used (octets)
    @param  hash_idx        The index of the hash desired
+   @param  mgf_hash_idx    The index of the hash desired for MGF1
    @param  modulus_bitlen  The bit length of the RSA modulus
    @param  res             [out] The result of the comparison, 1==valid, 0==invalid
    @return CRYPT_OK if successful (even if the comparison failed)
 */
-int pkcs_1_pss_decode(const unsigned char *msghash, unsigned long msghashlen,
-                      const unsigned char *sig,     unsigned long siglen,
-                            unsigned long saltlen,  int           hash_idx,
-                            unsigned long modulus_bitlen, int    *res)
+int pkcs_1_pss_decode_mgf1(const unsigned char *msghash, unsigned long msghashlen,
+                           const unsigned char *sig,     unsigned long siglen,
+                                 unsigned long saltlen,
+                                          int  hash_idx,           int mgf_hash_idx,
+                                 unsigned long modulus_bitlen,     int *res)
 {
    unsigned char *DB, *mask, *salt, *hash;
    unsigned long x, y, hLen, modulus_len;
@@ -41,6 +43,11 @@ int pkcs_1_pss_decode(const unsigned char *msghash, unsigned long msghashlen,
    if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) {
       return err;
    }
+   if (hash_idx != mgf_hash_idx) {
+      if ((err = hash_is_valid(mgf_hash_idx)) != CRYPT_OK) {
+         return err;
+      }
+   }
 
    hLen        = hash_descriptor[hash_idx].hashsize;
    modulus_bitlen--;
@@ -95,7 +102,7 @@ int pkcs_1_pss_decode(const unsigned char *msghash, unsigned long msghashlen,
    }
 
    /* generate mask of length modulus_len - hLen - 1 from hash */
-   if ((err = pkcs_1_mgf1(hash_idx, hash, hLen, mask, modulus_len - hLen - 1)) != CRYPT_OK) {
+   if ((err = pkcs_1_mgf1(mgf_hash_idx, hash, hLen, mask, modulus_len - hLen - 1)) != CRYPT_OK) {
       goto LBL_ERR;
    }
 
@@ -163,4 +170,25 @@ LBL_ERR:
    return err;
 }
 
+
+/**
+   PKCS #1 v2.00 PSS decode
+   @param  msghash         The hash to verify
+   @param  msghashlen      The length of the hash (octets)
+   @param  sig             The signature data (encoded data)
+   @param  siglen          The length of the signature data (octets)
+   @param  saltlen         The length of the salt used (octets)
+   @param  hash_idx        The index of the hash desired
+   @param  modulus_bitlen  The bit length of the RSA modulus
+   @param  res             [out] The result of the comparison, 1==valid, 0==invalid
+   @return CRYPT_OK if successful (even if the comparison failed)
+*/
+int pkcs_1_pss_decode(const unsigned char *msghash, unsigned long msghashlen,
+                      const unsigned char *sig,     unsigned long siglen,
+                            unsigned long saltlen,  int           hash_idx,
+                            unsigned long modulus_bitlen, int    *res)
+{
+   return pkcs_1_pss_decode_mgf1(msghash, msghashlen, sig, siglen, saltlen, hash_idx, hash_idx, modulus_bitlen, res);
+}
+
 #endif /* LTC_PKCS_1 */

src/pk/pkcs1/pkcs_1_pss_encode.c

@@ -17,16 +17,18 @@
    @param prng             An active PRNG context
    @param prng_idx         The index of the PRNG desired
    @param hash_idx         The index of the hash desired
+   @param mgf_hash_idx     The index of the hash desired for MGF1
    @param modulus_bitlen   The bit length of the RSA modulus
    @param out              [out] The destination of the encoding
    @param outlen           [in/out] The max size and resulting size of the encoded data
    @return CRYPT_OK if successful
 */
-int pkcs_1_pss_encode(const unsigned char *msghash, unsigned long msghashlen,
-                            unsigned long saltlen,  prng_state   *prng,
-                            int           prng_idx, int           hash_idx,
-                            unsigned long modulus_bitlen,
-                            unsigned char *out,     unsigned long *outlen)
+int pkcs_1_pss_encode_mgf1(const unsigned char *msghash,       unsigned long  msghashlen,
+                                 unsigned long saltlen,
+                                 prng_state    *prng,                    int  prng_idx,
+                                 int           hash_idx,                 int  mgf_hash_idx,
+                                 unsigned long modulus_bitlen,
+                                 unsigned char *out,           unsigned long *outlen)
 {
    unsigned char *DB, *mask, *salt, *hash;
    unsigned long x, y, hLen, modulus_len;
@@ -37,10 +39,15 @@ int pkcs_1_pss_encode(const unsigned char *msghash, unsigned long msghashlen,
    LTC_ARGCHK(out     != NULL);
    LTC_ARGCHK(outlen  != NULL);
 
-   /* ensure hash and PRNG are valid */
+   /* ensure hashes and PRNG are valid */
    if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) {
       return err;
    }
+   if (hash_idx != mgf_hash_idx) {
+      if ((err = hash_is_valid(mgf_hash_idx)) != CRYPT_OK) {
+         return err;
+      }
+   }
    if ((err = prng_is_valid(prng_idx)) != CRYPT_OK) {
       return err;
    }
@@ -111,7 +118,7 @@ int pkcs_1_pss_encode(const unsigned char *msghash, unsigned long msghashlen,
    /* x += saltlen; */
 
    /* generate mask of length modulus_len - hLen - 1 from hash */
-   if ((err = pkcs_1_mgf1(hash_idx, hash, hLen, mask, modulus_len - hLen - 1)) != CRYPT_OK) {
+   if ((err = pkcs_1_mgf1(mgf_hash_idx, hash, hLen, mask, modulus_len - hLen - 1)) != CRYPT_OK) {
       goto LBL_ERR;
    }
 
@@ -161,4 +168,27 @@ LBL_ERR:
    return err;
 }
 
+
+/**
+   PKCS #1 v2.00 Signature Encoding using MGF1 and both hashes are the same
+   @param msghash          The hash to encode
+   @param msghashlen       The length of the hash (octets)
+   @param saltlen          The length of the salt desired (octets)
+   @param prng             An active PRNG context
+   @param prng_idx         The index of the PRNG desired
+   @param hash_idx         The index of the hash desired
+   @param modulus_bitlen   The bit length of the RSA modulus
+   @param out              [out] The destination of the encoding
+   @param outlen           [in/out] The max size and resulting size of the encoded data
+   @return CRYPT_OK if successful
+*/
+int pkcs_1_pss_encode(const unsigned char *msghash,  unsigned long msghashlen,
+                            unsigned long saltlen,   prng_state   *prng,
+                            int           prng_idx,  int           hash_idx,
+                            unsigned long modulus_bitlen,
+                            unsigned char *out,      unsigned long *outlen)
+{
+   return pkcs_1_pss_encode_mgf1(msghash, msghashlen, saltlen, prng, prng_idx, hash_idx, hash_idx, modulus_bitlen, out, outlen);
+}
+
 #endif /* LTC_PKCS_1 */

src/pk/rsa/rsa_sign_hash.c

@@ -26,9 +26,10 @@
 int rsa_sign_hash_ex(const unsigned char *in,       unsigned long  inlen,
                            unsigned char *out,      unsigned long *outlen,
                            int            padding,
-                           prng_state    *prng,     int            prng_idx,
-                           int            hash_idx, unsigned long  saltlen,
-                     const rsa_key *key)
+                           prng_state    *prng,               int  prng_idx,
+                           int            hash_idx,           int  mgf_hash_idx,
+                           unsigned long  saltlen,
+                     const rsa_key       *key)
 {
    unsigned long modulus_bitlen, modulus_bytelen, x, y;
    int           err;
@@ -72,8 +73,8 @@ int rsa_sign_hash_ex(const unsigned char *in,       unsigned long  inlen,
   if (padding == LTC_PKCS_1_PSS) {
     /* PSS pad the key */
     x = *outlen;
-    if ((err = pkcs_1_pss_encode(in, inlen, saltlen, prng, prng_idx,
-                                 hash_idx, modulus_bitlen, out, &x)) != CRYPT_OK) {
+    if ((err = pkcs_1_pss_encode_mgf1(in, inlen, saltlen, prng, prng_idx,
+                                      hash_idx, mgf_hash_idx, modulus_bitlen, out, &x)) != CRYPT_OK) {
        return err;
     }
   } else {

src/pk/rsa/rsa_verify_hash.c

@@ -25,7 +25,8 @@
 int rsa_verify_hash_ex(const unsigned char *sig,            unsigned long  siglen,
                        const unsigned char *hash,           unsigned long  hashlen,
                              int            padding,
-                             int            hash_idx,       unsigned long  saltlen,
+                             int            hash_idx,                 int  mgf_hash_idx,
+                             unsigned long  saltlen,
                              int           *stat,     const rsa_key       *key)
 {
   unsigned long modulus_bitlen, modulus_bytelen, x;
@@ -87,10 +88,10 @@ int rsa_verify_hash_ex(const unsigned char *sig,            unsigned long  sigle
     /* PSS decode and verify it */
 
     if(modulus_bitlen%8 == 1){
-      err = pkcs_1_pss_decode(hash, hashlen, tmpbuf+1, x-1, saltlen, hash_idx, modulus_bitlen, stat);
+      err = pkcs_1_pss_decode_mgf1(hash, hashlen, tmpbuf+1, x-1, saltlen, hash_idx, mgf_hash_idx, modulus_bitlen, stat);
     }
     else{
-      err = pkcs_1_pss_decode(hash, hashlen, tmpbuf, x, saltlen, hash_idx, modulus_bitlen, stat);
+      err = pkcs_1_pss_decode_mgf1(hash, hashlen, tmpbuf, x, saltlen, hash_idx, mgf_hash_idx, modulus_bitlen, stat);
     }
 
   } else {

tests/pkcs_1_emsa_test.c

@@ -39,9 +39,9 @@ int pkcs_1_emsa_test(void)
         unsigned long buflen = sizeof(buf), obuflen = sizeof(obuf);
         int stat;
         DOX(hash_memory(hash_idx, s->o1, s->o1_l, buf, &buflen), s->name);
-        DOX(rsa_sign_hash_ex(buf, buflen, obuf, &obuflen, LTC_PKCS_1_V1_5, NULL, -1, hash_idx, 0, key), s->name);
+        DOX(rsa_sign_hash_ex(buf, buflen, obuf, &obuflen, LTC_PKCS_1_V1_5, NULL, -1, hash_idx, -1, 0, key), s->name);
         COMPARE_TESTVECTOR(obuf, obuflen, s->o2, s->o2_l,s->name, j);
-        DOX(rsa_verify_hash_ex(obuf, obuflen, buf, buflen, LTC_PKCS_1_V1_5, hash_idx, 0, &stat, key), s->name);
+        DOX(rsa_verify_hash_ex(obuf, obuflen, buf, buflen, LTC_PKCS_1_V1_5, hash_idx, -1, 0, &stat, key), s->name);
         DOX(stat == 1?CRYPT_OK:CRYPT_FAIL_TESTVECTOR, s->name);
     } /* for */
 

tests/rsa_test.c

@@ -191,13 +191,13 @@ static int rsa_compat_test(void)
 
    /* sign-verify a message with PKCS #1 v1.5 no ASN.1 */
    len = sizeof(buf);
-   DO(rsa_sign_hash_ex((unsigned char*)"test", 4, buf, &len, LTC_PKCS_1_V1_5_NA1, NULL, 0, 0, 0, &key));
+   DO(rsa_sign_hash_ex((unsigned char*)"test", 4, buf, &len, LTC_PKCS_1_V1_5_NA1, NULL, 0, 0, 0, 0, &key));
    if (len != sizeof(openssl_rsautl_pkcs) || memcmp(buf, openssl_rsautl_pkcs, len)) {
       fprintf(stderr, "RSA rsa_sign_hash_ex + LTC_PKCS_1_V1_5_NA1 failed\n");
       return 1;
    }
    stat = 0;
-   DO(rsa_verify_hash_ex(openssl_rsautl_pkcs, sizeof(openssl_rsautl_pkcs), (unsigned char*)"test", 4, LTC_PKCS_1_V1_5_NA1, 0, 0, &stat, &pubkey));
+   DO(rsa_verify_hash_ex(openssl_rsautl_pkcs, sizeof(openssl_rsautl_pkcs), (unsigned char*)"test", 4, LTC_PKCS_1_V1_5_NA1, 0, 0, 0, &stat, &pubkey));
    if (stat != 1) {
       fprintf(stderr, "RSA rsa_verify_hash_ex + LTC_PKCS_1_V1_5_NA1 failed\n");
       return 1;
@@ -331,9 +331,9 @@ static int s_rsa_cryptx_issue_69(void)
    l1 = sizeof(buf1);
    DO(radix_to_bin(sig1, 16, buf0, &l0));
    DO(radix_to_bin(hash, 16, buf1, &l1));
-   SHOULD_FAIL(rsa_verify_hash_ex(buf0, l0, buf1, l1, LTC_PKCS_1_V1_5, 0, 0, &stat, &key));
+   SHOULD_FAIL(rsa_verify_hash_ex(buf0, l0, buf1, l1, LTC_PKCS_1_V1_5, 0, 0, 0, &stat, &key));
    DO(radix_to_bin(sig2, 16, buf0, &l0));
-   SHOULD_FAIL(rsa_verify_hash_ex(buf0, l0, buf1, l1, LTC_PKCS_1_V1_5, 0, 0, &stat, &key));
+   SHOULD_FAIL(rsa_verify_hash_ex(buf0, l0, buf1, l1, LTC_PKCS_1_V1_5, 0, 0, 0, &stat, &key));
    rsa_free(&key);
    return CRYPT_OK;
 }
@@ -684,11 +684,11 @@ print_hex("q", tmp, len);
 
    /* sign a message with PKCS #1 v1.5 */
    len = sizeof(out);
-   DO(rsa_sign_hash_ex(in, 20, out, &len, LTC_PKCS_1_V1_5, &yarrow_prng, prng_idx, hash_idx, 8, &privKey));
-   DO(rsa_verify_hash_ex(out, len, in, 20, LTC_PKCS_1_V1_5, hash_idx, 8, &stat, &pubKey));
+   DO(rsa_sign_hash_ex(in, 20, out, &len, LTC_PKCS_1_V1_5, &yarrow_prng, prng_idx, hash_idx, 0, 8, &privKey));
+   DO(rsa_verify_hash_ex(out, len, in, 20, LTC_PKCS_1_V1_5, hash_idx, 0, 8, &stat, &pubKey));
    /* change a byte */
    in[0] ^= 1;
-   DO(rsa_verify_hash_ex(out, len, in, 20, LTC_PKCS_1_V1_5, hash_idx, 8, &stat2, &pubKey));
+   DO(rsa_verify_hash_ex(out, len, in, 20, LTC_PKCS_1_V1_5, hash_idx, 0, 8, &stat2, &pubKey));
 
    if (!(stat == 1 && stat2 == 0)) {
       fprintf(stderr, "rsa_verify_hash_ex failed, %d, %d", stat, stat2);
@@ -721,9 +721,9 @@ print_hex("q", tmp, len);
      len = sizeof(in);
      len2 = sizeof(out);
      /* (1) */
-     DO(rsa_sign_hash_ex(p, 20, p2, &len2, LTC_PKCS_1_V1_5, &yarrow_prng, prng_idx, hash_idx, 8, &privKey));
+     DO(rsa_sign_hash_ex(p, 20, p2, &len2, LTC_PKCS_1_V1_5, &yarrow_prng, prng_idx, hash_idx, 0, 8, &privKey));
      /* (2) */
-     DOX(rsa_verify_hash_ex(p2, len2, p, 20, LTC_PKCS_1_V1_5, hash_idx, -1, &stat, &pubKey), "should succeed");
+     DOX(rsa_verify_hash_ex(p2, len2, p, 20, LTC_PKCS_1_V1_5, hash_idx, 0, -1, &stat, &pubKey), "should succeed");
      DOX(stat == 1?CRYPT_OK:CRYPT_FAIL_TESTVECTOR, "should succeed");
      len3 = sizeof(tmp);
      /* (3) */
@@ -757,7 +757,7 @@ print_hex("q", tmp, len);
 
      len3 = sizeof(tmp);
      /* (6) */
-     SHOULD_FAIL(rsa_verify_hash_ex(p2, len2, p, 20, LTC_PKCS_1_V1_5, hash_idx, -1, &stat, &pubKey));
+     SHOULD_FAIL(rsa_verify_hash_ex(p2, len2, p, 20, LTC_PKCS_1_V1_5, hash_idx, -1, -1, &stat, &pubKey));
      DOX(stat == 0?CRYPT_OK:CRYPT_FAIL_TESTVECTOR, "should fail");
    }
    rsa_free(&key);