Fixed small padding error in the PKCS#1 PSS code.

The existing LTC code for padding meassages for PSS signatures
contained a small error. In particular, the PSS-passing algorithms is
supposed to be given (bitlength of key - 1) as an argument. The LTC
code passes (bitlength of key), and subtracts 1 in the middle of the
PSS-padding. This subtraction unfortunately comes too late: a
calculation using that argument has already been made. Fortunately,
this bug only appeared if the bit-length of the key was 1 mod 8, and
so is unlikely to show up in practice. Still, this patch fixes the
problem.

Conflicts:
	src/pk/pkcs1/pkcs_1_pss_decode.c

src/pk/pkcs1/pkcs_1_pss_decode.c

@@ -51,11 +51,12 @@ int pkcs_1_pss_decode(const unsigned char *msghash, unsigned long msghashlen,
    }
 
    hLen        = hash_descriptor[hash_idx].hashsize;
+   modulus_bitlen--;
    modulus_len = (modulus_bitlen>>3) + (modulus_bitlen & 7 ? 1 : 0);
 
    /* check sizes */
    if ((saltlen > modulus_len) ||
-       (modulus_len < hLen + saltlen + 2) || (siglen != modulus_len)) {
+       (modulus_len < hLen + saltlen + 2)) {
       return CRYPT_PK_INVALID_SIZE;
    }
 
@@ -95,8 +96,9 @@ int pkcs_1_pss_decode(const unsigned char *msghash, unsigned long msghashlen,
    XMEMCPY(hash, sig + x, hLen);
    x += hLen;
 
+
    /* check the MSB */
-   if ((sig[0] & ~(0xFF >> ((modulus_len<<3) - (modulus_bitlen-1)))) != 0) {
+   if ((sig[0] & ~(0xFF >> ((modulus_len<<3) - (modulus_bitlen)))) != 0) {
       err = CRYPT_INVALID_PACKET;
       goto LBL_ERR;
    }
@@ -112,7 +114,7 @@ int pkcs_1_pss_decode(const unsigned char *msghash, unsigned long msghashlen,
    }
 
    /* now clear the first byte [make sure smaller than modulus] */
-   DB[0] &= 0xFF >> ((modulus_len<<3) - (modulus_bitlen-1));
+   DB[0] &= 0xFF >> ((modulus_len<<3) - (modulus_bitlen));
 
    /* DB = PS || 0x01 || salt, PS == modulus_len - saltlen - hLen - 2 zero bytes */
 

src/pk/pkcs1/pkcs_1_pss_encode.c

@@ -54,6 +54,7 @@ int pkcs_1_pss_encode(const unsigned char *msghash, unsigned long msghashlen,
    }
 
    hLen        = hash_descriptor[hash_idx].hashsize;
+   modulus_bitlen--;
    modulus_len = (modulus_bitlen>>3) + (modulus_bitlen & 7 ? 1 : 0);
 
    /* check sizes */
@@ -147,7 +148,7 @@ int pkcs_1_pss_encode(const unsigned char *msghash, unsigned long msghashlen,
    out[y] = 0xBC;
 
    /* now clear the 8*modulus_len - modulus_bitlen most significant bits */
-   out[0] &= 0xFF >> ((modulus_len<<3) - (modulus_bitlen-1));
+   out[0] &= 0xFF >> ((modulus_len<<3) - modulus_bitlen);
 
    /* store output size */
    *outlen = modulus_len;

src/pk/rsa/rsa_verify_hash.c

@@ -92,7 +92,14 @@ int rsa_verify_hash_ex(const unsigned char *sig,      unsigned long siglen,
 
   if (padding == LTC_PKCS_1_PSS) {
     /* PSS decode and verify it */
-    err = pkcs_1_pss_decode(hash, hashlen, tmpbuf, x, saltlen, hash_idx, modulus_bitlen, stat);
+
+    if(modulus_bitlen%8 == 1){
+      err = pkcs_1_pss_decode(hash, hashlen, tmpbuf+1, x-1, saltlen, hash_idx, modulus_bitlen, stat);
+    }
+    else{
+      err = pkcs_1_pss_decode(hash, hashlen, tmpbuf, x, saltlen, hash_idx, modulus_bitlen, stat);
+    }
+
   } else {
     /* PKCS #1 v1.5 decode it */
     unsigned char *out;