Merge pull request #277 from libtom/pr/fix-rsa-wycheproof

RSA wycheproof tests

src/pk/asn1/der/sequence/der_decode_sequence_ex.c

@@ -310,7 +310,12 @@ int der_decode_sequence_ex(const unsigned char *in, unsigned long  inlen,
           goto LBL_ERR;
       }
    }
-   err = CRYPT_OK;
+
+   if (inlen == 0) {
+      err = CRYPT_OK;
+   } else {
+      err = CRYPT_PK_INVALID_SIZE;
+   }
 
 LBL_ERR:
    return err;

src/pk/dh/dh_import.c

@@ -37,7 +37,7 @@ int dh_import(const unsigned char *in, unsigned long inlen, dh_key *key)
                                    LTC_ASN1_SHORT_INTEGER, 1UL, &version,
                                    LTC_ASN1_BIT_STRING, 1UL, &flags,
                                    LTC_ASN1_EOL, 0UL, NULL);
-   if (err != CRYPT_OK) {
+   if (err != CRYPT_OK && err != CRYPT_PK_INVALID_SIZE) {
       goto error;
    }
 

src/pk/dsa/dsa_decrypt_key.c

@@ -46,8 +46,8 @@ int dsa_decrypt_key(const unsigned char *in,  unsigned long  inlen,
 
    /* decode to find out hash */
    LTC_SET_ASN1(decode, 0, LTC_ASN1_OBJECT_IDENTIFIER, hashOID, sizeof(hashOID)/sizeof(hashOID[0]));
-
-   if ((err = der_decode_sequence(in, inlen, decode, 1)) != CRYPT_OK) {
+   err = der_decode_sequence(in, inlen, decode, 1);
+   if (err != CRYPT_OK && err != CRYPT_PK_INVALID_SIZE) {
       return err;
    }
 

src/pk/dsa/dsa_import.c

@@ -39,9 +39,10 @@ int dsa_import(const unsigned char *in, unsigned long inlen, dsa_key *key)
    }
 
    /* try to match the old libtomcrypt format */
-   if ((err = der_decode_sequence_multi(in, inlen,
-                                  LTC_ASN1_BIT_STRING, 1UL, flags,
-                                  LTC_ASN1_EOL, 0UL, NULL)) == CRYPT_OK) {
+   err = der_decode_sequence_multi(in, inlen, LTC_ASN1_BIT_STRING, 1UL, flags,
+                                              LTC_ASN1_EOL,        0UL, NULL);
+
+   if (err == CRYPT_OK || err == CRYPT_PK_INVALID_SIZE) {
        /* private key */
        if (flags[0]) {
            if ((err = der_decode_sequence_multi(in, inlen,

src/pk/ecc/ecc_decrypt_key.c

@@ -52,8 +52,8 @@ int ecc_decrypt_key(const unsigned char *in,  unsigned long  inlen,
 
    /* decode to find out hash */
    LTC_SET_ASN1(decode, 0, LTC_ASN1_OBJECT_IDENTIFIER, hashOID, sizeof(hashOID)/sizeof(hashOID[0]));
-
-   if ((err = der_decode_sequence(in, inlen, decode, 1)) != CRYPT_OK) {
+   err = der_decode_sequence(in, inlen, decode, 1);
+   if (err != CRYPT_OK && err != CRYPT_PK_INVALID_SIZE) {
       return err;
    }
 

src/pk/ecc/ecc_import.c

@@ -105,9 +105,9 @@ int ecc_import_ex(const unsigned char *in, unsigned long inlen, ecc_key *key, co
    }
 
    /* find out what type of key it is */
-   if ((err = der_decode_sequence_multi(in, inlen,
-                                  LTC_ASN1_BIT_STRING, 1UL, flags,
-                                  LTC_ASN1_EOL,        0UL, NULL)) != CRYPT_OK) {
+   err = der_decode_sequence_multi(in, inlen, LTC_ASN1_BIT_STRING, 1UL, flags,
+                                              LTC_ASN1_EOL,        0UL, NULL);
+   if (err != CRYPT_OK && err != CRYPT_PK_INVALID_SIZE) {
       goto done;
    }
 

src/pk/rsa/rsa_import.c

@@ -66,9 +66,10 @@ int rsa_import(const unsigned char *in, unsigned long inlen, rsa_key *key)
    }
 
    /* not SSL public key, try to match against PKCS #1 standards */
-   if ((err = der_decode_sequence_multi(in, inlen,
-                                  LTC_ASN1_INTEGER, 1UL, key->N,
-                                  LTC_ASN1_EOL,     0UL, NULL)) != CRYPT_OK) {
+   err = der_decode_sequence_multi(in, inlen, LTC_ASN1_INTEGER, 1UL, key->N,
+                                              LTC_ASN1_EOL,     0UL, NULL);
+
+   if (err != CRYPT_OK && err != CRYPT_PK_INVALID_SIZE) {
       goto LBL_ERR;
    }
 

src/pk/rsa/rsa_verify_hash.c

@@ -143,8 +143,12 @@ int rsa_verify_hash_ex(const unsigned char *sig,      unsigned long siglen,
       LTC_SET_ASN1(siginfo,    1, LTC_ASN1_OCTET_STRING,      tmpbuf,                        siglen);
 
       if ((err = der_decode_sequence(out, outlen, siginfo, 2)) != CRYPT_OK) {
-         XFREE(out);
-         goto bail_2;
+         /* fallback to Legacy:missing NULL */
+         LTC_SET_ASN1(siginfo, 0, LTC_ASN1_SEQUENCE,          digestinfo,                    1);
+         if ((err = der_decode_sequence(out, outlen, siginfo, 2)) != CRYPT_OK) {
+           XFREE(out);
+           goto bail_2;
+         }
       }
 
       if ((err = der_length_sequence(siginfo, 2, &reallen)) != CRYPT_OK) {