Ensure that hash overflow is detected

Previously it was not detected if `inlen` itself was too big and would
overflow the multiplication by 8.

Related to #592

Signed-off-by: Steffen Jaeckel <[email protected]>

src/headers/tomcrypt_private.h

@@ -90,7 +90,8 @@ int func_name (hash_state * md, const unsigned char *in, unsigned long inlen)
     if (md-> state_var .curlen > sizeof(md-> state_var .buf)) {                             \
        return CRYPT_INVALID_ARG;                                                            \
     }                                                                                       \
-    if ((md-> state_var .length + inlen * 8) < md-> state_var .length) {                        \
+    if (((md-> state_var .length + inlen * 8) < md-> state_var .length)                     \
+          || ((inlen * 8) < inlen)) {                                                       \
       return CRYPT_HASH_OVERFLOW;                                                           \
     }                                                                                       \
     while (inlen > 0) {                                                                     \

tests/multi_test.c

@@ -15,6 +15,15 @@ int multi_test(void)
 
 /* HASH testing */
    len = sizeof(buf[0]);
+#if defined(ENDIAN_32BITWORD) || defined(_MSC_VER)
+   len2 = 0x80000000UL;
+#else
+   /* Check against the max. input limit of SHA-1 as of RFC8017 */
+   len2 = 0x1ULL << 61;
+#endif
+   SHOULD_FAIL_WITH(hash_memory(find_hash("sha256"), buf[0], len2, buf[0], &len), CRYPT_HASH_OVERFLOW);
+
+   len = sizeof(buf[0]);
    hash_memory(find_hash("sha256"), (unsigned char*)"hello", 5, buf[0], &len);
    len2 = sizeof(buf[0]);
    hash_memory_multi(find_hash("sha256"), buf[1], &len2, (unsigned char*)"hello", 5, NULL);