Merge pull request #152 from libtom/pr/wchar-troubles-max-FFFF

better handling wchar_t when __WCHAR_MAX__ is 0xFFFF (2 bytes only)

src/headers/tomcrypt_pk.h

@@ -619,6 +619,7 @@ int der_encode_utf8_string(const wchar_t *in,  unsigned long inlen,
 int der_decode_utf8_string(const unsigned char *in,  unsigned long inlen,
                                        wchar_t *out, unsigned long *outlen);
 unsigned long der_utf8_charsize(const wchar_t c);
+int der_utf8_valid_char(const wchar_t c);
 int der_length_utf8_string(const wchar_t *in, unsigned long noctets, unsigned long *outlen);
 
 

src/pk/asn1/der/utf8/der_encode_utf8_string.c

@@ -37,9 +37,7 @@ int der_encode_utf8_string(const wchar_t *in,  unsigned long inlen,
 
    /* get the size */
    for (x = len = 0; x < inlen; x++) {
-       if (in[x] < 0 || in[x] > 0x1FFFF) {
-          return CRYPT_INVALID_ARG;
-       }
+       if (!der_utf8_valid_char(in[x])) return CRYPT_INVALID_ARG;
        len += der_utf8_charsize(in[x]);
    }
 

src/pk/asn1/der/utf8/der_length_utf8_string.c

@@ -27,11 +27,33 @@ unsigned long der_utf8_charsize(const wchar_t c)
       return 1;
    } else if (c <= 0x7FF) {
       return 2;
+#if __WCHAR_MAX__ == 0xFFFF
+   } else {
+      return 3;
+   }
+#else
    } else if (c <= 0xFFFF) {
       return 3;
    } else {
       return 4;
    }
+#endif
+}
+
+/**
+  Test whether the given code point is valid character
+  @param c   The UTF-8 character to test
+  @return    1 - valid, 0 - invalid
+*/
+int der_utf8_valid_char(const wchar_t c)
+{
+#if !defined(__WCHAR_MAX__) || __WCHAR_MAX__ > 0xFFFF
+   if (in[x] > 0x10FFFF) return 0;
+#endif
+#if !defined(__WCHAR_MAX__) || __WCHAR_MAX__ != 0xFFFF && __WCHAR_MAX__ != 0xFFFFFFFF
+   if (in[x] < 0) return 0;
+#endif
+   return 1;
 }
 
 /**
@@ -50,9 +72,7 @@ int der_length_utf8_string(const wchar_t *in, unsigned long noctets, unsigned lo
 
    len = 0;
    for (x = 0; x < noctets; x++) {
-      if (in[x] < 0 || in[x] > 0x10FFFF) {
-         return CRYPT_INVALID_ARG;
-      }
+      if (!der_utf8_valid_char(in[x])) return CRYPT_INVALID_ARG;
       len += der_utf8_charsize(in[x]);
    }