Fix returning too many or too few bits

original patch by @friedrichsenm

This closes #543

src/math/rand_bn.c

@@ -19,7 +19,7 @@ int rand_bn_bits(void *N, int bits, prng_state *prng, int wprng)
    if ((res = prng_is_valid(wprng)) != CRYPT_OK) return res;
 
    bytes = (bits+7) >> 3;
-   mask = 0xff << (8 - bits % 8);
+   mask = 0xff >> (bits % 8 == 0 ? 0 : 8 - bits % 8);
 
    /* allocate buffer */
    if ((buf = XCALLOC(1, bytes)) == NULL) return CRYPT_MEM;
@@ -30,7 +30,7 @@ int rand_bn_bits(void *N, int bits, prng_state *prng, int wprng)
       goto cleanup;
    }
    /* mask bits */
-   buf[0] &= ~mask;
+   buf[0] &= mask;
    /* load value */
    if ((res = mp_read_unsigned_bin(N, buf, bytes)) != CRYPT_OK) goto cleanup;