ltc: ctr: improve performance

When accel_ctr_encrypt() is not used, accel_ecb_encrypt() is used via
ecb_encrypt() instead. The accel_ecb_encrypt() is frequently called at
every single block process. VFP assembly code called from the
accel_ecb_encrypt() is protected by
tomcrypt_arm_neon_enable()/disable(). FIQ enable/disable and VFP
register save/restore (64bitx32 registers!) to/from memory are done in
the tomcrypt_arm_neon_enable()/disable(). These overhead exist in each
single block process cause the degradation of system performance
eventually. Cases where h/w accelerated AES-CTR did not show any effects
or showed less performance than pure software processing have been
observed.

This patch resolves the issue by increasing utilization rate of
accel_ctr_encrypt().

Signed-off-by: Tetsuya Yoshizaki <[email protected]>
Signed-off-by: Victor Chong <[email protected]>

src/modes/ctr/ctr_encrypt.c

@@ -17,47 +17,17 @@
 #ifdef LTC_CTR_MODE
 
 /**
-  CTR encrypt
+  CTR encrypt sub
   @param pt     Plaintext
   @param ct     [out] Ciphertext
   @param len    Length of plaintext (octets)
   @param ctr    CTR state
   @return CRYPT_OK if successful
 */
-int ctr_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_CTR *ctr)
+static int ctr_encrypt_sub(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_CTR *ctr)
 {
    int x, err;
 
-   LTC_ARGCHK(pt != NULL);
-   LTC_ARGCHK(ct != NULL);
-   LTC_ARGCHK(ctr != NULL);
-
-   if ((err = cipher_is_valid(ctr->cipher)) != CRYPT_OK) {
-       return err;
-   }
-
-   /* is blocklen/padlen valid? */
-   if (ctr->blocklen < 1 || ctr->blocklen > (int)sizeof(ctr->ctr) ||
-       ctr->padlen   < 0 || ctr->padlen   > (int)sizeof(ctr->pad)) {
-      return CRYPT_INVALID_ARG;
-   }
-
-#ifdef LTC_FAST
-   if (ctr->blocklen % sizeof(LTC_FAST_TYPE)) {
-      return CRYPT_INVALID_ARG;
-   }
-#endif
-
-   /* handle acceleration only if pad is empty, accelerator is present and length is >= a block size */
-   if ((ctr->padlen == ctr->blocklen) && cipher_descriptor[ctr->cipher].accel_ctr_encrypt != NULL && (len >= (unsigned long)ctr->blocklen)) {
-      if ((err = cipher_descriptor[ctr->cipher].accel_ctr_encrypt(pt, ct, len/ctr->blocklen, ctr->ctr, ctr->mode, &ctr->key)) != CRYPT_OK) {
-         return err;
-      }
-      pt += (len / ctr->blocklen) * ctr->blocklen;
-      ct += (len / ctr->blocklen) * ctr->blocklen;
-      len %= ctr->blocklen;
-   }
-
    while (len) {
       /* is the pad empty? */
       if (ctr->padlen == ctr->blocklen) {
@@ -105,6 +75,63 @@ int ctr_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, s
    return CRYPT_OK;
 }
 
+/**
+  CTR encrypt
+  @param pt     Plaintext
+  @param ct     [out] Ciphertext
+  @param len    Length of plaintext (octets)
+  @param ctr    CTR state
+  @return CRYPT_OK if successful
+*/
+int ctr_encrypt(const unsigned char *pt, unsigned char *ct, unsigned long len, symmetric_CTR *ctr)
+{
+   int err, fr;
+
+   LTC_ARGCHK(pt != NULL);
+   LTC_ARGCHK(ct != NULL);
+   LTC_ARGCHK(ctr != NULL);
+
+   if ((err = cipher_is_valid(ctr->cipher)) != CRYPT_OK) {
+       return err;
+   }
+
+   /* is blocklen/padlen valid? */
+   if (ctr->blocklen < 1 || ctr->blocklen > (int)sizeof(ctr->ctr) ||
+       ctr->padlen   < 0 || ctr->padlen   > (int)sizeof(ctr->pad)) {
+      return CRYPT_INVALID_ARG;
+   }
+
+#ifdef LTC_FAST
+   if (ctr->blocklen % sizeof(LTC_FAST_TYPE)) {
+      return CRYPT_INVALID_ARG;
+   }
+#endif
+
+   /* handle acceleration only if pad is empty, accelerator is present and length is >= a block size */
+   if (cipher_descriptor[ctr->cipher].accel_ctr_encrypt != NULL && len >= (unsigned long)ctr->blocklen) {
+     if (ctr->padlen < ctr->blocklen) {
+       fr = ctr->blocklen - ctr->padlen;
+       if ((err = ctr_encrypt_sub(pt, ct, fr, ctr)) != CRYPT_OK) {
+	 return err;
+       }
+       pt += fr;
+       ct += fr;
+       len -= fr;
+     }
+
+     if (len >= (unsigned long)ctr->blocklen) {
+       if ((err = cipher_descriptor[ctr->cipher].accel_ctr_encrypt(pt, ct, len/ctr->blocklen, ctr->ctr, ctr->mode, &ctr->key)) != CRYPT_OK) {
+       return err;
+       }
+       pt += (len / ctr->blocklen) * ctr->blocklen;
+       ct += (len / ctr->blocklen) * ctr->blocklen;
+       len %= ctr->blocklen;
+     }
+   }
+
+   return ctr_encrypt_sub(pt, ct, len, ctr);
+}
+
 #endif
 
 /* ref:         $Format:%D$ */