arm64/crypto: use crypto instructions to generate AES key schedule

[firefly-linux-kernel-4.4.55.git] / arch / arm64 / crypto / aes-ce-cipher.c

diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-cipher.c
index 2075e1acae6b745ed9b8de2a828f449cfb8f08fa..ce47792a983dda8b2909609502ef80d54c1215f8 100644 (file)
--- a/arch/arm64/crypto/aes-ce-cipher.c
+++ b/arch/arm64/crypto/aes-ce-cipher.c
@@ -14,6 +14,8 @@
 #include <linux/crypto.h>
 #include <linux/module.h>
 
+#include "aes-ce-setkey.h"
+
 MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 MODULE_LICENSE("GPL v2");
@@ -124,6 +126,114 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
        kernel_neon_end();
 }
 
+/*
+ * aes_sub() - use the aese instruction to perform the AES sbox substitution
+ *             on each byte in 'input'
+ */
+static u32 aes_sub(u32 input)
+{
+       u32 ret;
+
+       __asm__("dup    v1.4s, %w[in]           ;"
+               "movi   v0.16b, #0              ;"
+               "aese   v0.16b, v1.16b          ;"
+               "umov   %w[out], v0.4s[0]       ;"
+
+       :       [out]   "=r"(ret)
+       :       [in]    "r"(input)
+       :               "v0","v1");
+
+       return ret;
+}
+
+int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
+                    unsigned int key_len)
+{
+       /*
+        * The AES key schedule round constants
+        */
+       static u8 const rcon[] = {
+               0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
+       };
+
+       u32 kwords = key_len / sizeof(u32);
+       struct aes_block *key_enc, *key_dec;
+       int i, j;
+
+       if (key_len != AES_KEYSIZE_128 &&
+           key_len != AES_KEYSIZE_192 &&
+           key_len != AES_KEYSIZE_256)
+               return -EINVAL;
+
+       memcpy(ctx->key_enc, in_key, key_len);
+       ctx->key_length = key_len;
+
+       kernel_neon_begin_partial(2);
+       for (i = 0; i < sizeof(rcon); i++) {
+               u32 *rki = ctx->key_enc + (i * kwords);
+               u32 *rko = rki + kwords;
+
+               rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];
+               rko[1] = rko[0] ^ rki[1];
+               rko[2] = rko[1] ^ rki[2];
+               rko[3] = rko[2] ^ rki[3];
+
+               if (key_len == AES_KEYSIZE_192) {
+                       if (i >= 7)
+                               break;
+                       rko[4] = rko[3] ^ rki[4];
+                       rko[5] = rko[4] ^ rki[5];
+               } else if (key_len == AES_KEYSIZE_256) {
+                       if (i >= 6)
+                               break;
+                       rko[4] = aes_sub(rko[3]) ^ rki[4];
+                       rko[5] = rko[4] ^ rki[5];
+                       rko[6] = rko[5] ^ rki[6];
+                       rko[7] = rko[6] ^ rki[7];
+               }
+       }
+
+       /*
+        * Generate the decryption keys for the Equivalent Inverse Cipher.
+        * This involves reversing the order of the round keys, and applying
+        * the Inverse Mix Columns transformation on all but the first and
+        * the last one.
+        */
+       key_enc = (struct aes_block *)ctx->key_enc;
+       key_dec = (struct aes_block *)ctx->key_dec;
+       j = num_rounds(ctx);
+
+       key_dec[0] = key_enc[j];
+       for (i = 1, j--; j > 0; i++, j--)
+               __asm__("ld1    {v0.16b}, %[in]         ;"
+                       "aesimc v1.16b, v0.16b          ;"
+                       "st1    {v1.16b}, %[out]        ;"
+
+               :       [out]   "=Q"(key_dec[i])
+               :       [in]    "Q"(key_enc[j])
+               :               "v0","v1");
+       key_dec[i] = key_enc[0];
+
+       kernel_neon_end();
+       return 0;
+}
+EXPORT_SYMBOL(ce_aes_expandkey);
+
+int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
+                 unsigned int key_len)
+{
+       struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+       int ret;
+
+       ret = ce_aes_expandkey(ctx, in_key, key_len);
+       if (!ret)
+               return 0;
+
+       tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+       return -EINVAL;
+}
+EXPORT_SYMBOL(ce_aes_setkey);
+
 static struct crypto_alg aes_alg = {
        .cra_name               = "aes",
        .cra_driver_name        = "aes-ce",
@@ -135,7 +245,7 @@ static struct crypto_alg aes_alg = {
        .cra_cipher = {
                .cia_min_keysize        = AES_MIN_KEY_SIZE,
                .cia_max_keysize        = AES_MAX_KEY_SIZE,
-               .cia_setkey             = crypto_aes_set_key,
+               .cia_setkey             = ce_aes_setkey,
                .cia_encrypt            = aes_cipher_encrypt,
                .cia_decrypt            = aes_cipher_decrypt
        }