arm64/crypto: AES using ARMv8 Crypto Extensions

[firefly-linux-kernel-4.4.55.git] / arch / arm64 / crypto / aes-ce-cipher.c

diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-cipher.c
new file mode 100644 (file)
index 0000000..2075e1a
--- /dev/null
+++ b/arch/arm64/crypto/aes-ce-cipher.c
@@ -0,0 +1,155 @@
+/*
+ * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions
+ *
+ * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/neon.h>
+#include <crypto/aes.h>
+#include <linux/cpufeature.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+struct aes_block {
+       u8 b[AES_BLOCK_SIZE];
+};
+
+static int num_rounds(struct crypto_aes_ctx *ctx)
+{
+       /*
+        * # of rounds specified by AES:
+        * 128 bit key          10 rounds
+        * 192 bit key          12 rounds
+        * 256 bit key          14 rounds
+        * => n byte key        => 6 + (n/4) rounds
+        */
+       return 6 + ctx->key_length / 4;
+}
+
+static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
+{
+       struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+       struct aes_block *out = (struct aes_block *)dst;
+       struct aes_block const *in = (struct aes_block *)src;
+       void *dummy0;
+       int dummy1;
+
+       kernel_neon_begin_partial(4);
+
+       __asm__("       ld1     {v0.16b}, %[in]                 ;"
+               "       ld1     {v1.2d}, [%[key]], #16          ;"
+               "       cmp     %w[rounds], #10                 ;"
+               "       bmi     0f                              ;"
+               "       bne     3f                              ;"
+               "       mov     v3.16b, v1.16b                  ;"
+               "       b       2f                              ;"
+               "0:     mov     v2.16b, v1.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "1:     aese    v0.16b, v2.16b                  ;"
+               "       aesmc   v0.16b, v0.16b                  ;"
+               "2:     ld1     {v1.2d}, [%[key]], #16          ;"
+               "       aese    v0.16b, v3.16b                  ;"
+               "       aesmc   v0.16b, v0.16b                  ;"
+               "3:     ld1     {v2.2d}, [%[key]], #16          ;"
+               "       subs    %w[rounds], %w[rounds], #3      ;"
+               "       aese    v0.16b, v1.16b                  ;"
+               "       aesmc   v0.16b, v0.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "       bpl     1b                              ;"
+               "       aese    v0.16b, v2.16b                  ;"
+               "       eor     v0.16b, v0.16b, v3.16b          ;"
+               "       st1     {v0.16b}, %[out]                ;"
+
+       :       [out]           "=Q"(*out),
+               [key]           "=r"(dummy0),
+               [rounds]        "=r"(dummy1)
+       :       [in]            "Q"(*in),
+                               "1"(ctx->key_enc),
+                               "2"(num_rounds(ctx) - 2)
+       :       "cc");
+
+       kernel_neon_end();
+}
+
+static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
+{
+       struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+       struct aes_block *out = (struct aes_block *)dst;
+       struct aes_block const *in = (struct aes_block *)src;
+       void *dummy0;
+       int dummy1;
+
+       kernel_neon_begin_partial(4);
+
+       __asm__("       ld1     {v0.16b}, %[in]                 ;"
+               "       ld1     {v1.2d}, [%[key]], #16          ;"
+               "       cmp     %w[rounds], #10                 ;"
+               "       bmi     0f                              ;"
+               "       bne     3f                              ;"
+               "       mov     v3.16b, v1.16b                  ;"
+               "       b       2f                              ;"
+               "0:     mov     v2.16b, v1.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "1:     aesd    v0.16b, v2.16b                  ;"
+               "       aesimc  v0.16b, v0.16b                  ;"
+               "2:     ld1     {v1.2d}, [%[key]], #16          ;"
+               "       aesd    v0.16b, v3.16b                  ;"
+               "       aesimc  v0.16b, v0.16b                  ;"
+               "3:     ld1     {v2.2d}, [%[key]], #16          ;"
+               "       subs    %w[rounds], %w[rounds], #3      ;"
+               "       aesd    v0.16b, v1.16b                  ;"
+               "       aesimc  v0.16b, v0.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "       bpl     1b                              ;"
+               "       aesd    v0.16b, v2.16b                  ;"
+               "       eor     v0.16b, v0.16b, v3.16b          ;"
+               "       st1     {v0.16b}, %[out]                ;"
+
+       :       [out]           "=Q"(*out),
+               [key]           "=r"(dummy0),
+               [rounds]        "=r"(dummy1)
+       :       [in]            "Q"(*in),
+                               "1"(ctx->key_dec),
+                               "2"(num_rounds(ctx) - 2)
+       :       "cc");
+
+       kernel_neon_end();
+}
+
+static struct crypto_alg aes_alg = {
+       .cra_name               = "aes",
+       .cra_driver_name        = "aes-ce",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
+       .cra_blocksize          = AES_BLOCK_SIZE,
+       .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
+       .cra_module             = THIS_MODULE,
+       .cra_cipher = {
+               .cia_min_keysize        = AES_MIN_KEY_SIZE,
+               .cia_max_keysize        = AES_MAX_KEY_SIZE,
+               .cia_setkey             = crypto_aes_set_key,
+               .cia_encrypt            = aes_cipher_encrypt,
+               .cia_decrypt            = aes_cipher_decrypt
+       }
+};
+
+static int __init aes_mod_init(void)
+{
+       return crypto_register_alg(&aes_alg);
+}
+
+static void __exit aes_mod_exit(void)
+{
+       crypto_unregister_alg(&aes_alg);
+}
+
+module_cpu_feature_match(AES, aes_mod_init);
+module_exit(aes_mod_exit);