arch/arm64/crypto/crc32-arm64.c

   1 /*
   2  * crc32-arm64.c - CRC32 and CRC32C using optional ARMv8 instructions
   3  *
   4  * Module based on crypto/crc32c_generic.c
   5  *
   6  * CRC32 loop taken from Ed Nevill's Hadoop CRC patch
   7  * http://mail-archives.apache.org/mod_mbox/hadoop-common-dev/201406.mbox/%3C1403687030.3355.19.camel%40localhost.localdomain%3E
   8  *
   9  * Using inline assembly instead of intrinsics in order to be backwards
  10  * compatible with older compilers.
  11  *
  12  * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License version 2 as
  16  * published by the Free Software Foundation.
  17  */
  18
  19 #include <linux/unaligned/access_ok.h>
  20 #include <linux/cpufeature.h>
  21 #include <linux/init.h>
  22 #include <linux/kernel.h>
  23 #include <linux/module.h>
  24 #include <linux/string.h>
  25
  26 #include <crypto/internal/hash.h>
  27
  28 MODULE_AUTHOR("Yazen Ghannam <yazen.ghannam@linaro.org>");
  29 MODULE_DESCRIPTION("CRC32 and CRC32C using optional ARMv8 instructions");
  30 MODULE_LICENSE("GPL v2");
  31
  32 #define CRC32X(crc, value) __asm__("crc32x %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
  33 #define CRC32W(crc, value) __asm__("crc32w %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
  34 #define CRC32H(crc, value) __asm__("crc32h %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
  35 #define CRC32B(crc, value) __asm__("crc32b %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
  36 #define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
  37 #define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
  38 #define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
  39 #define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
  40
  41 static u32 crc32_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
  42 {
  43         s64 length = len;
  44
  45         while ((length -= sizeof(u64)) >= 0) {
  46                 CRC32X(crc, get_unaligned_le64(p));
  47                 p += sizeof(u64);
  48         }
  49
  50         /* The following is more efficient than the straight loop */
  51         if (length & sizeof(u32)) {
  52                 CRC32W(crc, get_unaligned_le32(p));
  53                 p += sizeof(u32);
  54         }
  55         if (length & sizeof(u16)) {
  56                 CRC32H(crc, get_unaligned_le16(p));
  57                 p += sizeof(u16);
  58         }
  59         if (length & sizeof(u8))
  60                 CRC32B(crc, *p);
  61
  62         return crc;
  63 }
  64
  65 static u32 crc32c_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
  66 {
  67         s64 length = len;
  68
  69         while ((length -= sizeof(u64)) >= 0) {
  70                 CRC32CX(crc, get_unaligned_le64(p));
  71                 p += sizeof(u64);
  72         }
  73
  74         /* The following is more efficient than the straight loop */
  75         if (length & sizeof(u32)) {
  76                 CRC32CW(crc, get_unaligned_le32(p));
  77                 p += sizeof(u32);
  78         }
  79         if (length & sizeof(u16)) {
  80                 CRC32CH(crc, get_unaligned_le16(p));
  81                 p += sizeof(u16);
  82         }
  83         if (length & sizeof(u8))
  84                 CRC32CB(crc, *p);
  85
  86         return crc;
  87 }
  88
  89 #define CHKSUM_BLOCK_SIZE       1
  90 #define CHKSUM_DIGEST_SIZE      4
  91
  92 struct chksum_ctx {
  93         u32 key;
  94 };
  95
  96 struct chksum_desc_ctx {
  97         u32 crc;
  98 };
  99
 100 static int chksum_init(struct shash_desc *desc)
 101 {
 102         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 103         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 104
 105         ctx->crc = mctx->key;
 106
 107         return 0;
 108 }
 109
 110 /*
 111  * Setting the seed allows arbitrary accumulators and flexible XOR policy
 112  * If your algorithm starts with ~0, then XOR with ~0 before you set
 113  * the seed.
 114  */
 115 static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,
 116                          unsigned int keylen)
 117 {
 118         struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
 119
 120         if (keylen != sizeof(mctx->key)) {
 121                 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 122                 return -EINVAL;
 123         }
 124         mctx->key = get_unaligned_le32(key);
 125         return 0;
 126 }
 127
 128 static int chksum_update(struct shash_desc *desc, const u8 *data,
 129                          unsigned int length)
 130 {
 131         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 132
 133         ctx->crc = crc32_arm64_le_hw(ctx->crc, data, length);
 134         return 0;
 135 }
 136
 137 static int chksumc_update(struct shash_desc *desc, const u8 *data,
 138                          unsigned int length)
 139 {
 140         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 141
 142         ctx->crc = crc32c_arm64_le_hw(ctx->crc, data, length);
 143         return 0;
 144 }
 145
 146 static int chksum_final(struct shash_desc *desc, u8 *out)
 147 {
 148         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 149
 150         put_unaligned_le32(ctx->crc, out);
 151         return 0;
 152 }
 153
 154 static int chksumc_final(struct shash_desc *desc, u8 *out)
 155 {
 156         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 157
 158         put_unaligned_le32(~ctx->crc, out);
 159         return 0;
 160 }
 161
 162 static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
 163 {
 164         put_unaligned_le32(crc32_arm64_le_hw(crc, data, len), out);
 165         return 0;
 166 }
 167
 168 static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
 169 {
 170         put_unaligned_le32(~crc32c_arm64_le_hw(crc, data, len), out);
 171         return 0;
 172 }
 173
 174 static int chksum_finup(struct shash_desc *desc, const u8 *data,
 175                         unsigned int len, u8 *out)
 176 {
 177         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 178
 179         return __chksum_finup(ctx->crc, data, len, out);
 180 }
 181
 182 static int chksumc_finup(struct shash_desc *desc, const u8 *data,
 183                         unsigned int len, u8 *out)
 184 {
 185         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 186
 187         return __chksumc_finup(ctx->crc, data, len, out);
 188 }
 189
 190 static int chksum_digest(struct shash_desc *desc, const u8 *data,
 191                          unsigned int length, u8 *out)
 192 {
 193         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 194
 195         return __chksum_finup(mctx->key, data, length, out);
 196 }
 197
 198 static int chksumc_digest(struct shash_desc *desc, const u8 *data,
 199                          unsigned int length, u8 *out)
 200 {
 201         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 202
 203         return __chksumc_finup(mctx->key, data, length, out);
 204 }
 205
 206 static int crc32_cra_init(struct crypto_tfm *tfm)
 207 {
 208         struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
 209
 210         mctx->key = 0;
 211         return 0;
 212 }
 213
 214 static int crc32c_cra_init(struct crypto_tfm *tfm)
 215 {
 216         struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
 217
 218         mctx->key = ~0;
 219         return 0;
 220 }
 221
 222 static struct shash_alg crc32_alg = {
 223         .digestsize             =       CHKSUM_DIGEST_SIZE,
 224         .setkey                 =       chksum_setkey,
 225         .init                   =       chksum_init,
 226         .update                 =       chksum_update,
 227         .final                  =       chksum_final,
 228         .finup                  =       chksum_finup,
 229         .digest                 =       chksum_digest,
 230         .descsize               =       sizeof(struct chksum_desc_ctx),
 231         .base                   =       {
 232                 .cra_name               =       "crc32",
 233                 .cra_driver_name        =       "crc32-arm64-hw",
 234                 .cra_priority           =       300,
 235                 .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
 236                 .cra_alignmask          =       0,
 237                 .cra_ctxsize            =       sizeof(struct chksum_ctx),
 238                 .cra_module             =       THIS_MODULE,
 239                 .cra_init               =       crc32_cra_init,
 240         }
 241 };
 242
 243 static struct shash_alg crc32c_alg = {
 244         .digestsize             =       CHKSUM_DIGEST_SIZE,
 245         .setkey                 =       chksum_setkey,
 246         .init                   =       chksum_init,
 247         .update                 =       chksumc_update,
 248         .final                  =       chksumc_final,
 249         .finup                  =       chksumc_finup,
 250         .digest                 =       chksumc_digest,
 251         .descsize               =       sizeof(struct chksum_desc_ctx),
 252         .base                   =       {
 253                 .cra_name               =       "crc32c",
 254                 .cra_driver_name        =       "crc32c-arm64-hw",
 255                 .cra_priority           =       300,
 256                 .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
 257                 .cra_alignmask          =       0,
 258                 .cra_ctxsize            =       sizeof(struct chksum_ctx),
 259                 .cra_module             =       THIS_MODULE,
 260                 .cra_init               =       crc32c_cra_init,
 261         }
 262 };
 263
 264 static int __init crc32_mod_init(void)
 265 {
 266         int err;
 267
 268         err = crypto_register_shash(&crc32_alg);
 269
 270         if (err)
 271                 return err;
 272
 273         err = crypto_register_shash(&crc32c_alg);
 274
 275         if (err) {
 276                 crypto_unregister_shash(&crc32_alg);
 277                 return err;
 278         }
 279
 280         return 0;
 281 }
 282
 283 static void __exit crc32_mod_exit(void)
 284 {
 285         crypto_unregister_shash(&crc32_alg);
 286         crypto_unregister_shash(&crc32c_alg);
 287 }
 288
 289 module_cpu_feature_match(CRC32, crc32_mod_init);
 290 module_exit(crc32_mod_exit);