crypto: nx - rename nx-842-crypto.c to nx-842.c
authorDan Streetman <ddstreet@ieee.org>
Wed, 22 Jul 2015 18:26:37 +0000 (14:26 -0400)
committerHerbert Xu <herbert@gondor.apana.org.au>
Thu, 23 Jul 2015 10:14:32 +0000 (18:14 +0800)
The last commit merged nx-842.c's code into nx-842-crypto.c.  It
did not rename nx-842-crypto.c to nx-842.c, in order to let the
patch more clearly show what was merged.  This just renames
nx-842-crypto.c to nx-842.c, with no changes to its code.

Signed-off-by: Dan Streetman <ddstreet@ieee.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
drivers/crypto/nx/Makefile
drivers/crypto/nx/nx-842-crypto.c [deleted file]
drivers/crypto/nx/nx-842.c [new file with mode: 0644]

index e29ee85b24313b84e1aed51ed1b2ae31d208c3b6..ffcc86c44e158f03db22e21709cca26bfc09500d 100644 (file)
@@ -13,7 +13,7 @@ nx-crypto-objs := nx.o \
 obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS) += nx-compress.o nx-compress-platform.o
 obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o
 obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o
-nx-compress-objs := nx-842-crypto.o
+nx-compress-objs := nx-842.o
 nx-compress-platform-objs := nx-842-platform.o
 nx-compress-pseries-objs := nx-842-pseries.o
 nx-compress-powernv-objs := nx-842-powernv.o
diff --git a/drivers/crypto/nx/nx-842-crypto.c b/drivers/crypto/nx/nx-842-crypto.c
deleted file mode 100644 (file)
index 4472e20..0000000
+++ /dev/null
@@ -1,600 +0,0 @@
-/*
- * Cryptographic API for the NX-842 hardware compression.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * Copyright (C) IBM Corporation, 2011-2015
- *
- * Designer of the Power data compression engine:
- *   Bulent Abali <abali@us.ibm.com>
- *
- * Original Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
- *                   Seth Jennings <sjenning@linux.vnet.ibm.com>
- *
- * Rewrite: Dan Streetman <ddstreet@ieee.org>
- *
- * This is an interface to the NX-842 compression hardware in PowerPC
- * processors.  Most of the complexity of this drvier is due to the fact that
- * the NX-842 compression hardware requires the input and output data buffers
- * to be specifically aligned, to be a specific multiple in length, and within
- * specific minimum and maximum lengths.  Those restrictions, provided by the
- * nx-842 driver via nx842_constraints, mean this driver must use bounce
- * buffers and headers to correct misaligned in or out buffers, and to split
- * input buffers that are too large.
- *
- * This driver will fall back to software decompression if the hardware
- * decompression fails, so this driver's decompression should never fail as
- * long as the provided compressed buffer is valid.  Any compressed buffer
- * created by this driver will have a header (except ones where the input
- * perfectly matches the constraints); so users of this driver cannot simply
- * pass a compressed buffer created by this driver over to the 842 software
- * decompression library.  Instead, users must use this driver to decompress;
- * if the hardware fails or is unavailable, the compressed buffer will be
- * parsed and the header removed, and the raw 842 buffer(s) passed to the 842
- * software decompression library.
- *
- * This does not fall back to software compression, however, since the caller
- * of this function is specifically requesting hardware compression; if the
- * hardware compression fails, the caller can fall back to software
- * compression, and the raw 842 compressed buffer that the software compressor
- * creates can be passed to this driver for hardware decompression; any
- * buffer without our specific header magic is assumed to be a raw 842 buffer
- * and passed directly to the hardware.  Note that the software compression
- * library will produce a compressed buffer that is incompatible with the
- * hardware decompressor if the original input buffer length is not a multiple
- * of 8; if such a compressed buffer is passed to this driver for
- * decompression, the hardware will reject it and this driver will then pass
- * it over to the software library for decompression.
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/crypto.h>
-#include <linux/vmalloc.h>
-#include <linux/sw842.h>
-#include <linux/ratelimit.h>
-#include <linux/spinlock.h>
-
-#include "nx-842.h"
-
-/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit
- * template (see lib/842/842.h), so this magic number will never appear at
- * the start of a raw 842 compressed buffer.  That is important, as any buffer
- * passed to us without this magic is assumed to be a raw 842 compressed
- * buffer, and passed directly to the hardware to decompress.
- */
-#define NX842_CRYPTO_MAGIC     (0xf842)
-#define NX842_CRYPTO_GROUP_MAX (0x20)
-#define NX842_CRYPTO_HEADER_SIZE(g)                            \
-       (sizeof(struct nx842_crypto_header) +                   \
-        sizeof(struct nx842_crypto_header_group) * (g))
-#define NX842_CRYPTO_HEADER_MAX_SIZE                           \
-       NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX)
-
-/* bounce buffer size */
-#define BOUNCE_BUFFER_ORDER    (2)
-#define BOUNCE_BUFFER_SIZE                                     \
-       ((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER))
-
-/* try longer on comp because we can fallback to sw decomp if hw is busy */
-#define COMP_BUSY_TIMEOUT      (250) /* ms */
-#define DECOMP_BUSY_TIMEOUT    (50) /* ms */
-
-struct nx842_crypto_header_group {
-       __be16 padding;                 /* unused bytes at start of group */
-       __be32 compressed_length;       /* compressed bytes in group */
-       __be32 uncompressed_length;     /* bytes after decompression */
-} __packed;
-
-struct nx842_crypto_header {
-       __be16 magic;           /* NX842_CRYPTO_MAGIC */
-       __be16 ignore;          /* decompressed end bytes to ignore */
-       u8 groups;              /* total groups in this header */
-       struct nx842_crypto_header_group group[];
-} __packed;
-
-struct nx842_crypto_param {
-       u8 *in;
-       unsigned int iremain;
-       u8 *out;
-       unsigned int oremain;
-       unsigned int ototal;
-};
-
-static int update_param(struct nx842_crypto_param *p,
-                       unsigned int slen, unsigned int dlen)
-{
-       if (p->iremain < slen)
-               return -EOVERFLOW;
-       if (p->oremain < dlen)
-               return -ENOSPC;
-
-       p->in += slen;
-       p->iremain -= slen;
-       p->out += dlen;
-       p->oremain -= dlen;
-       p->ototal += dlen;
-
-       return 0;
-}
-
-struct nx842_crypto_ctx {
-       spinlock_t lock;
-
-       u8 *wmem;
-       u8 *sbounce, *dbounce;
-
-       struct nx842_crypto_header header;
-       struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX];
-};
-
-static int nx842_crypto_init(struct crypto_tfm *tfm)
-{
-       struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
-
-       spin_lock_init(&ctx->lock);
-       ctx->wmem = kmalloc(nx842_workmem_size(), GFP_KERNEL);
-       ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
-       ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
-       if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) {
-               kfree(ctx->wmem);
-               free_page((unsigned long)ctx->sbounce);
-               free_page((unsigned long)ctx->dbounce);
-               return -ENOMEM;
-       }
-
-       return 0;
-}
-
-static void nx842_crypto_exit(struct crypto_tfm *tfm)
-{
-       struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
-
-       kfree(ctx->wmem);
-       free_page((unsigned long)ctx->sbounce);
-       free_page((unsigned long)ctx->dbounce);
-}
-
-static void check_constraints(struct nx842_constraints *c)
-{
-       /* limit maximum, to always have enough bounce buffer to decompress */
-       if (c->maximum > BOUNCE_BUFFER_SIZE)
-               c->maximum = BOUNCE_BUFFER_SIZE;
-}
-
-static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf)
-{
-       int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
-
-       /* compress should have added space for header */
-       if (s > be16_to_cpu(hdr->group[0].padding)) {
-               pr_err("Internal error: no space for header\n");
-               return -EINVAL;
-       }
-
-       memcpy(buf, hdr, s);
-
-       print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0);
-
-       return 0;
-}
-
-static int compress(struct nx842_crypto_ctx *ctx,
-                   struct nx842_crypto_param *p,
-                   struct nx842_crypto_header_group *g,
-                   struct nx842_constraints *c,
-                   u16 *ignore,
-                   unsigned int hdrsize)
-{
-       unsigned int slen = p->iremain, dlen = p->oremain, tmplen;
-       unsigned int adj_slen = slen;
-       u8 *src = p->in, *dst = p->out;
-       int ret, dskip = 0;
-       ktime_t timeout;
-
-       if (p->iremain == 0)
-               return -EOVERFLOW;
-
-       if (p->oremain == 0 || hdrsize + c->minimum > dlen)
-               return -ENOSPC;
-
-       if (slen % c->multiple)
-               adj_slen = round_up(slen, c->multiple);
-       if (slen < c->minimum)
-               adj_slen = c->minimum;
-       if (slen > c->maximum)
-               adj_slen = slen = c->maximum;
-       if (adj_slen > slen || (u64)src % c->alignment) {
-               adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE);
-               slen = min(slen, BOUNCE_BUFFER_SIZE);
-               if (adj_slen > slen)
-                       memset(ctx->sbounce + slen, 0, adj_slen - slen);
-               memcpy(ctx->sbounce, src, slen);
-               src = ctx->sbounce;
-               slen = adj_slen;
-               pr_debug("using comp sbounce buffer, len %x\n", slen);
-       }
-
-       dst += hdrsize;
-       dlen -= hdrsize;
-
-       if ((u64)dst % c->alignment) {
-               dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst);
-               dst += dskip;
-               dlen -= dskip;
-       }
-       if (dlen % c->multiple)
-               dlen = round_down(dlen, c->multiple);
-       if (dlen < c->minimum) {
-nospc:
-               dst = ctx->dbounce;
-               dlen = min(p->oremain, BOUNCE_BUFFER_SIZE);
-               dlen = round_down(dlen, c->multiple);
-               dskip = 0;
-               pr_debug("using comp dbounce buffer, len %x\n", dlen);
-       }
-       if (dlen > c->maximum)
-               dlen = c->maximum;
-
-       tmplen = dlen;
-       timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT);
-       do {
-               dlen = tmplen; /* reset dlen, if we're retrying */
-               ret = nx842_platform_driver()->compress(src, slen,
-                                                       dst, &dlen,
-                                                       ctx->wmem);
-               /* possibly we should reduce the slen here, instead of
-                * retrying with the dbounce buffer?
-                */
-               if (ret == -ENOSPC && dst != ctx->dbounce)
-                       goto nospc;
-       } while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
-       if (ret)
-               return ret;
-
-       dskip += hdrsize;
-
-       if (dst == ctx->dbounce)
-               memcpy(p->out + dskip, dst, dlen);
-
-       g->padding = cpu_to_be16(dskip);
-       g->compressed_length = cpu_to_be32(dlen);
-       g->uncompressed_length = cpu_to_be32(slen);
-
-       if (p->iremain < slen) {
-               *ignore = slen - p->iremain;
-               slen = p->iremain;
-       }
-
-       pr_debug("compress slen %x ignore %x dlen %x padding %x\n",
-                slen, *ignore, dlen, dskip);
-
-       return update_param(p, slen, dskip + dlen);
-}
-
-static int nx842_crypto_compress(struct crypto_tfm *tfm,
-                                const u8 *src, unsigned int slen,
-                                u8 *dst, unsigned int *dlen)
-{
-       struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
-       struct nx842_crypto_header *hdr = &ctx->header;
-       struct nx842_crypto_param p;
-       struct nx842_constraints c = *nx842_platform_driver()->constraints;
-       unsigned int groups, hdrsize, h;
-       int ret, n;
-       bool add_header;
-       u16 ignore = 0;
-
-       check_constraints(&c);
-
-       p.in = (u8 *)src;
-       p.iremain = slen;
-       p.out = dst;
-       p.oremain = *dlen;
-       p.ototal = 0;
-
-       *dlen = 0;
-
-       groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX,
-                      DIV_ROUND_UP(p.iremain, c.maximum));
-       hdrsize = NX842_CRYPTO_HEADER_SIZE(groups);
-
-       spin_lock_bh(&ctx->lock);
-
-       /* skip adding header if the buffers meet all constraints */
-       add_header = (p.iremain % c.multiple    ||
-                     p.iremain < c.minimum     ||
-                     p.iremain > c.maximum     ||
-                     (u64)p.in % c.alignment   ||
-                     p.oremain % c.multiple    ||
-                     p.oremain < c.minimum     ||
-                     p.oremain > c.maximum     ||
-                     (u64)p.out % c.alignment);
-
-       hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC);
-       hdr->groups = 0;
-       hdr->ignore = 0;
-
-       while (p.iremain > 0) {
-               n = hdr->groups++;
-               ret = -ENOSPC;
-               if (hdr->groups > NX842_CRYPTO_GROUP_MAX)
-                       goto unlock;
-
-               /* header goes before first group */
-               h = !n && add_header ? hdrsize : 0;
-
-               if (ignore)
-                       pr_warn("interal error, ignore is set %x\n", ignore);
-
-               ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h);
-               if (ret)
-                       goto unlock;
-       }
-
-       if (!add_header && hdr->groups > 1) {
-               pr_err("Internal error: No header but multiple groups\n");
-               ret = -EINVAL;
-               goto unlock;
-       }
-
-       /* ignore indicates the input stream needed to be padded */
-       hdr->ignore = cpu_to_be16(ignore);
-       if (ignore)
-               pr_debug("marked %d bytes as ignore\n", ignore);
-
-       if (add_header)
-               ret = nx842_crypto_add_header(hdr, dst);
-       if (ret)
-               goto unlock;
-
-       *dlen = p.ototal;
-
-       pr_debug("compress total slen %x dlen %x\n", slen, *dlen);
-
-unlock:
-       spin_unlock_bh(&ctx->lock);
-       return ret;
-}
-
-static int decompress(struct nx842_crypto_ctx *ctx,
-                     struct nx842_crypto_param *p,
-                     struct nx842_crypto_header_group *g,
-                     struct nx842_constraints *c,
-                     u16 ignore)
-{
-       unsigned int slen = be32_to_cpu(g->compressed_length);
-       unsigned int required_len = be32_to_cpu(g->uncompressed_length);
-       unsigned int dlen = p->oremain, tmplen;
-       unsigned int adj_slen = slen;
-       u8 *src = p->in, *dst = p->out;
-       u16 padding = be16_to_cpu(g->padding);
-       int ret, spadding = 0, dpadding = 0;
-       ktime_t timeout;
-
-       if (!slen || !required_len)
-               return -EINVAL;
-
-       if (p->iremain <= 0 || padding + slen > p->iremain)
-               return -EOVERFLOW;
-
-       if (p->oremain <= 0 || required_len - ignore > p->oremain)
-               return -ENOSPC;
-
-       src += padding;
-
-       if (slen % c->multiple)
-               adj_slen = round_up(slen, c->multiple);
-       if (slen < c->minimum)
-               adj_slen = c->minimum;
-       if (slen > c->maximum)
-               goto usesw;
-       if (slen < adj_slen || (u64)src % c->alignment) {
-               /* we can append padding bytes because the 842 format defines
-                * an "end" template (see lib/842/842_decompress.c) and will
-                * ignore any bytes following it.
-                */
-               if (slen < adj_slen)
-                       memset(ctx->sbounce + slen, 0, adj_slen - slen);
-               memcpy(ctx->sbounce, src, slen);
-               src = ctx->sbounce;
-               spadding = adj_slen - slen;
-               slen = adj_slen;
-               pr_debug("using decomp sbounce buffer, len %x\n", slen);
-       }
-
-       if (dlen % c->multiple)
-               dlen = round_down(dlen, c->multiple);
-       if (dlen < required_len || (u64)dst % c->alignment) {
-               dst = ctx->dbounce;
-               dlen = min(required_len, BOUNCE_BUFFER_SIZE);
-               pr_debug("using decomp dbounce buffer, len %x\n", dlen);
-       }
-       if (dlen < c->minimum)
-               goto usesw;
-       if (dlen > c->maximum)
-               dlen = c->maximum;
-
-       tmplen = dlen;
-       timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT);
-       do {
-               dlen = tmplen; /* reset dlen, if we're retrying */
-               ret = nx842_platform_driver()->decompress(src, slen,
-                                                         dst, &dlen,
-                                                         ctx->wmem);
-       } while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
-       if (ret) {
-usesw:
-               /* reset everything, sw doesn't have constraints */
-               src = p->in + padding;
-               slen = be32_to_cpu(g->compressed_length);
-               spadding = 0;
-               dst = p->out;
-               dlen = p->oremain;
-               dpadding = 0;
-               if (dlen < required_len) { /* have ignore bytes */
-                       dst = ctx->dbounce;
-                       dlen = BOUNCE_BUFFER_SIZE;
-               }
-               pr_info_ratelimited("using software 842 decompression\n");
-               ret = sw842_decompress(src, slen, dst, &dlen);
-       }
-       if (ret)
-               return ret;
-
-       slen -= spadding;
-
-       dlen -= ignore;
-       if (ignore)
-               pr_debug("ignoring last %x bytes\n", ignore);
-
-       if (dst == ctx->dbounce)
-               memcpy(p->out, dst, dlen);
-
-       pr_debug("decompress slen %x padding %x dlen %x ignore %x\n",
-                slen, padding, dlen, ignore);
-
-       return update_param(p, slen + padding, dlen);
-}
-
-static int nx842_crypto_decompress(struct crypto_tfm *tfm,
-                                  const u8 *src, unsigned int slen,
-                                  u8 *dst, unsigned int *dlen)
-{
-       struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
-       struct nx842_crypto_header *hdr;
-       struct nx842_crypto_param p;
-       struct nx842_constraints c = *nx842_platform_driver()->constraints;
-       int n, ret, hdr_len;
-       u16 ignore = 0;
-
-       check_constraints(&c);
-
-       p.in = (u8 *)src;
-       p.iremain = slen;
-       p.out = dst;
-       p.oremain = *dlen;
-       p.ototal = 0;
-
-       *dlen = 0;
-
-       hdr = (struct nx842_crypto_header *)src;
-
-       spin_lock_bh(&ctx->lock);
-
-       /* If it doesn't start with our header magic number, assume it's a raw
-        * 842 compressed buffer and pass it directly to the hardware driver
-        */
-       if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) {
-               struct nx842_crypto_header_group g = {
-                       .padding =              0,
-                       .compressed_length =    cpu_to_be32(p.iremain),
-                       .uncompressed_length =  cpu_to_be32(p.oremain),
-               };
-
-               ret = decompress(ctx, &p, &g, &c, 0);
-               if (ret)
-                       goto unlock;
-
-               goto success;
-       }
-
-       if (!hdr->groups) {
-               pr_err("header has no groups\n");
-               ret = -EINVAL;
-               goto unlock;
-       }
-       if (hdr->groups > NX842_CRYPTO_GROUP_MAX) {
-               pr_err("header has too many groups %x, max %x\n",
-                      hdr->groups, NX842_CRYPTO_GROUP_MAX);
-               ret = -EINVAL;
-               goto unlock;
-       }
-
-       hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
-       if (hdr_len > slen) {
-               ret = -EOVERFLOW;
-               goto unlock;
-       }
-
-       memcpy(&ctx->header, src, hdr_len);
-       hdr = &ctx->header;
-
-       for (n = 0; n < hdr->groups; n++) {
-               /* ignore applies to last group */
-               if (n + 1 == hdr->groups)
-                       ignore = be16_to_cpu(hdr->ignore);
-
-               ret = decompress(ctx, &p, &hdr->group[n], &c, ignore);
-               if (ret)
-                       goto unlock;
-       }
-
-success:
-       *dlen = p.ototal;
-
-       pr_debug("decompress total slen %x dlen %x\n", slen, *dlen);
-
-       ret = 0;
-
-unlock:
-       spin_unlock_bh(&ctx->lock);
-
-       return ret;
-}
-
-static struct crypto_alg alg = {
-       .cra_name               = "842",
-       .cra_driver_name        = "842-nx",
-       .cra_priority           = 300,
-       .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
-       .cra_ctxsize            = sizeof(struct nx842_crypto_ctx),
-       .cra_module             = THIS_MODULE,
-       .cra_init               = nx842_crypto_init,
-       .cra_exit               = nx842_crypto_exit,
-       .cra_u                  = { .compress = {
-       .coa_compress           = nx842_crypto_compress,
-       .coa_decompress         = nx842_crypto_decompress } }
-};
-
-static int __init nx842_crypto_mod_init(void)
-{
-       request_module("nx-compress-powernv");
-       request_module("nx-compress-pseries");
-
-       /* we prevent loading/registering if there's no platform driver,
-        * and we get the platform module that set it so it won't unload,
-        * so we don't need to check if it's set in any of our functions
-        */
-       if (!nx842_platform_driver_get()) {
-               pr_err("no nx842 platform driver found.\n");
-               return -ENODEV;
-       }
-
-       return crypto_register_alg(&alg);
-}
-module_init(nx842_crypto_mod_init);
-
-static void __exit nx842_crypto_mod_exit(void)
-{
-       crypto_unregister_alg(&alg);
-
-       nx842_platform_driver_put();
-}
-module_exit(nx842_crypto_mod_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Driver");
-MODULE_ALIAS_CRYPTO("842");
-MODULE_ALIAS_CRYPTO("842-nx");
-MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c
new file mode 100644 (file)
index 0000000..4472e20
--- /dev/null
@@ -0,0 +1,600 @@
+/*
+ * Cryptographic API for the NX-842 hardware compression.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * Copyright (C) IBM Corporation, 2011-2015
+ *
+ * Designer of the Power data compression engine:
+ *   Bulent Abali <abali@us.ibm.com>
+ *
+ * Original Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
+ *                   Seth Jennings <sjenning@linux.vnet.ibm.com>
+ *
+ * Rewrite: Dan Streetman <ddstreet@ieee.org>
+ *
+ * This is an interface to the NX-842 compression hardware in PowerPC
+ * processors.  Most of the complexity of this drvier is due to the fact that
+ * the NX-842 compression hardware requires the input and output data buffers
+ * to be specifically aligned, to be a specific multiple in length, and within
+ * specific minimum and maximum lengths.  Those restrictions, provided by the
+ * nx-842 driver via nx842_constraints, mean this driver must use bounce
+ * buffers and headers to correct misaligned in or out buffers, and to split
+ * input buffers that are too large.
+ *
+ * This driver will fall back to software decompression if the hardware
+ * decompression fails, so this driver's decompression should never fail as
+ * long as the provided compressed buffer is valid.  Any compressed buffer
+ * created by this driver will have a header (except ones where the input
+ * perfectly matches the constraints); so users of this driver cannot simply
+ * pass a compressed buffer created by this driver over to the 842 software
+ * decompression library.  Instead, users must use this driver to decompress;
+ * if the hardware fails or is unavailable, the compressed buffer will be
+ * parsed and the header removed, and the raw 842 buffer(s) passed to the 842
+ * software decompression library.
+ *
+ * This does not fall back to software compression, however, since the caller
+ * of this function is specifically requesting hardware compression; if the
+ * hardware compression fails, the caller can fall back to software
+ * compression, and the raw 842 compressed buffer that the software compressor
+ * creates can be passed to this driver for hardware decompression; any
+ * buffer without our specific header magic is assumed to be a raw 842 buffer
+ * and passed directly to the hardware.  Note that the software compression
+ * library will produce a compressed buffer that is incompatible with the
+ * hardware decompressor if the original input buffer length is not a multiple
+ * of 8; if such a compressed buffer is passed to this driver for
+ * decompression, the hardware will reject it and this driver will then pass
+ * it over to the software library for decompression.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/crypto.h>
+#include <linux/vmalloc.h>
+#include <linux/sw842.h>
+#include <linux/ratelimit.h>
+#include <linux/spinlock.h>
+
+#include "nx-842.h"
+
+/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit
+ * template (see lib/842/842.h), so this magic number will never appear at
+ * the start of a raw 842 compressed buffer.  That is important, as any buffer
+ * passed to us without this magic is assumed to be a raw 842 compressed
+ * buffer, and passed directly to the hardware to decompress.
+ */
+#define NX842_CRYPTO_MAGIC     (0xf842)
+#define NX842_CRYPTO_GROUP_MAX (0x20)
+#define NX842_CRYPTO_HEADER_SIZE(g)                            \
+       (sizeof(struct nx842_crypto_header) +                   \
+        sizeof(struct nx842_crypto_header_group) * (g))
+#define NX842_CRYPTO_HEADER_MAX_SIZE                           \
+       NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX)
+
+/* bounce buffer size */
+#define BOUNCE_BUFFER_ORDER    (2)
+#define BOUNCE_BUFFER_SIZE                                     \
+       ((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER))
+
+/* try longer on comp because we can fallback to sw decomp if hw is busy */
+#define COMP_BUSY_TIMEOUT      (250) /* ms */
+#define DECOMP_BUSY_TIMEOUT    (50) /* ms */
+
+struct nx842_crypto_header_group {
+       __be16 padding;                 /* unused bytes at start of group */
+       __be32 compressed_length;       /* compressed bytes in group */
+       __be32 uncompressed_length;     /* bytes after decompression */
+} __packed;
+
+struct nx842_crypto_header {
+       __be16 magic;           /* NX842_CRYPTO_MAGIC */
+       __be16 ignore;          /* decompressed end bytes to ignore */
+       u8 groups;              /* total groups in this header */
+       struct nx842_crypto_header_group group[];
+} __packed;
+
+struct nx842_crypto_param {
+       u8 *in;
+       unsigned int iremain;
+       u8 *out;
+       unsigned int oremain;
+       unsigned int ototal;
+};
+
+static int update_param(struct nx842_crypto_param *p,
+                       unsigned int slen, unsigned int dlen)
+{
+       if (p->iremain < slen)
+               return -EOVERFLOW;
+       if (p->oremain < dlen)
+               return -ENOSPC;
+
+       p->in += slen;
+       p->iremain -= slen;
+       p->out += dlen;
+       p->oremain -= dlen;
+       p->ototal += dlen;
+
+       return 0;
+}
+
+struct nx842_crypto_ctx {
+       spinlock_t lock;
+
+       u8 *wmem;
+       u8 *sbounce, *dbounce;
+
+       struct nx842_crypto_header header;
+       struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX];
+};
+
+static int nx842_crypto_init(struct crypto_tfm *tfm)
+{
+       struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
+
+       spin_lock_init(&ctx->lock);
+       ctx->wmem = kmalloc(nx842_workmem_size(), GFP_KERNEL);
+       ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
+       ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
+       if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) {
+               kfree(ctx->wmem);
+               free_page((unsigned long)ctx->sbounce);
+               free_page((unsigned long)ctx->dbounce);
+               return -ENOMEM;
+       }
+
+       return 0;
+}
+
+static void nx842_crypto_exit(struct crypto_tfm *tfm)
+{
+       struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
+
+       kfree(ctx->wmem);
+       free_page((unsigned long)ctx->sbounce);
+       free_page((unsigned long)ctx->dbounce);
+}
+
+static void check_constraints(struct nx842_constraints *c)
+{
+       /* limit maximum, to always have enough bounce buffer to decompress */
+       if (c->maximum > BOUNCE_BUFFER_SIZE)
+               c->maximum = BOUNCE_BUFFER_SIZE;
+}
+
+static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf)
+{
+       int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
+
+       /* compress should have added space for header */
+       if (s > be16_to_cpu(hdr->group[0].padding)) {
+               pr_err("Internal error: no space for header\n");
+               return -EINVAL;
+       }
+
+       memcpy(buf, hdr, s);
+
+       print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0);
+
+       return 0;
+}
+
+static int compress(struct nx842_crypto_ctx *ctx,
+                   struct nx842_crypto_param *p,
+                   struct nx842_crypto_header_group *g,
+                   struct nx842_constraints *c,
+                   u16 *ignore,
+                   unsigned int hdrsize)
+{
+       unsigned int slen = p->iremain, dlen = p->oremain, tmplen;
+       unsigned int adj_slen = slen;
+       u8 *src = p->in, *dst = p->out;
+       int ret, dskip = 0;
+       ktime_t timeout;
+
+       if (p->iremain == 0)
+               return -EOVERFLOW;
+
+       if (p->oremain == 0 || hdrsize + c->minimum > dlen)
+               return -ENOSPC;
+
+       if (slen % c->multiple)
+               adj_slen = round_up(slen, c->multiple);
+       if (slen < c->minimum)
+               adj_slen = c->minimum;
+       if (slen > c->maximum)
+               adj_slen = slen = c->maximum;
+       if (adj_slen > slen || (u64)src % c->alignment) {
+               adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE);
+               slen = min(slen, BOUNCE_BUFFER_SIZE);
+               if (adj_slen > slen)
+                       memset(ctx->sbounce + slen, 0, adj_slen - slen);
+               memcpy(ctx->sbounce, src, slen);
+               src = ctx->sbounce;
+               slen = adj_slen;
+               pr_debug("using comp sbounce buffer, len %x\n", slen);
+       }
+
+       dst += hdrsize;
+       dlen -= hdrsize;
+
+       if ((u64)dst % c->alignment) {
+               dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst);
+               dst += dskip;
+               dlen -= dskip;
+       }
+       if (dlen % c->multiple)
+               dlen = round_down(dlen, c->multiple);
+       if (dlen < c->minimum) {
+nospc:
+               dst = ctx->dbounce;
+               dlen = min(p->oremain, BOUNCE_BUFFER_SIZE);
+               dlen = round_down(dlen, c->multiple);
+               dskip = 0;
+               pr_debug("using comp dbounce buffer, len %x\n", dlen);
+       }
+       if (dlen > c->maximum)
+               dlen = c->maximum;
+
+       tmplen = dlen;
+       timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT);
+       do {
+               dlen = tmplen; /* reset dlen, if we're retrying */
+               ret = nx842_platform_driver()->compress(src, slen,
+                                                       dst, &dlen,
+                                                       ctx->wmem);
+               /* possibly we should reduce the slen here, instead of
+                * retrying with the dbounce buffer?
+                */
+               if (ret == -ENOSPC && dst != ctx->dbounce)
+                       goto nospc;
+       } while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
+       if (ret)
+               return ret;
+
+       dskip += hdrsize;
+
+       if (dst == ctx->dbounce)
+               memcpy(p->out + dskip, dst, dlen);
+
+       g->padding = cpu_to_be16(dskip);
+       g->compressed_length = cpu_to_be32(dlen);
+       g->uncompressed_length = cpu_to_be32(slen);
+
+       if (p->iremain < slen) {
+               *ignore = slen - p->iremain;
+               slen = p->iremain;
+       }
+
+       pr_debug("compress slen %x ignore %x dlen %x padding %x\n",
+                slen, *ignore, dlen, dskip);
+
+       return update_param(p, slen, dskip + dlen);
+}
+
+static int nx842_crypto_compress(struct crypto_tfm *tfm,
+                                const u8 *src, unsigned int slen,
+                                u8 *dst, unsigned int *dlen)
+{
+       struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
+       struct nx842_crypto_header *hdr = &ctx->header;
+       struct nx842_crypto_param p;
+       struct nx842_constraints c = *nx842_platform_driver()->constraints;
+       unsigned int groups, hdrsize, h;
+       int ret, n;
+       bool add_header;
+       u16 ignore = 0;
+
+       check_constraints(&c);
+
+       p.in = (u8 *)src;
+       p.iremain = slen;
+       p.out = dst;
+       p.oremain = *dlen;
+       p.ototal = 0;
+
+       *dlen = 0;
+
+       groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX,
+                      DIV_ROUND_UP(p.iremain, c.maximum));
+       hdrsize = NX842_CRYPTO_HEADER_SIZE(groups);
+
+       spin_lock_bh(&ctx->lock);
+
+       /* skip adding header if the buffers meet all constraints */
+       add_header = (p.iremain % c.multiple    ||
+                     p.iremain < c.minimum     ||
+                     p.iremain > c.maximum     ||
+                     (u64)p.in % c.alignment   ||
+                     p.oremain % c.multiple    ||
+                     p.oremain < c.minimum     ||
+                     p.oremain > c.maximum     ||
+                     (u64)p.out % c.alignment);
+
+       hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC);
+       hdr->groups = 0;
+       hdr->ignore = 0;
+
+       while (p.iremain > 0) {
+               n = hdr->groups++;
+               ret = -ENOSPC;
+               if (hdr->groups > NX842_CRYPTO_GROUP_MAX)
+                       goto unlock;
+
+               /* header goes before first group */
+               h = !n && add_header ? hdrsize : 0;
+
+               if (ignore)
+                       pr_warn("interal error, ignore is set %x\n", ignore);
+
+               ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h);
+               if (ret)
+                       goto unlock;
+       }
+
+       if (!add_header && hdr->groups > 1) {
+               pr_err("Internal error: No header but multiple groups\n");
+               ret = -EINVAL;
+               goto unlock;
+       }
+
+       /* ignore indicates the input stream needed to be padded */
+       hdr->ignore = cpu_to_be16(ignore);
+       if (ignore)
+               pr_debug("marked %d bytes as ignore\n", ignore);
+
+       if (add_header)
+               ret = nx842_crypto_add_header(hdr, dst);
+       if (ret)
+               goto unlock;
+
+       *dlen = p.ototal;
+
+       pr_debug("compress total slen %x dlen %x\n", slen, *dlen);
+
+unlock:
+       spin_unlock_bh(&ctx->lock);
+       return ret;
+}
+
+static int decompress(struct nx842_crypto_ctx *ctx,
+                     struct nx842_crypto_param *p,
+                     struct nx842_crypto_header_group *g,
+                     struct nx842_constraints *c,
+                     u16 ignore)
+{
+       unsigned int slen = be32_to_cpu(g->compressed_length);
+       unsigned int required_len = be32_to_cpu(g->uncompressed_length);
+       unsigned int dlen = p->oremain, tmplen;
+       unsigned int adj_slen = slen;
+       u8 *src = p->in, *dst = p->out;
+       u16 padding = be16_to_cpu(g->padding);
+       int ret, spadding = 0, dpadding = 0;
+       ktime_t timeout;
+
+       if (!slen || !required_len)
+               return -EINVAL;
+
+       if (p->iremain <= 0 || padding + slen > p->iremain)
+               return -EOVERFLOW;
+
+       if (p->oremain <= 0 || required_len - ignore > p->oremain)
+               return -ENOSPC;
+
+       src += padding;
+
+       if (slen % c->multiple)
+               adj_slen = round_up(slen, c->multiple);
+       if (slen < c->minimum)
+               adj_slen = c->minimum;
+       if (slen > c->maximum)
+               goto usesw;
+       if (slen < adj_slen || (u64)src % c->alignment) {
+               /* we can append padding bytes because the 842 format defines
+                * an "end" template (see lib/842/842_decompress.c) and will
+                * ignore any bytes following it.
+                */
+               if (slen < adj_slen)
+                       memset(ctx->sbounce + slen, 0, adj_slen - slen);
+               memcpy(ctx->sbounce, src, slen);
+               src = ctx->sbounce;
+               spadding = adj_slen - slen;
+               slen = adj_slen;
+               pr_debug("using decomp sbounce buffer, len %x\n", slen);
+       }
+
+       if (dlen % c->multiple)
+               dlen = round_down(dlen, c->multiple);
+       if (dlen < required_len || (u64)dst % c->alignment) {
+               dst = ctx->dbounce;
+               dlen = min(required_len, BOUNCE_BUFFER_SIZE);
+               pr_debug("using decomp dbounce buffer, len %x\n", dlen);
+       }
+       if (dlen < c->minimum)
+               goto usesw;
+       if (dlen > c->maximum)
+               dlen = c->maximum;
+
+       tmplen = dlen;
+       timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT);
+       do {
+               dlen = tmplen; /* reset dlen, if we're retrying */
+               ret = nx842_platform_driver()->decompress(src, slen,
+                                                         dst, &dlen,
+                                                         ctx->wmem);
+       } while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
+       if (ret) {
+usesw:
+               /* reset everything, sw doesn't have constraints */
+               src = p->in + padding;
+               slen = be32_to_cpu(g->compressed_length);
+               spadding = 0;
+               dst = p->out;
+               dlen = p->oremain;
+               dpadding = 0;
+               if (dlen < required_len) { /* have ignore bytes */
+                       dst = ctx->dbounce;
+                       dlen = BOUNCE_BUFFER_SIZE;
+               }
+               pr_info_ratelimited("using software 842 decompression\n");
+               ret = sw842_decompress(src, slen, dst, &dlen);
+       }
+       if (ret)
+               return ret;
+
+       slen -= spadding;
+
+       dlen -= ignore;
+       if (ignore)
+               pr_debug("ignoring last %x bytes\n", ignore);
+
+       if (dst == ctx->dbounce)
+               memcpy(p->out, dst, dlen);
+
+       pr_debug("decompress slen %x padding %x dlen %x ignore %x\n",
+                slen, padding, dlen, ignore);
+
+       return update_param(p, slen + padding, dlen);
+}
+
+static int nx842_crypto_decompress(struct crypto_tfm *tfm,
+                                  const u8 *src, unsigned int slen,
+                                  u8 *dst, unsigned int *dlen)
+{
+       struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
+       struct nx842_crypto_header *hdr;
+       struct nx842_crypto_param p;
+       struct nx842_constraints c = *nx842_platform_driver()->constraints;
+       int n, ret, hdr_len;
+       u16 ignore = 0;
+
+       check_constraints(&c);
+
+       p.in = (u8 *)src;
+       p.iremain = slen;
+       p.out = dst;
+       p.oremain = *dlen;
+       p.ototal = 0;
+
+       *dlen = 0;
+
+       hdr = (struct nx842_crypto_header *)src;
+
+       spin_lock_bh(&ctx->lock);
+
+       /* If it doesn't start with our header magic number, assume it's a raw
+        * 842 compressed buffer and pass it directly to the hardware driver
+        */
+       if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) {
+               struct nx842_crypto_header_group g = {
+                       .padding =              0,
+                       .compressed_length =    cpu_to_be32(p.iremain),
+                       .uncompressed_length =  cpu_to_be32(p.oremain),
+               };
+
+               ret = decompress(ctx, &p, &g, &c, 0);
+               if (ret)
+                       goto unlock;
+
+               goto success;
+       }
+
+       if (!hdr->groups) {
+               pr_err("header has no groups\n");
+               ret = -EINVAL;
+               goto unlock;
+       }
+       if (hdr->groups > NX842_CRYPTO_GROUP_MAX) {
+               pr_err("header has too many groups %x, max %x\n",
+                      hdr->groups, NX842_CRYPTO_GROUP_MAX);
+               ret = -EINVAL;
+               goto unlock;
+       }
+
+       hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
+       if (hdr_len > slen) {
+               ret = -EOVERFLOW;
+               goto unlock;
+       }
+
+       memcpy(&ctx->header, src, hdr_len);
+       hdr = &ctx->header;
+
+       for (n = 0; n < hdr->groups; n++) {
+               /* ignore applies to last group */
+               if (n + 1 == hdr->groups)
+                       ignore = be16_to_cpu(hdr->ignore);
+
+               ret = decompress(ctx, &p, &hdr->group[n], &c, ignore);
+               if (ret)
+                       goto unlock;
+       }
+
+success:
+       *dlen = p.ototal;
+
+       pr_debug("decompress total slen %x dlen %x\n", slen, *dlen);
+
+       ret = 0;
+
+unlock:
+       spin_unlock_bh(&ctx->lock);
+
+       return ret;
+}
+
+static struct crypto_alg alg = {
+       .cra_name               = "842",
+       .cra_driver_name        = "842-nx",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
+       .cra_ctxsize            = sizeof(struct nx842_crypto_ctx),
+       .cra_module             = THIS_MODULE,
+       .cra_init               = nx842_crypto_init,
+       .cra_exit               = nx842_crypto_exit,
+       .cra_u                  = { .compress = {
+       .coa_compress           = nx842_crypto_compress,
+       .coa_decompress         = nx842_crypto_decompress } }
+};
+
+static int __init nx842_crypto_mod_init(void)
+{
+       request_module("nx-compress-powernv");
+       request_module("nx-compress-pseries");
+
+       /* we prevent loading/registering if there's no platform driver,
+        * and we get the platform module that set it so it won't unload,
+        * so we don't need to check if it's set in any of our functions
+        */
+       if (!nx842_platform_driver_get()) {
+               pr_err("no nx842 platform driver found.\n");
+               return -ENODEV;
+       }
+
+       return crypto_register_alg(&alg);
+}
+module_init(nx842_crypto_mod_init);
+
+static void __exit nx842_crypto_mod_exit(void)
+{
+       crypto_unregister_alg(&alg);
+
+       nx842_platform_driver_put();
+}
+module_exit(nx842_crypto_mod_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Driver");
+MODULE_ALIAS_CRYPTO("842");
+MODULE_ALIAS_CRYPTO("842-nx");
+MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");