drivers/crypto/omap-des.c

   1 /*
   2  * Support for OMAP DES and Triple DES HW acceleration.
   3  *
   4  * Copyright (c) 2013 Texas Instruments Incorporated
   5  * Author: Joel Fernandes <joelf@ti.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as published
   9  * by the Free Software Foundation.
  10  *
  11  */
  12
  13 #define pr_fmt(fmt) "%s: " fmt, __func__
  14
  15 #ifdef DEBUG
  16 #define prn(num) printk(#num "=%d\n", num)
  17 #define prx(num) printk(#num "=%x\n", num)
  18 #else
  19 #define prn(num) do { } while (0)
  20 #define prx(num)  do { } while (0)
  21 #endif
  22
  23 #include <linux/err.h>
  24 #include <linux/module.h>
  25 #include <linux/init.h>
  26 #include <linux/errno.h>
  27 #include <linux/kernel.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/scatterlist.h>
  30 #include <linux/dma-mapping.h>
  31 #include <linux/dmaengine.h>
  32 #include <linux/omap-dma.h>
  33 #include <linux/pm_runtime.h>
  34 #include <linux/of.h>
  35 #include <linux/of_device.h>
  36 #include <linux/of_address.h>
  37 #include <linux/io.h>
  38 #include <linux/crypto.h>
  39 #include <linux/interrupt.h>
  40 #include <crypto/scatterwalk.h>
  41 #include <crypto/des.h>
  42
  43 #define DST_MAXBURST                    2
  44
  45 #define DES_BLOCK_WORDS         (DES_BLOCK_SIZE >> 2)
  46
  47 #define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset)
  48
  49 #define DES_REG_KEY(dd, x)              ((dd)->pdata->key_ofs - \
  50                                                 ((x ^ 0x01) * 0x04))
  51
  52 #define DES_REG_IV(dd, x)               ((dd)->pdata->iv_ofs + ((x) * 0x04))
  53
  54 #define DES_REG_CTRL(dd)                ((dd)->pdata->ctrl_ofs)
  55 #define DES_REG_CTRL_CBC                BIT(4)
  56 #define DES_REG_CTRL_TDES               BIT(3)
  57 #define DES_REG_CTRL_DIRECTION          BIT(2)
  58 #define DES_REG_CTRL_INPUT_READY        BIT(1)
  59 #define DES_REG_CTRL_OUTPUT_READY       BIT(0)
  60
  61 #define DES_REG_DATA_N(dd, x)           ((dd)->pdata->data_ofs + ((x) * 0x04))
  62
  63 #define DES_REG_REV(dd)                 ((dd)->pdata->rev_ofs)
  64
  65 #define DES_REG_MASK(dd)                ((dd)->pdata->mask_ofs)
  66
  67 #define DES_REG_LENGTH_N(x)             (0x24 + ((x) * 0x04))
  68
  69 #define DES_REG_IRQ_STATUS(dd)         ((dd)->pdata->irq_status_ofs)
  70 #define DES_REG_IRQ_ENABLE(dd)         ((dd)->pdata->irq_enable_ofs)
  71 #define DES_REG_IRQ_DATA_IN            BIT(1)
  72 #define DES_REG_IRQ_DATA_OUT           BIT(2)
  73
  74 #define FLAGS_MODE_MASK         0x000f
  75 #define FLAGS_ENCRYPT           BIT(0)
  76 #define FLAGS_CBC               BIT(1)
  77 #define FLAGS_INIT              BIT(4)
  78 #define FLAGS_BUSY              BIT(6)
  79
  80 struct omap_des_ctx {
  81         struct omap_des_dev *dd;
  82
  83         int             keylen;
  84         u32             key[(3 * DES_KEY_SIZE) / sizeof(u32)];
  85         unsigned long   flags;
  86 };
  87
  88 struct omap_des_reqctx {
  89         unsigned long mode;
  90 };
  91
  92 #define OMAP_DES_QUEUE_LENGTH   1
  93 #define OMAP_DES_CACHE_SIZE     0
  94
  95 struct omap_des_algs_info {
  96         struct crypto_alg       *algs_list;
  97         unsigned int            size;
  98         unsigned int            registered;
  99 };
 100
 101 struct omap_des_pdata {
 102         struct omap_des_algs_info       *algs_info;
 103         unsigned int    algs_info_size;
 104
 105         void            (*trigger)(struct omap_des_dev *dd, int length);
 106
 107         u32             key_ofs;
 108         u32             iv_ofs;
 109         u32             ctrl_ofs;
 110         u32             data_ofs;
 111         u32             rev_ofs;
 112         u32             mask_ofs;
 113         u32             irq_enable_ofs;
 114         u32             irq_status_ofs;
 115
 116         u32             dma_enable_in;
 117         u32             dma_enable_out;
 118         u32             dma_start;
 119
 120         u32             major_mask;
 121         u32             major_shift;
 122         u32             minor_mask;
 123         u32             minor_shift;
 124 };
 125
 126 struct omap_des_dev {
 127         struct list_head        list;
 128         unsigned long           phys_base;
 129         void __iomem            *io_base;
 130         struct omap_des_ctx     *ctx;
 131         struct device           *dev;
 132         unsigned long           flags;
 133         int                     err;
 134
 135         /* spinlock used for queues */
 136         spinlock_t              lock;
 137         struct crypto_queue     queue;
 138
 139         struct tasklet_struct   done_task;
 140         struct tasklet_struct   queue_task;
 141
 142         struct ablkcipher_request       *req;
 143         /*
 144          * total is used by PIO mode for book keeping so introduce
 145          * variable total_save as need it to calc page_order
 146          */
 147         size_t                          total;
 148         size_t                          total_save;
 149
 150         struct scatterlist              *in_sg;
 151         struct scatterlist              *out_sg;
 152
 153         /* Buffers for copying for unaligned cases */
 154         struct scatterlist              in_sgl;
 155         struct scatterlist              out_sgl;
 156         struct scatterlist              *orig_out;
 157         int                             sgs_copied;
 158
 159         struct scatter_walk             in_walk;
 160         struct scatter_walk             out_walk;
 161         int                     dma_in;
 162         struct dma_chan         *dma_lch_in;
 163         int                     dma_out;
 164         struct dma_chan         *dma_lch_out;
 165         int                     in_sg_len;
 166         int                     out_sg_len;
 167         int                     pio_only;
 168         const struct omap_des_pdata     *pdata;
 169 };
 170
 171 /* keep registered devices data here */
 172 static LIST_HEAD(dev_list);
 173 static DEFINE_SPINLOCK(list_lock);
 174
 175 #ifdef DEBUG
 176 #define omap_des_read(dd, offset)                               \
 177         ({                                                              \
 178          int _read_ret;                                          \
 179          _read_ret = __raw_readl(dd->io_base + offset);          \
 180          pr_err("omap_des_read(" #offset "=%#x)= %#x\n",       \
 181                  offset, _read_ret);                            \
 182          _read_ret;                                              \
 183          })
 184 #else
 185 static inline u32 omap_des_read(struct omap_des_dev *dd, u32 offset)
 186 {
 187         return __raw_readl(dd->io_base + offset);
 188 }
 189 #endif
 190
 191 #ifdef DEBUG
 192 #define omap_des_write(dd, offset, value)                               \
 193         do {                                                            \
 194                 pr_err("omap_des_write(" #offset "=%#x) value=%#x\n", \
 195                                 offset, value);                                \
 196                 __raw_writel(value, dd->io_base + offset);              \
 197         } while (0)
 198 #else
 199 static inline void omap_des_write(struct omap_des_dev *dd, u32 offset,
 200                 u32 value)
 201 {
 202         __raw_writel(value, dd->io_base + offset);
 203 }
 204 #endif
 205
 206 static inline void omap_des_write_mask(struct omap_des_dev *dd, u32 offset,
 207                                         u32 value, u32 mask)
 208 {
 209         u32 val;
 210
 211         val = omap_des_read(dd, offset);
 212         val &= ~mask;
 213         val |= value;
 214         omap_des_write(dd, offset, val);
 215 }
 216
 217 static void omap_des_write_n(struct omap_des_dev *dd, u32 offset,
 218                                         u32 *value, int count)
 219 {
 220         for (; count--; value++, offset += 4)
 221                 omap_des_write(dd, offset, *value);
 222 }
 223
 224 static int omap_des_hw_init(struct omap_des_dev *dd)
 225 {
 226         /*
 227          * clocks are enabled when request starts and disabled when finished.
 228          * It may be long delays between requests.
 229          * Device might go to off mode to save power.
 230          */
 231         pm_runtime_get_sync(dd->dev);
 232
 233         if (!(dd->flags & FLAGS_INIT)) {
 234                 dd->flags |= FLAGS_INIT;
 235                 dd->err = 0;
 236         }
 237
 238         return 0;
 239 }
 240
 241 static int omap_des_write_ctrl(struct omap_des_dev *dd)
 242 {
 243         unsigned int key32;
 244         int i, err;
 245         u32 val = 0, mask = 0;
 246
 247         err = omap_des_hw_init(dd);
 248         if (err)
 249                 return err;
 250
 251         key32 = dd->ctx->keylen / sizeof(u32);
 252
 253         /* it seems a key should always be set even if it has not changed */
 254         for (i = 0; i < key32; i++) {
 255                 omap_des_write(dd, DES_REG_KEY(dd, i),
 256                                __le32_to_cpu(dd->ctx->key[i]));
 257         }
 258
 259         if ((dd->flags & FLAGS_CBC) && dd->req->info)
 260                 omap_des_write_n(dd, DES_REG_IV(dd, 0), dd->req->info, 2);
 261
 262         if (dd->flags & FLAGS_CBC)
 263                 val |= DES_REG_CTRL_CBC;
 264         if (dd->flags & FLAGS_ENCRYPT)
 265                 val |= DES_REG_CTRL_DIRECTION;
 266         if (key32 == 6)
 267                 val |= DES_REG_CTRL_TDES;
 268
 269         mask |= DES_REG_CTRL_CBC | DES_REG_CTRL_DIRECTION | DES_REG_CTRL_TDES;
 270
 271         omap_des_write_mask(dd, DES_REG_CTRL(dd), val, mask);
 272
 273         return 0;
 274 }
 275
 276 static void omap_des_dma_trigger_omap4(struct omap_des_dev *dd, int length)
 277 {
 278         u32 mask, val;
 279
 280         omap_des_write(dd, DES_REG_LENGTH_N(0), length);
 281
 282         val = dd->pdata->dma_start;
 283
 284         if (dd->dma_lch_out != NULL)
 285                 val |= dd->pdata->dma_enable_out;
 286         if (dd->dma_lch_in != NULL)
 287                 val |= dd->pdata->dma_enable_in;
 288
 289         mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
 290                dd->pdata->dma_start;
 291
 292         omap_des_write_mask(dd, DES_REG_MASK(dd), val, mask);
 293 }
 294
 295 static void omap_des_dma_stop(struct omap_des_dev *dd)
 296 {
 297         u32 mask;
 298
 299         mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
 300                dd->pdata->dma_start;
 301
 302         omap_des_write_mask(dd, DES_REG_MASK(dd), 0, mask);
 303 }
 304
 305 static struct omap_des_dev *omap_des_find_dev(struct omap_des_ctx *ctx)
 306 {
 307         struct omap_des_dev *dd = NULL, *tmp;
 308
 309         spin_lock_bh(&list_lock);
 310         if (!ctx->dd) {
 311                 list_for_each_entry(tmp, &dev_list, list) {
 312                         /* FIXME: take fist available des core */
 313                         dd = tmp;
 314                         break;
 315                 }
 316                 ctx->dd = dd;
 317         } else {
 318                 /* already found before */
 319                 dd = ctx->dd;
 320         }
 321         spin_unlock_bh(&list_lock);
 322
 323         return dd;
 324 }
 325
 326 static void omap_des_dma_out_callback(void *data)
 327 {
 328         struct omap_des_dev *dd = data;
 329
 330         /* dma_lch_out - completed */
 331         tasklet_schedule(&dd->done_task);
 332 }
 333
 334 static int omap_des_dma_init(struct omap_des_dev *dd)
 335 {
 336         int err = -ENOMEM;
 337         dma_cap_mask_t mask;
 338
 339         dd->dma_lch_out = NULL;
 340         dd->dma_lch_in = NULL;
 341
 342         dma_cap_zero(mask);
 343         dma_cap_set(DMA_SLAVE, mask);
 344
 345         dd->dma_lch_in = dma_request_slave_channel_compat(mask,
 346                                                           omap_dma_filter_fn,
 347                                                           &dd->dma_in,
 348                                                           dd->dev, "rx");
 349         if (!dd->dma_lch_in) {
 350                 dev_err(dd->dev, "Unable to request in DMA channel\n");
 351                 goto err_dma_in;
 352         }
 353
 354         dd->dma_lch_out = dma_request_slave_channel_compat(mask,
 355                                                            omap_dma_filter_fn,
 356                                                            &dd->dma_out,
 357                                                            dd->dev, "tx");
 358         if (!dd->dma_lch_out) {
 359                 dev_err(dd->dev, "Unable to request out DMA channel\n");
 360                 goto err_dma_out;
 361         }
 362
 363         return 0;
 364
 365 err_dma_out:
 366         dma_release_channel(dd->dma_lch_in);
 367 err_dma_in:
 368         if (err)
 369                 pr_err("error: %d\n", err);
 370         return err;
 371 }
 372
 373 static void omap_des_dma_cleanup(struct omap_des_dev *dd)
 374 {
 375         dma_release_channel(dd->dma_lch_out);
 376         dma_release_channel(dd->dma_lch_in);
 377 }
 378
 379 static void sg_copy_buf(void *buf, struct scatterlist *sg,
 380                               unsigned int start, unsigned int nbytes, int out)
 381 {
 382         struct scatter_walk walk;
 383
 384         if (!nbytes)
 385                 return;
 386
 387         scatterwalk_start(&walk, sg);
 388         scatterwalk_advance(&walk, start);
 389         scatterwalk_copychunks(buf, &walk, nbytes, out);
 390         scatterwalk_done(&walk, out, 0);
 391 }
 392
 393 static int omap_des_crypt_dma(struct crypto_tfm *tfm,
 394                 struct scatterlist *in_sg, struct scatterlist *out_sg,
 395                 int in_sg_len, int out_sg_len)
 396 {
 397         struct omap_des_ctx *ctx = crypto_tfm_ctx(tfm);
 398         struct omap_des_dev *dd = ctx->dd;
 399         struct dma_async_tx_descriptor *tx_in, *tx_out;
 400         struct dma_slave_config cfg;
 401         int ret;
 402
 403         if (dd->pio_only) {
 404                 scatterwalk_start(&dd->in_walk, dd->in_sg);
 405                 scatterwalk_start(&dd->out_walk, dd->out_sg);
 406
 407                 /* Enable DATAIN interrupt and let it take
 408                    care of the rest */
 409                 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2);
 410                 return 0;
 411         }
 412
 413         dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);
 414
 415         memset(&cfg, 0, sizeof(cfg));
 416
 417         cfg.src_addr = dd->phys_base + DES_REG_DATA_N(dd, 0);
 418         cfg.dst_addr = dd->phys_base + DES_REG_DATA_N(dd, 0);
 419         cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 420         cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 421         cfg.src_maxburst = DST_MAXBURST;
 422         cfg.dst_maxburst = DST_MAXBURST;
 423
 424         /* IN */
 425         ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);
 426         if (ret) {
 427                 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
 428                         ret);
 429                 return ret;
 430         }
 431
 432         tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,
 433                                         DMA_MEM_TO_DEV,
 434                                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 435         if (!tx_in) {
 436                 dev_err(dd->dev, "IN prep_slave_sg() failed\n");
 437                 return -EINVAL;
 438         }
 439
 440         /* No callback necessary */
 441         tx_in->callback_param = dd;
 442
 443         /* OUT */
 444         ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);
 445         if (ret) {
 446                 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
 447                         ret);
 448                 return ret;
 449         }
 450
 451         tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len,
 452                                         DMA_DEV_TO_MEM,
 453                                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 454         if (!tx_out) {
 455                 dev_err(dd->dev, "OUT prep_slave_sg() failed\n");
 456                 return -EINVAL;
 457         }
 458
 459         tx_out->callback = omap_des_dma_out_callback;
 460         tx_out->callback_param = dd;
 461
 462         dmaengine_submit(tx_in);
 463         dmaengine_submit(tx_out);
 464
 465         dma_async_issue_pending(dd->dma_lch_in);
 466         dma_async_issue_pending(dd->dma_lch_out);
 467
 468         /* start DMA */
 469         dd->pdata->trigger(dd, dd->total);
 470
 471         return 0;
 472 }
 473
 474 static int omap_des_crypt_dma_start(struct omap_des_dev *dd)
 475 {
 476         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
 477                                         crypto_ablkcipher_reqtfm(dd->req));
 478         int err;
 479
 480         pr_debug("total: %d\n", dd->total);
 481
 482         if (!dd->pio_only) {
 483                 err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,
 484                                  DMA_TO_DEVICE);
 485                 if (!err) {
 486                         dev_err(dd->dev, "dma_map_sg() error\n");
 487                         return -EINVAL;
 488                 }
 489
 490                 err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,
 491                                  DMA_FROM_DEVICE);
 492                 if (!err) {
 493                         dev_err(dd->dev, "dma_map_sg() error\n");
 494                         return -EINVAL;
 495                 }
 496         }
 497
 498         err = omap_des_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len,
 499                                  dd->out_sg_len);
 500         if (err && !dd->pio_only) {
 501                 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
 502                 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
 503                              DMA_FROM_DEVICE);
 504         }
 505
 506         return err;
 507 }
 508
 509 static void omap_des_finish_req(struct omap_des_dev *dd, int err)
 510 {
 511         struct ablkcipher_request *req = dd->req;
 512
 513         pr_debug("err: %d\n", err);
 514
 515         pm_runtime_put(dd->dev);
 516         dd->flags &= ~FLAGS_BUSY;
 517
 518         req->base.complete(&req->base, err);
 519 }
 520
 521 static int omap_des_crypt_dma_stop(struct omap_des_dev *dd)
 522 {
 523         int err = 0;
 524
 525         pr_debug("total: %d\n", dd->total);
 526
 527         omap_des_dma_stop(dd);
 528
 529         dmaengine_terminate_all(dd->dma_lch_in);
 530         dmaengine_terminate_all(dd->dma_lch_out);
 531
 532         dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
 533         dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
 534
 535         return err;
 536 }
 537
 538 static int omap_des_copy_needed(struct scatterlist *sg)
 539 {
 540         while (sg) {
 541                 if (!IS_ALIGNED(sg->offset, 4))
 542                         return -1;
 543                 if (!IS_ALIGNED(sg->length, DES_BLOCK_SIZE))
 544                         return -1;
 545                 sg = sg_next(sg);
 546         }
 547         return 0;
 548 }
 549
 550 static int omap_des_copy_sgs(struct omap_des_dev *dd)
 551 {
 552         void *buf_in, *buf_out;
 553         int pages;
 554
 555         pages = dd->total >> PAGE_SHIFT;
 556
 557         if (dd->total & (PAGE_SIZE-1))
 558                 pages++;
 559
 560         BUG_ON(!pages);
 561
 562         buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
 563         buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
 564
 565         if (!buf_in || !buf_out) {
 566                 pr_err("Couldn't allocated pages for unaligned cases.\n");
 567                 return -1;
 568         }
 569
 570         dd->orig_out = dd->out_sg;
 571
 572         sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0);
 573
 574         sg_init_table(&dd->in_sgl, 1);
 575         sg_set_buf(&dd->in_sgl, buf_in, dd->total);
 576         dd->in_sg = &dd->in_sgl;
 577
 578         sg_init_table(&dd->out_sgl, 1);
 579         sg_set_buf(&dd->out_sgl, buf_out, dd->total);
 580         dd->out_sg = &dd->out_sgl;
 581
 582         return 0;
 583 }
 584
 585 static int omap_des_handle_queue(struct omap_des_dev *dd,
 586                                struct ablkcipher_request *req)
 587 {
 588         struct crypto_async_request *async_req, *backlog;
 589         struct omap_des_ctx *ctx;
 590         struct omap_des_reqctx *rctx;
 591         unsigned long flags;
 592         int err, ret = 0;
 593
 594         spin_lock_irqsave(&dd->lock, flags);
 595         if (req)
 596                 ret = ablkcipher_enqueue_request(&dd->queue, req);
 597         if (dd->flags & FLAGS_BUSY) {
 598                 spin_unlock_irqrestore(&dd->lock, flags);
 599                 return ret;
 600         }
 601         backlog = crypto_get_backlog(&dd->queue);
 602         async_req = crypto_dequeue_request(&dd->queue);
 603         if (async_req)
 604                 dd->flags |= FLAGS_BUSY;
 605         spin_unlock_irqrestore(&dd->lock, flags);
 606
 607         if (!async_req)
 608                 return ret;
 609
 610         if (backlog)
 611                 backlog->complete(backlog, -EINPROGRESS);
 612
 613         req = ablkcipher_request_cast(async_req);
 614
 615         /* assign new request to device */
 616         dd->req = req;
 617         dd->total = req->nbytes;
 618         dd->total_save = req->nbytes;
 619         dd->in_sg = req->src;
 620         dd->out_sg = req->dst;
 621
 622         if (omap_des_copy_needed(dd->in_sg) ||
 623             omap_des_copy_needed(dd->out_sg)) {
 624                 if (omap_des_copy_sgs(dd))
 625                         pr_err("Failed to copy SGs for unaligned cases\n");
 626                 dd->sgs_copied = 1;
 627         } else {
 628                 dd->sgs_copied = 0;
 629         }
 630
 631         dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total);
 632         dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, dd->total);
 633         BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0);
 634
 635         rctx = ablkcipher_request_ctx(req);
 636         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
 637         rctx->mode &= FLAGS_MODE_MASK;
 638         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 639
 640         dd->ctx = ctx;
 641         ctx->dd = dd;
 642
 643         err = omap_des_write_ctrl(dd);
 644         if (!err)
 645                 err = omap_des_crypt_dma_start(dd);
 646         if (err) {
 647                 /* des_task will not finish it, so do it here */
 648                 omap_des_finish_req(dd, err);
 649                 tasklet_schedule(&dd->queue_task);
 650         }
 651
 652         return ret; /* return ret, which is enqueue return value */
 653 }
 654
 655 static void omap_des_done_task(unsigned long data)
 656 {
 657         struct omap_des_dev *dd = (struct omap_des_dev *)data;
 658         void *buf_in, *buf_out;
 659         int pages;
 660
 661         pr_debug("enter done_task\n");
 662
 663         if (!dd->pio_only) {
 664                 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
 665                                        DMA_FROM_DEVICE);
 666                 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
 667                 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
 668                              DMA_FROM_DEVICE);
 669                 omap_des_crypt_dma_stop(dd);
 670         }
 671
 672         if (dd->sgs_copied) {
 673                 buf_in = sg_virt(&dd->in_sgl);
 674                 buf_out = sg_virt(&dd->out_sgl);
 675
 676                 sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1);
 677
 678                 pages = get_order(dd->total_save);
 679                 free_pages((unsigned long)buf_in, pages);
 680                 free_pages((unsigned long)buf_out, pages);
 681         }
 682
 683         omap_des_finish_req(dd, 0);
 684         omap_des_handle_queue(dd, NULL);
 685
 686         pr_debug("exit\n");
 687 }
 688
 689 static void omap_des_queue_task(unsigned long data)
 690 {
 691         struct omap_des_dev *dd = (struct omap_des_dev *)data;
 692
 693         omap_des_handle_queue(dd, NULL);
 694 }
 695
 696 static int omap_des_crypt(struct ablkcipher_request *req, unsigned long mode)
 697 {
 698         struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(
 699                         crypto_ablkcipher_reqtfm(req));
 700         struct omap_des_reqctx *rctx = ablkcipher_request_ctx(req);
 701         struct omap_des_dev *dd;
 702
 703         pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
 704                  !!(mode & FLAGS_ENCRYPT),
 705                  !!(mode & FLAGS_CBC));
 706
 707         if (!IS_ALIGNED(req->nbytes, DES_BLOCK_SIZE)) {
 708                 pr_err("request size is not exact amount of DES blocks\n");
 709                 return -EINVAL;
 710         }
 711
 712         dd = omap_des_find_dev(ctx);
 713         if (!dd)
 714                 return -ENODEV;
 715
 716         rctx->mode = mode;
 717
 718         return omap_des_handle_queue(dd, req);
 719 }
 720
 721 /* ********************** ALG API ************************************ */
 722
 723 static int omap_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 724                            unsigned int keylen)
 725 {
 726         struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 727
 728         if (keylen != DES_KEY_SIZE && keylen != (3*DES_KEY_SIZE))
 729                 return -EINVAL;
 730
 731         pr_debug("enter, keylen: %d\n", keylen);
 732
 733         memcpy(ctx->key, key, keylen);
 734         ctx->keylen = keylen;
 735
 736         return 0;
 737 }
 738
 739 static int omap_des_ecb_encrypt(struct ablkcipher_request *req)
 740 {
 741         return omap_des_crypt(req, FLAGS_ENCRYPT);
 742 }
 743
 744 static int omap_des_ecb_decrypt(struct ablkcipher_request *req)
 745 {
 746         return omap_des_crypt(req, 0);
 747 }
 748
 749 static int omap_des_cbc_encrypt(struct ablkcipher_request *req)
 750 {
 751         return omap_des_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
 752 }
 753
 754 static int omap_des_cbc_decrypt(struct ablkcipher_request *req)
 755 {
 756         return omap_des_crypt(req, FLAGS_CBC);
 757 }
 758
 759 static int omap_des_cra_init(struct crypto_tfm *tfm)
 760 {
 761         pr_debug("enter\n");
 762
 763         tfm->crt_ablkcipher.reqsize = sizeof(struct omap_des_reqctx);
 764
 765         return 0;
 766 }
 767
 768 static void omap_des_cra_exit(struct crypto_tfm *tfm)
 769 {
 770         pr_debug("enter\n");
 771 }
 772
 773 /* ********************** ALGS ************************************ */
 774
 775 static struct crypto_alg algs_ecb_cbc[] = {
 776 {
 777         .cra_name               = "ecb(des)",
 778         .cra_driver_name        = "ecb-des-omap",
 779         .cra_priority           = 100,
 780         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 781                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 782                                   CRYPTO_ALG_ASYNC,
 783         .cra_blocksize          = DES_BLOCK_SIZE,
 784         .cra_ctxsize            = sizeof(struct omap_des_ctx),
 785         .cra_alignmask          = 0,
 786         .cra_type               = &crypto_ablkcipher_type,
 787         .cra_module             = THIS_MODULE,
 788         .cra_init               = omap_des_cra_init,
 789         .cra_exit               = omap_des_cra_exit,
 790         .cra_u.ablkcipher = {
 791                 .min_keysize    = DES_KEY_SIZE,
 792                 .max_keysize    = DES_KEY_SIZE,
 793                 .setkey         = omap_des_setkey,
 794                 .encrypt        = omap_des_ecb_encrypt,
 795                 .decrypt        = omap_des_ecb_decrypt,
 796         }
 797 },
 798 {
 799         .cra_name               = "cbc(des)",
 800         .cra_driver_name        = "cbc-des-omap",
 801         .cra_priority           = 100,
 802         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 803                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 804                                   CRYPTO_ALG_ASYNC,
 805         .cra_blocksize          = DES_BLOCK_SIZE,
 806         .cra_ctxsize            = sizeof(struct omap_des_ctx),
 807         .cra_alignmask          = 0,
 808         .cra_type               = &crypto_ablkcipher_type,
 809         .cra_module             = THIS_MODULE,
 810         .cra_init               = omap_des_cra_init,
 811         .cra_exit               = omap_des_cra_exit,
 812         .cra_u.ablkcipher = {
 813                 .min_keysize    = DES_KEY_SIZE,
 814                 .max_keysize    = DES_KEY_SIZE,
 815                 .ivsize         = DES_BLOCK_SIZE,
 816                 .setkey         = omap_des_setkey,
 817                 .encrypt        = omap_des_cbc_encrypt,
 818                 .decrypt        = omap_des_cbc_decrypt,
 819         }
 820 },
 821 {
 822         .cra_name               = "ecb(des3_ede)",
 823         .cra_driver_name        = "ecb-des3-omap",
 824         .cra_priority           = 100,
 825         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 826                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 827                                   CRYPTO_ALG_ASYNC,
 828         .cra_blocksize          = DES_BLOCK_SIZE,
 829         .cra_ctxsize            = sizeof(struct omap_des_ctx),
 830         .cra_alignmask          = 0,
 831         .cra_type               = &crypto_ablkcipher_type,
 832         .cra_module             = THIS_MODULE,
 833         .cra_init               = omap_des_cra_init,
 834         .cra_exit               = omap_des_cra_exit,
 835         .cra_u.ablkcipher = {
 836                 .min_keysize    = 3*DES_KEY_SIZE,
 837                 .max_keysize    = 3*DES_KEY_SIZE,
 838                 .setkey         = omap_des_setkey,
 839                 .encrypt        = omap_des_ecb_encrypt,
 840                 .decrypt        = omap_des_ecb_decrypt,
 841         }
 842 },
 843 {
 844         .cra_name               = "cbc(des3_ede)",
 845         .cra_driver_name        = "cbc-des3-omap",
 846         .cra_priority           = 100,
 847         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 848                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 849                                   CRYPTO_ALG_ASYNC,
 850         .cra_blocksize          = DES_BLOCK_SIZE,
 851         .cra_ctxsize            = sizeof(struct omap_des_ctx),
 852         .cra_alignmask          = 0,
 853         .cra_type               = &crypto_ablkcipher_type,
 854         .cra_module             = THIS_MODULE,
 855         .cra_init               = omap_des_cra_init,
 856         .cra_exit               = omap_des_cra_exit,
 857         .cra_u.ablkcipher = {
 858                 .min_keysize    = 3*DES_KEY_SIZE,
 859                 .max_keysize    = 3*DES_KEY_SIZE,
 860                 .ivsize         = DES_BLOCK_SIZE,
 861                 .setkey         = omap_des_setkey,
 862                 .encrypt        = omap_des_cbc_encrypt,
 863                 .decrypt        = omap_des_cbc_decrypt,
 864         }
 865 }
 866 };
 867
 868 static struct omap_des_algs_info omap_des_algs_info_ecb_cbc[] = {
 869         {
 870                 .algs_list      = algs_ecb_cbc,
 871                 .size           = ARRAY_SIZE(algs_ecb_cbc),
 872         },
 873 };
 874
 875 #ifdef CONFIG_OF
 876 static const struct omap_des_pdata omap_des_pdata_omap4 = {
 877         .algs_info      = omap_des_algs_info_ecb_cbc,
 878         .algs_info_size = ARRAY_SIZE(omap_des_algs_info_ecb_cbc),
 879         .trigger        = omap_des_dma_trigger_omap4,
 880         .key_ofs        = 0x14,
 881         .iv_ofs         = 0x18,
 882         .ctrl_ofs       = 0x20,
 883         .data_ofs       = 0x28,
 884         .rev_ofs        = 0x30,
 885         .mask_ofs       = 0x34,
 886         .irq_status_ofs = 0x3c,
 887         .irq_enable_ofs = 0x40,
 888         .dma_enable_in  = BIT(5),
 889         .dma_enable_out = BIT(6),
 890         .major_mask     = 0x0700,
 891         .major_shift    = 8,
 892         .minor_mask     = 0x003f,
 893         .minor_shift    = 0,
 894 };
 895
 896 static irqreturn_t omap_des_irq(int irq, void *dev_id)
 897 {
 898         struct omap_des_dev *dd = dev_id;
 899         u32 status, i;
 900         u32 *src, *dst;
 901
 902         status = omap_des_read(dd, DES_REG_IRQ_STATUS(dd));
 903         if (status & DES_REG_IRQ_DATA_IN) {
 904                 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0);
 905
 906                 BUG_ON(!dd->in_sg);
 907
 908                 BUG_ON(_calc_walked(in) > dd->in_sg->length);
 909
 910                 src = sg_virt(dd->in_sg) + _calc_walked(in);
 911
 912                 for (i = 0; i < DES_BLOCK_WORDS; i++) {
 913                         omap_des_write(dd, DES_REG_DATA_N(dd, i), *src);
 914
 915                         scatterwalk_advance(&dd->in_walk, 4);
 916                         if (dd->in_sg->length == _calc_walked(in)) {
 917                                 dd->in_sg = scatterwalk_sg_next(dd->in_sg);
 918                                 if (dd->in_sg) {
 919                                         scatterwalk_start(&dd->in_walk,
 920                                                           dd->in_sg);
 921                                         src = sg_virt(dd->in_sg) +
 922                                               _calc_walked(in);
 923                                 }
 924                         } else {
 925                                 src++;
 926                         }
 927                 }
 928
 929                 /* Clear IRQ status */
 930                 status &= ~DES_REG_IRQ_DATA_IN;
 931                 omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status);
 932
 933                 /* Enable DATA_OUT interrupt */
 934                 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x4);
 935
 936         } else if (status & DES_REG_IRQ_DATA_OUT) {
 937                 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0);
 938
 939                 BUG_ON(!dd->out_sg);
 940
 941                 BUG_ON(_calc_walked(out) > dd->out_sg->length);
 942
 943                 dst = sg_virt(dd->out_sg) + _calc_walked(out);
 944
 945                 for (i = 0; i < DES_BLOCK_WORDS; i++) {
 946                         *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i));
 947                         scatterwalk_advance(&dd->out_walk, 4);
 948                         if (dd->out_sg->length == _calc_walked(out)) {
 949                                 dd->out_sg = scatterwalk_sg_next(dd->out_sg);
 950                                 if (dd->out_sg) {
 951                                         scatterwalk_start(&dd->out_walk,
 952                                                           dd->out_sg);
 953                                         dst = sg_virt(dd->out_sg) +
 954                                               _calc_walked(out);
 955                                 }
 956                         } else {
 957                                 dst++;
 958                         }
 959                 }
 960
 961                 dd->total -= DES_BLOCK_SIZE;
 962
 963                 BUG_ON(dd->total < 0);
 964
 965                 /* Clear IRQ status */
 966                 status &= ~DES_REG_IRQ_DATA_OUT;
 967                 omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status);
 968
 969                 if (!dd->total)
 970                         /* All bytes read! */
 971                         tasklet_schedule(&dd->done_task);
 972                 else
 973                         /* Enable DATA_IN interrupt for next block */
 974                         omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2);
 975         }
 976
 977         return IRQ_HANDLED;
 978 }
 979
 980 static const struct of_device_id omap_des_of_match[] = {
 981         {
 982                 .compatible     = "ti,omap4-des",
 983                 .data           = &omap_des_pdata_omap4,
 984         },
 985         {},
 986 };
 987 MODULE_DEVICE_TABLE(of, omap_des_of_match);
 988
 989 static int omap_des_get_of(struct omap_des_dev *dd,
 990                 struct platform_device *pdev)
 991 {
 992         const struct of_device_id *match;
 993
 994         match = of_match_device(of_match_ptr(omap_des_of_match), &pdev->dev);
 995         if (!match) {
 996                 dev_err(&pdev->dev, "no compatible OF match\n");
 997                 return -EINVAL;
 998         }
 999
1000         dd->dma_out = -1; /* Dummy value that's unused */
1001         dd->dma_in = -1; /* Dummy value that's unused */
1002         dd->pdata = match->data;
1003
1004         return 0;
1005 }
1006 #else
1007 static int omap_des_get_of(struct omap_des_dev *dd,
1008                 struct device *dev)
1009 {
1010         return -EINVAL;
1011 }
1012 #endif
1013
1014 static int omap_des_get_pdev(struct omap_des_dev *dd,
1015                 struct platform_device *pdev)
1016 {
1017         struct device *dev = &pdev->dev;
1018         struct resource *r;
1019         int err = 0;
1020
1021         /* Get the DMA out channel */
1022         r = platform_get_resource(pdev, IORESOURCE_DMA, 0);
1023         if (!r) {
1024                 dev_err(dev, "no DMA out resource info\n");
1025                 err = -ENODEV;
1026                 goto err;
1027         }
1028         dd->dma_out = r->start;
1029
1030         /* Get the DMA in channel */
1031         r = platform_get_resource(pdev, IORESOURCE_DMA, 1);
1032         if (!r) {
1033                 dev_err(dev, "no DMA in resource info\n");
1034                 err = -ENODEV;
1035                 goto err;
1036         }
1037         dd->dma_in = r->start;
1038
1039         /* non-DT devices get pdata from pdev */
1040         dd->pdata = pdev->dev.platform_data;
1041
1042 err:
1043         return err;
1044 }
1045
1046 static int omap_des_probe(struct platform_device *pdev)
1047 {
1048         struct device *dev = &pdev->dev;
1049         struct omap_des_dev *dd;
1050         struct crypto_alg *algp;
1051         struct resource *res;
1052         int err = -ENOMEM, i, j, irq = -1;
1053         u32 reg;
1054
1055         dd = devm_kzalloc(dev, sizeof(struct omap_des_dev), GFP_KERNEL);
1056         if (dd == NULL) {
1057                 dev_err(dev, "unable to alloc data struct.\n");
1058                 goto err_data;
1059         }
1060         dd->dev = dev;
1061         platform_set_drvdata(pdev, dd);
1062
1063         spin_lock_init(&dd->lock);
1064         crypto_init_queue(&dd->queue, OMAP_DES_QUEUE_LENGTH);
1065
1066         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1067         if (!res) {
1068                 dev_err(dev, "no MEM resource info\n");
1069                 goto err_res;
1070         }
1071
1072         err = (dev->of_node) ? omap_des_get_of(dd, pdev) :
1073                                omap_des_get_pdev(dd, pdev);
1074         if (err)
1075                 goto err_res;
1076
1077         dd->io_base = devm_request_and_ioremap(dev, res);
1078         if (!dd->io_base) {
1079                 dev_err(dev, "can't ioremap\n");
1080                 err = -ENOMEM;
1081                 goto err_res;
1082         }
1083         dd->phys_base = res->start;
1084
1085         pm_runtime_enable(dev);
1086         pm_runtime_get_sync(dev);
1087
1088         omap_des_dma_stop(dd);
1089
1090         reg = omap_des_read(dd, DES_REG_REV(dd));
1091
1092         pm_runtime_put_sync(dev);
1093
1094         dev_info(dev, "OMAP DES hw accel rev: %u.%u\n",
1095                  (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,
1096                  (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
1097
1098         tasklet_init(&dd->done_task, omap_des_done_task, (unsigned long)dd);
1099         tasklet_init(&dd->queue_task, omap_des_queue_task, (unsigned long)dd);
1100
1101         err = omap_des_dma_init(dd);
1102         if (err && DES_REG_IRQ_STATUS(dd) && DES_REG_IRQ_ENABLE(dd)) {
1103                 dd->pio_only = 1;
1104
1105                 irq = platform_get_irq(pdev, 0);
1106                 if (irq < 0) {
1107                         dev_err(dev, "can't get IRQ resource\n");
1108                         goto err_irq;
1109                 }
1110
1111                 err = devm_request_irq(dev, irq, omap_des_irq, 0,
1112                                 dev_name(dev), dd);
1113                 if (err) {
1114                         dev_err(dev, "Unable to grab omap-des IRQ\n");
1115                         goto err_irq;
1116                 }
1117         }
1118
1119
1120         INIT_LIST_HEAD(&dd->list);
1121         spin_lock(&list_lock);
1122         list_add_tail(&dd->list, &dev_list);
1123         spin_unlock(&list_lock);
1124
1125         for (i = 0; i < dd->pdata->algs_info_size; i++) {
1126                 for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
1127                         algp = &dd->pdata->algs_info[i].algs_list[j];
1128
1129                         pr_debug("reg alg: %s\n", algp->cra_name);
1130                         INIT_LIST_HEAD(&algp->cra_list);
1131
1132                         err = crypto_register_alg(algp);
1133                         if (err)
1134                                 goto err_algs;
1135
1136                         dd->pdata->algs_info[i].registered++;
1137                 }
1138         }
1139
1140         return 0;
1141 err_algs:
1142         for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1143                 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1144                         crypto_unregister_alg(
1145                                         &dd->pdata->algs_info[i].algs_list[j]);
1146         if (!dd->pio_only)
1147                 omap_des_dma_cleanup(dd);
1148 err_irq:
1149         tasklet_kill(&dd->done_task);
1150         tasklet_kill(&dd->queue_task);
1151         pm_runtime_disable(dev);
1152 err_res:
1153         dd = NULL;
1154 err_data:
1155         dev_err(dev, "initialization failed.\n");
1156         return err;
1157 }
1158
1159 static int omap_des_remove(struct platform_device *pdev)
1160 {
1161         struct omap_des_dev *dd = platform_get_drvdata(pdev);
1162         int i, j;
1163
1164         if (!dd)
1165                 return -ENODEV;
1166
1167         spin_lock(&list_lock);
1168         list_del(&dd->list);
1169         spin_unlock(&list_lock);
1170
1171         for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1172                 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1173                         crypto_unregister_alg(
1174                                         &dd->pdata->algs_info[i].algs_list[j]);
1175
1176         tasklet_kill(&dd->done_task);
1177         tasklet_kill(&dd->queue_task);
1178         omap_des_dma_cleanup(dd);
1179         pm_runtime_disable(dd->dev);
1180         dd = NULL;
1181
1182         return 0;
1183 }
1184
1185 #ifdef CONFIG_PM_SLEEP
1186 static int omap_des_suspend(struct device *dev)
1187 {
1188         pm_runtime_put_sync(dev);
1189         return 0;
1190 }
1191
1192 static int omap_des_resume(struct device *dev)
1193 {
1194         pm_runtime_get_sync(dev);
1195         return 0;
1196 }
1197 #endif
1198
1199 static SIMPLE_DEV_PM_OPS(omap_des_pm_ops, omap_des_suspend, omap_des_resume);
1200
1201 static struct platform_driver omap_des_driver = {
1202         .probe  = omap_des_probe,
1203         .remove = omap_des_remove,
1204         .driver = {
1205                 .name   = "omap-des",
1206                 .owner  = THIS_MODULE,
1207                 .pm     = &omap_des_pm_ops,
1208                 .of_match_table = of_match_ptr(omap_des_of_match),
1209         },
1210 };
1211
1212 module_platform_driver(omap_des_driver);
1213
1214 MODULE_DESCRIPTION("OMAP DES hw acceleration support.");
1215 MODULE_LICENSE("GPL v2");
1216 MODULE_AUTHOR("Joel Fernandes <joelf@ti.com>");