drivers/usb/chipidea/udc.c

   1 /*
   2  * udc.c - ChipIdea UDC driver
   3  *
   4  * Copyright (C) 2008 Chipidea - MIPS Technologies, Inc. All rights reserved.
   5  *
   6  * Author: David Lopo
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/delay.h>
  14 #include <linux/device.h>
  15 #include <linux/dmapool.h>
  16 #include <linux/dma-mapping.h>
  17 #include <linux/err.h>
  18 #include <linux/init.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/module.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/io.h>
  23 #include <linux/irq.h>
  24 #include <linux/kernel.h>
  25 #include <linux/slab.h>
  26 #include <linux/pm_runtime.h>
  27 #include <linux/usb/ch9.h>
  28 #include <linux/usb/gadget.h>
  29 #include <linux/usb/otg.h>
  30 #include <linux/usb/chipidea.h>
  31
  32 #include "ci.h"
  33 #include "udc.h"
  34 #include "bits.h"
  35 #include "debug.h"
  36
  37 /* control endpoint description */
  38 static const struct usb_endpoint_descriptor
  39 ctrl_endpt_out_desc = {
  40         .bLength         = USB_DT_ENDPOINT_SIZE,
  41         .bDescriptorType = USB_DT_ENDPOINT,
  42
  43         .bEndpointAddress = USB_DIR_OUT,
  44         .bmAttributes    = USB_ENDPOINT_XFER_CONTROL,
  45         .wMaxPacketSize  = cpu_to_le16(CTRL_PAYLOAD_MAX),
  46 };
  47
  48 static const struct usb_endpoint_descriptor
  49 ctrl_endpt_in_desc = {
  50         .bLength         = USB_DT_ENDPOINT_SIZE,
  51         .bDescriptorType = USB_DT_ENDPOINT,
  52
  53         .bEndpointAddress = USB_DIR_IN,
  54         .bmAttributes    = USB_ENDPOINT_XFER_CONTROL,
  55         .wMaxPacketSize  = cpu_to_le16(CTRL_PAYLOAD_MAX),
  56 };
  57
  58 /**
  59  * hw_ep_bit: calculates the bit number
  60  * @num: endpoint number
  61  * @dir: endpoint direction
  62  *
  63  * This function returns bit number
  64  */
  65 static inline int hw_ep_bit(int num, int dir)
  66 {
  67         return num + (dir ? 16 : 0);
  68 }
  69
  70 static inline int ep_to_bit(struct ci13xxx *ci, int n)
  71 {
  72         int fill = 16 - ci->hw_ep_max / 2;
  73
  74         if (n >= ci->hw_ep_max / 2)
  75                 n += fill;
  76
  77         return n;
  78 }
  79
  80 /**
  81  * hw_device_state: enables/disables interrupts (execute without interruption)
  82  * @dma: 0 => disable, !0 => enable and set dma engine
  83  *
  84  * This function returns an error code
  85  */
  86 static int hw_device_state(struct ci13xxx *ci, u32 dma)
  87 {
  88         if (dma) {
  89                 hw_write(ci, OP_ENDPTLISTADDR, ~0, dma);
  90                 /* interrupt, error, port change, reset, sleep/suspend */
  91                 hw_write(ci, OP_USBINTR, ~0,
  92                              USBi_UI|USBi_UEI|USBi_PCI|USBi_URI|USBi_SLI);
  93         } else {
  94                 hw_write(ci, OP_USBINTR, ~0, 0);
  95         }
  96         return 0;
  97 }
  98
  99 /**
 100  * hw_ep_flush: flush endpoint fifo (execute without interruption)
 101  * @num: endpoint number
 102  * @dir: endpoint direction
 103  *
 104  * This function returns an error code
 105  */
 106 static int hw_ep_flush(struct ci13xxx *ci, int num, int dir)
 107 {
 108         int n = hw_ep_bit(num, dir);
 109
 110         do {
 111                 /* flush any pending transfer */
 112                 hw_write(ci, OP_ENDPTFLUSH, BIT(n), BIT(n));
 113                 while (hw_read(ci, OP_ENDPTFLUSH, BIT(n)))
 114                         cpu_relax();
 115         } while (hw_read(ci, OP_ENDPTSTAT, BIT(n)));
 116
 117         return 0;
 118 }
 119
 120 /**
 121  * hw_ep_disable: disables endpoint (execute without interruption)
 122  * @num: endpoint number
 123  * @dir: endpoint direction
 124  *
 125  * This function returns an error code
 126  */
 127 static int hw_ep_disable(struct ci13xxx *ci, int num, int dir)
 128 {
 129         hw_ep_flush(ci, num, dir);
 130         hw_write(ci, OP_ENDPTCTRL + num,
 131                  dir ? ENDPTCTRL_TXE : ENDPTCTRL_RXE, 0);
 132         return 0;
 133 }
 134
 135 /**
 136  * hw_ep_enable: enables endpoint (execute without interruption)
 137  * @num:  endpoint number
 138  * @dir:  endpoint direction
 139  * @type: endpoint type
 140  *
 141  * This function returns an error code
 142  */
 143 static int hw_ep_enable(struct ci13xxx *ci, int num, int dir, int type)
 144 {
 145         u32 mask, data;
 146
 147         if (dir) {
 148                 mask  = ENDPTCTRL_TXT;  /* type    */
 149                 data  = type << ffs_nr(mask);
 150
 151                 mask |= ENDPTCTRL_TXS;  /* unstall */
 152                 mask |= ENDPTCTRL_TXR;  /* reset data toggle */
 153                 data |= ENDPTCTRL_TXR;
 154                 mask |= ENDPTCTRL_TXE;  /* enable  */
 155                 data |= ENDPTCTRL_TXE;
 156         } else {
 157                 mask  = ENDPTCTRL_RXT;  /* type    */
 158                 data  = type << ffs_nr(mask);
 159
 160                 mask |= ENDPTCTRL_RXS;  /* unstall */
 161                 mask |= ENDPTCTRL_RXR;  /* reset data toggle */
 162                 data |= ENDPTCTRL_RXR;
 163                 mask |= ENDPTCTRL_RXE;  /* enable  */
 164                 data |= ENDPTCTRL_RXE;
 165         }
 166         hw_write(ci, OP_ENDPTCTRL + num, mask, data);
 167         return 0;
 168 }
 169
 170 /**
 171  * hw_ep_get_halt: return endpoint halt status
 172  * @num: endpoint number
 173  * @dir: endpoint direction
 174  *
 175  * This function returns 1 if endpoint halted
 176  */
 177 static int hw_ep_get_halt(struct ci13xxx *ci, int num, int dir)
 178 {
 179         u32 mask = dir ? ENDPTCTRL_TXS : ENDPTCTRL_RXS;
 180
 181         return hw_read(ci, OP_ENDPTCTRL + num, mask) ? 1 : 0;
 182 }
 183
 184 /**
 185  * hw_test_and_clear_setup_status: test & clear setup status (execute without
 186  *                                 interruption)
 187  * @n: endpoint number
 188  *
 189  * This function returns setup status
 190  */
 191 static int hw_test_and_clear_setup_status(struct ci13xxx *ci, int n)
 192 {
 193         n = ep_to_bit(ci, n);
 194         return hw_test_and_clear(ci, OP_ENDPTSETUPSTAT, BIT(n));
 195 }
 196
 197 /**
 198  * hw_ep_prime: primes endpoint (execute without interruption)
 199  * @num:     endpoint number
 200  * @dir:     endpoint direction
 201  * @is_ctrl: true if control endpoint
 202  *
 203  * This function returns an error code
 204  */
 205 static int hw_ep_prime(struct ci13xxx *ci, int num, int dir, int is_ctrl)
 206 {
 207         int n = hw_ep_bit(num, dir);
 208
 209         if (is_ctrl && dir == RX && hw_read(ci, OP_ENDPTSETUPSTAT, BIT(num)))
 210                 return -EAGAIN;
 211
 212         hw_write(ci, OP_ENDPTPRIME, BIT(n), BIT(n));
 213
 214         while (hw_read(ci, OP_ENDPTPRIME, BIT(n)))
 215                 cpu_relax();
 216         if (is_ctrl && dir == RX && hw_read(ci, OP_ENDPTSETUPSTAT, BIT(num)))
 217                 return -EAGAIN;
 218
 219         /* status shoult be tested according with manual but it doesn't work */
 220         return 0;
 221 }
 222
 223 /**
 224  * hw_ep_set_halt: configures ep halt & resets data toggle after clear (execute
 225  *                 without interruption)
 226  * @num:   endpoint number
 227  * @dir:   endpoint direction
 228  * @value: true => stall, false => unstall
 229  *
 230  * This function returns an error code
 231  */
 232 static int hw_ep_set_halt(struct ci13xxx *ci, int num, int dir, int value)
 233 {
 234         if (value != 0 && value != 1)
 235                 return -EINVAL;
 236
 237         do {
 238                 enum ci13xxx_regs reg = OP_ENDPTCTRL + num;
 239                 u32 mask_xs = dir ? ENDPTCTRL_TXS : ENDPTCTRL_RXS;
 240                 u32 mask_xr = dir ? ENDPTCTRL_TXR : ENDPTCTRL_RXR;
 241
 242                 /* data toggle - reserved for EP0 but it's in ESS */
 243                 hw_write(ci, reg, mask_xs|mask_xr,
 244                           value ? mask_xs : mask_xr);
 245         } while (value != hw_ep_get_halt(ci, num, dir));
 246
 247         return 0;
 248 }
 249
 250 /**
 251  * hw_is_port_high_speed: test if port is high speed
 252  *
 253  * This function returns true if high speed port
 254  */
 255 static int hw_port_is_high_speed(struct ci13xxx *ci)
 256 {
 257         return ci->hw_bank.lpm ? hw_read(ci, OP_DEVLC, DEVLC_PSPD) :
 258                 hw_read(ci, OP_PORTSC, PORTSC_HSP);
 259 }
 260
 261 /**
 262  * hw_read_intr_enable: returns interrupt enable register
 263  *
 264  * This function returns register data
 265  */
 266 static u32 hw_read_intr_enable(struct ci13xxx *ci)
 267 {
 268         return hw_read(ci, OP_USBINTR, ~0);
 269 }
 270
 271 /**
 272  * hw_read_intr_status: returns interrupt status register
 273  *
 274  * This function returns register data
 275  */
 276 static u32 hw_read_intr_status(struct ci13xxx *ci)
 277 {
 278         return hw_read(ci, OP_USBSTS, ~0);
 279 }
 280
 281 /**
 282  * hw_test_and_clear_complete: test & clear complete status (execute without
 283  *                             interruption)
 284  * @n: endpoint number
 285  *
 286  * This function returns complete status
 287  */
 288 static int hw_test_and_clear_complete(struct ci13xxx *ci, int n)
 289 {
 290         n = ep_to_bit(ci, n);
 291         return hw_test_and_clear(ci, OP_ENDPTCOMPLETE, BIT(n));
 292 }
 293
 294 /**
 295  * hw_test_and_clear_intr_active: test & clear active interrupts (execute
 296  *                                without interruption)
 297  *
 298  * This function returns active interrutps
 299  */
 300 static u32 hw_test_and_clear_intr_active(struct ci13xxx *ci)
 301 {
 302         u32 reg = hw_read_intr_status(ci) & hw_read_intr_enable(ci);
 303
 304         hw_write(ci, OP_USBSTS, ~0, reg);
 305         return reg;
 306 }
 307
 308 static void hw_enable_vbus_intr(struct ci13xxx *ci)
 309 {
 310         hw_write(ci, OP_OTGSC, OTGSC_AVVIS, OTGSC_AVVIS);
 311         hw_write(ci, OP_OTGSC, OTGSC_AVVIE, OTGSC_AVVIE);
 312         queue_work(ci->wq, &ci->vbus_work);
 313 }
 314
 315 static void hw_disable_vbus_intr(struct ci13xxx *ci)
 316 {
 317         hw_write(ci, OP_OTGSC, OTGSC_AVVIE, 0);
 318 }
 319
 320 /**
 321  * hw_test_and_clear_setup_guard: test & clear setup guard (execute without
 322  *                                interruption)
 323  *
 324  * This function returns guard value
 325  */
 326 static int hw_test_and_clear_setup_guard(struct ci13xxx *ci)
 327 {
 328         return hw_test_and_write(ci, OP_USBCMD, USBCMD_SUTW, 0);
 329 }
 330
 331 /**
 332  * hw_test_and_set_setup_guard: test & set setup guard (execute without
 333  *                              interruption)
 334  *
 335  * This function returns guard value
 336  */
 337 static int hw_test_and_set_setup_guard(struct ci13xxx *ci)
 338 {
 339         return hw_test_and_write(ci, OP_USBCMD, USBCMD_SUTW, USBCMD_SUTW);
 340 }
 341
 342 /**
 343  * hw_usb_set_address: configures USB address (execute without interruption)
 344  * @value: new USB address
 345  *
 346  * This function explicitly sets the address, without the "USBADRA" (advance)
 347  * feature, which is not supported by older versions of the controller.
 348  */
 349 static void hw_usb_set_address(struct ci13xxx *ci, u8 value)
 350 {
 351         hw_write(ci, OP_DEVICEADDR, DEVICEADDR_USBADR,
 352                  value << ffs_nr(DEVICEADDR_USBADR));
 353 }
 354
 355 /**
 356  * hw_usb_reset: restart device after a bus reset (execute without
 357  *               interruption)
 358  *
 359  * This function returns an error code
 360  */
 361 static int hw_usb_reset(struct ci13xxx *ci)
 362 {
 363         hw_usb_set_address(ci, 0);
 364
 365         /* ESS flushes only at end?!? */
 366         hw_write(ci, OP_ENDPTFLUSH,    ~0, ~0);
 367
 368         /* clear setup token semaphores */
 369         hw_write(ci, OP_ENDPTSETUPSTAT, 0,  0);
 370
 371         /* clear complete status */
 372         hw_write(ci, OP_ENDPTCOMPLETE,  0,  0);
 373
 374         /* wait until all bits cleared */
 375         while (hw_read(ci, OP_ENDPTPRIME, ~0))
 376                 udelay(10);             /* not RTOS friendly */
 377
 378         /* reset all endpoints ? */
 379
 380         /* reset internal status and wait for further instructions
 381            no need to verify the port reset status (ESS does it) */
 382
 383         return 0;
 384 }
 385
 386 static void vbus_work(struct work_struct *work)
 387 {
 388         struct ci13xxx *ci = container_of(work, struct ci13xxx, vbus_work);
 389
 390         if (hw_read(ci, OP_OTGSC, OTGSC_AVV))
 391                 usb_gadget_vbus_connect(&ci->gadget);
 392         else
 393                 usb_gadget_vbus_disconnect(&ci->gadget);
 394 }
 395
 396 /******************************************************************************
 397  * UTIL block
 398  *****************************************************************************/
 399 /**
 400  * _usb_addr: calculates endpoint address from direction & number
 401  * @ep:  endpoint
 402  */
 403 static inline u8 _usb_addr(struct ci13xxx_ep *ep)
 404 {
 405         return ((ep->dir == TX) ? USB_ENDPOINT_DIR_MASK : 0) | ep->num;
 406 }
 407
 408 /**
 409  * _hardware_queue: configures a request at hardware level
 410  * @gadget: gadget
 411  * @mEp:    endpoint
 412  *
 413  * This function returns an error code
 414  */
 415 static int _hardware_enqueue(struct ci13xxx_ep *mEp, struct ci13xxx_req *mReq)
 416 {
 417         struct ci13xxx *ci = mEp->ci;
 418         unsigned i;
 419         int ret = 0;
 420         unsigned length = mReq->req.length;
 421
 422         /* don't queue twice */
 423         if (mReq->req.status == -EALREADY)
 424                 return -EALREADY;
 425
 426         mReq->req.status = -EALREADY;
 427
 428         if (mReq->req.zero && length && (length % mEp->ep.maxpacket == 0)) {
 429                 mReq->zptr = dma_pool_alloc(mEp->td_pool, GFP_ATOMIC,
 430                                            &mReq->zdma);
 431                 if (mReq->zptr == NULL)
 432                         return -ENOMEM;
 433
 434                 memset(mReq->zptr, 0, sizeof(*mReq->zptr));
 435                 mReq->zptr->next    = TD_TERMINATE;
 436                 mReq->zptr->token   = TD_STATUS_ACTIVE;
 437                 if (!mReq->req.no_interrupt)
 438                         mReq->zptr->token   |= TD_IOC;
 439         }
 440         ret = usb_gadget_map_request(&ci->gadget, &mReq->req, mEp->dir);
 441         if (ret)
 442                 return ret;
 443
 444         /*
 445          * TD configuration
 446          * TODO - handle requests which spawns into several TDs
 447          */
 448         memset(mReq->ptr, 0, sizeof(*mReq->ptr));
 449         mReq->ptr->token    = length << ffs_nr(TD_TOTAL_BYTES);
 450         mReq->ptr->token   &= TD_TOTAL_BYTES;
 451         mReq->ptr->token   |= TD_STATUS_ACTIVE;
 452         if (mReq->zptr) {
 453                 mReq->ptr->next    = mReq->zdma;
 454         } else {
 455                 mReq->ptr->next    = TD_TERMINATE;
 456                 if (!mReq->req.no_interrupt)
 457                         mReq->ptr->token  |= TD_IOC;
 458         }
 459         mReq->ptr->page[0]  = mReq->req.dma;
 460         for (i = 1; i < 5; i++)
 461                 mReq->ptr->page[i] =
 462                         (mReq->req.dma + i * CI13XXX_PAGE_SIZE) & ~TD_RESERVED_MASK;
 463
 464         if (!list_empty(&mEp->qh.queue)) {
 465                 struct ci13xxx_req *mReqPrev;
 466                 int n = hw_ep_bit(mEp->num, mEp->dir);
 467                 int tmp_stat;
 468
 469                 mReqPrev = list_entry(mEp->qh.queue.prev,
 470                                 struct ci13xxx_req, queue);
 471                 if (mReqPrev->zptr)
 472                         mReqPrev->zptr->next = mReq->dma & TD_ADDR_MASK;
 473                 else
 474                         mReqPrev->ptr->next = mReq->dma & TD_ADDR_MASK;
 475                 wmb();
 476                 if (hw_read(ci, OP_ENDPTPRIME, BIT(n)))
 477                         goto done;
 478                 do {
 479                         hw_write(ci, OP_USBCMD, USBCMD_ATDTW, USBCMD_ATDTW);
 480                         tmp_stat = hw_read(ci, OP_ENDPTSTAT, BIT(n));
 481                 } while (!hw_read(ci, OP_USBCMD, USBCMD_ATDTW));
 482                 hw_write(ci, OP_USBCMD, USBCMD_ATDTW, 0);
 483                 if (tmp_stat)
 484                         goto done;
 485         }
 486
 487         /*  QH configuration */
 488         mEp->qh.ptr->td.next   = mReq->dma;    /* TERMINATE = 0 */
 489         mEp->qh.ptr->td.token &= ~TD_STATUS;   /* clear status */
 490         mEp->qh.ptr->cap |=  QH_ZLT;
 491
 492         wmb();   /* synchronize before ep prime */
 493
 494         ret = hw_ep_prime(ci, mEp->num, mEp->dir,
 495                            mEp->type == USB_ENDPOINT_XFER_CONTROL);
 496 done:
 497         return ret;
 498 }
 499
 500 /**
 501  * _hardware_dequeue: handles a request at hardware level
 502  * @gadget: gadget
 503  * @mEp:    endpoint
 504  *
 505  * This function returns an error code
 506  */
 507 static int _hardware_dequeue(struct ci13xxx_ep *mEp, struct ci13xxx_req *mReq)
 508 {
 509         if (mReq->req.status != -EALREADY)
 510                 return -EINVAL;
 511
 512         if ((TD_STATUS_ACTIVE & mReq->ptr->token) != 0)
 513                 return -EBUSY;
 514
 515         if (mReq->zptr) {
 516                 if ((TD_STATUS_ACTIVE & mReq->zptr->token) != 0)
 517                         return -EBUSY;
 518                 dma_pool_free(mEp->td_pool, mReq->zptr, mReq->zdma);
 519                 mReq->zptr = NULL;
 520         }
 521
 522         mReq->req.status = 0;
 523
 524         usb_gadget_unmap_request(&mEp->ci->gadget, &mReq->req, mEp->dir);
 525
 526         mReq->req.status = mReq->ptr->token & TD_STATUS;
 527         if ((TD_STATUS_HALTED & mReq->req.status) != 0)
 528                 mReq->req.status = -1;
 529         else if ((TD_STATUS_DT_ERR & mReq->req.status) != 0)
 530                 mReq->req.status = -1;
 531         else if ((TD_STATUS_TR_ERR & mReq->req.status) != 0)
 532                 mReq->req.status = -1;
 533
 534         mReq->req.actual   = mReq->ptr->token & TD_TOTAL_BYTES;
 535         mReq->req.actual >>= ffs_nr(TD_TOTAL_BYTES);
 536         mReq->req.actual   = mReq->req.length - mReq->req.actual;
 537         mReq->req.actual   = mReq->req.status ? 0 : mReq->req.actual;
 538
 539         return mReq->req.actual;
 540 }
 541
 542 /**
 543  * _ep_nuke: dequeues all endpoint requests
 544  * @mEp: endpoint
 545  *
 546  * This function returns an error code
 547  * Caller must hold lock
 548  */
 549 static int _ep_nuke(struct ci13xxx_ep *mEp)
 550 __releases(mEp->lock)
 551 __acquires(mEp->lock)
 552 {
 553         if (mEp == NULL)
 554                 return -EINVAL;
 555
 556         hw_ep_flush(mEp->ci, mEp->num, mEp->dir);
 557
 558         while (!list_empty(&mEp->qh.queue)) {
 559
 560                 /* pop oldest request */
 561                 struct ci13xxx_req *mReq = \
 562                         list_entry(mEp->qh.queue.next,
 563                                    struct ci13xxx_req, queue);
 564                 list_del_init(&mReq->queue);
 565                 mReq->req.status = -ESHUTDOWN;
 566
 567                 if (mReq->req.complete != NULL) {
 568                         spin_unlock(mEp->lock);
 569                         mReq->req.complete(&mEp->ep, &mReq->req);
 570                         spin_lock(mEp->lock);
 571                 }
 572         }
 573         return 0;
 574 }
 575
 576 /**
 577  * _gadget_stop_activity: stops all USB activity, flushes & disables all endpts
 578  * @gadget: gadget
 579  *
 580  * This function returns an error code
 581  */
 582 static int _gadget_stop_activity(struct usb_gadget *gadget)
 583 {
 584         struct usb_ep *ep;
 585         struct ci13xxx    *ci = container_of(gadget, struct ci13xxx, gadget);
 586         unsigned long flags;
 587
 588         spin_lock_irqsave(&ci->lock, flags);
 589         ci->gadget.speed = USB_SPEED_UNKNOWN;
 590         ci->remote_wakeup = 0;
 591         ci->suspended = 0;
 592         spin_unlock_irqrestore(&ci->lock, flags);
 593
 594         /* flush all endpoints */
 595         gadget_for_each_ep(ep, gadget) {
 596                 usb_ep_fifo_flush(ep);
 597         }
 598         usb_ep_fifo_flush(&ci->ep0out->ep);
 599         usb_ep_fifo_flush(&ci->ep0in->ep);
 600
 601         if (ci->driver)
 602                 ci->driver->disconnect(gadget);
 603
 604         /* make sure to disable all endpoints */
 605         gadget_for_each_ep(ep, gadget) {
 606                 usb_ep_disable(ep);
 607         }
 608
 609         if (ci->status != NULL) {
 610                 usb_ep_free_request(&ci->ep0in->ep, ci->status);
 611                 ci->status = NULL;
 612         }
 613
 614         return 0;
 615 }
 616
 617 /******************************************************************************
 618  * ISR block
 619  *****************************************************************************/
 620 /**
 621  * isr_reset_handler: USB reset interrupt handler
 622  * @ci: UDC device
 623  *
 624  * This function resets USB engine after a bus reset occurred
 625  */
 626 static void isr_reset_handler(struct ci13xxx *ci)
 627 __releases(ci->lock)
 628 __acquires(ci->lock)
 629 {
 630         int retval;
 631
 632         dbg_event(0xFF, "BUS RST", 0);
 633
 634         spin_unlock(&ci->lock);
 635         retval = _gadget_stop_activity(&ci->gadget);
 636         if (retval)
 637                 goto done;
 638
 639         retval = hw_usb_reset(ci);
 640         if (retval)
 641                 goto done;
 642
 643         ci->status = usb_ep_alloc_request(&ci->ep0in->ep, GFP_ATOMIC);
 644         if (ci->status == NULL)
 645                 retval = -ENOMEM;
 646
 647 done:
 648         spin_lock(&ci->lock);
 649
 650         if (retval)
 651                 dev_err(ci->dev, "error: %i\n", retval);
 652 }
 653
 654 /**
 655  * isr_get_status_complete: get_status request complete function
 656  * @ep:  endpoint
 657  * @req: request handled
 658  *
 659  * Caller must release lock
 660  */
 661 static void isr_get_status_complete(struct usb_ep *ep, struct usb_request *req)
 662 {
 663         if (ep == NULL || req == NULL)
 664                 return;
 665
 666         kfree(req->buf);
 667         usb_ep_free_request(ep, req);
 668 }
 669
 670 /**
 671  * isr_get_status_response: get_status request response
 672  * @ci: ci struct
 673  * @setup: setup request packet
 674  *
 675  * This function returns an error code
 676  */
 677 static int isr_get_status_response(struct ci13xxx *ci,
 678                                    struct usb_ctrlrequest *setup)
 679 __releases(mEp->lock)
 680 __acquires(mEp->lock)
 681 {
 682         struct ci13xxx_ep *mEp = ci->ep0in;
 683         struct usb_request *req = NULL;
 684         gfp_t gfp_flags = GFP_ATOMIC;
 685         int dir, num, retval;
 686
 687         if (mEp == NULL || setup == NULL)
 688                 return -EINVAL;
 689
 690         spin_unlock(mEp->lock);
 691         req = usb_ep_alloc_request(&mEp->ep, gfp_flags);
 692         spin_lock(mEp->lock);
 693         if (req == NULL)
 694                 return -ENOMEM;
 695
 696         req->complete = isr_get_status_complete;
 697         req->length   = 2;
 698         req->buf      = kzalloc(req->length, gfp_flags);
 699         if (req->buf == NULL) {
 700                 retval = -ENOMEM;
 701                 goto err_free_req;
 702         }
 703
 704         if ((setup->bRequestType & USB_RECIP_MASK) == USB_RECIP_DEVICE) {
 705                 /* Assume that device is bus powered for now. */
 706                 *(u16 *)req->buf = ci->remote_wakeup << 1;
 707                 retval = 0;
 708         } else if ((setup->bRequestType & USB_RECIP_MASK) \
 709                    == USB_RECIP_ENDPOINT) {
 710                 dir = (le16_to_cpu(setup->wIndex) & USB_ENDPOINT_DIR_MASK) ?
 711                         TX : RX;
 712                 num =  le16_to_cpu(setup->wIndex) & USB_ENDPOINT_NUMBER_MASK;
 713                 *(u16 *)req->buf = hw_ep_get_halt(ci, num, dir);
 714         }
 715         /* else do nothing; reserved for future use */
 716
 717         spin_unlock(mEp->lock);
 718         retval = usb_ep_queue(&mEp->ep, req, gfp_flags);
 719         spin_lock(mEp->lock);
 720         if (retval)
 721                 goto err_free_buf;
 722
 723         return 0;
 724
 725  err_free_buf:
 726         kfree(req->buf);
 727  err_free_req:
 728         spin_unlock(mEp->lock);
 729         usb_ep_free_request(&mEp->ep, req);
 730         spin_lock(mEp->lock);
 731         return retval;
 732 }
 733
 734 /**
 735  * isr_setup_status_complete: setup_status request complete function
 736  * @ep:  endpoint
 737  * @req: request handled
 738  *
 739  * Caller must release lock. Put the port in test mode if test mode
 740  * feature is selected.
 741  */
 742 static void
 743 isr_setup_status_complete(struct usb_ep *ep, struct usb_request *req)
 744 {
 745         struct ci13xxx *ci = req->context;
 746         unsigned long flags;
 747
 748         if (ci->setaddr) {
 749                 hw_usb_set_address(ci, ci->address);
 750                 ci->setaddr = false;
 751         }
 752
 753         spin_lock_irqsave(&ci->lock, flags);
 754         if (ci->test_mode)
 755                 hw_port_test_set(ci, ci->test_mode);
 756         spin_unlock_irqrestore(&ci->lock, flags);
 757 }
 758
 759 /**
 760  * isr_setup_status_phase: queues the status phase of a setup transation
 761  * @ci: ci struct
 762  *
 763  * This function returns an error code
 764  */
 765 static int isr_setup_status_phase(struct ci13xxx *ci)
 766 __releases(mEp->lock)
 767 __acquires(mEp->lock)
 768 {
 769         int retval;
 770         struct ci13xxx_ep *mEp;
 771
 772         mEp = (ci->ep0_dir == TX) ? ci->ep0out : ci->ep0in;
 773         ci->status->context = ci;
 774         ci->status->complete = isr_setup_status_complete;
 775
 776         spin_unlock(mEp->lock);
 777         retval = usb_ep_queue(&mEp->ep, ci->status, GFP_ATOMIC);
 778         spin_lock(mEp->lock);
 779
 780         return retval;
 781 }
 782
 783 /**
 784  * isr_tr_complete_low: transaction complete low level handler
 785  * @mEp: endpoint
 786  *
 787  * This function returns an error code
 788  * Caller must hold lock
 789  */
 790 static int isr_tr_complete_low(struct ci13xxx_ep *mEp)
 791 __releases(mEp->lock)
 792 __acquires(mEp->lock)
 793 {
 794         struct ci13xxx_req *mReq, *mReqTemp;
 795         struct ci13xxx_ep *mEpTemp = mEp;
 796         int retval = 0;
 797
 798         list_for_each_entry_safe(mReq, mReqTemp, &mEp->qh.queue,
 799                         queue) {
 800                 retval = _hardware_dequeue(mEp, mReq);
 801                 if (retval < 0)
 802                         break;
 803                 list_del_init(&mReq->queue);
 804                 dbg_done(_usb_addr(mEp), mReq->ptr->token, retval);
 805                 if (mReq->req.complete != NULL) {
 806                         spin_unlock(mEp->lock);
 807                         if ((mEp->type == USB_ENDPOINT_XFER_CONTROL) &&
 808                                         mReq->req.length)
 809                                 mEpTemp = mEp->ci->ep0in;
 810                         mReq->req.complete(&mEpTemp->ep, &mReq->req);
 811                         spin_lock(mEp->lock);
 812                 }
 813         }
 814
 815         if (retval == -EBUSY)
 816                 retval = 0;
 817         if (retval < 0)
 818                 dbg_event(_usb_addr(mEp), "DONE", retval);
 819
 820         return retval;
 821 }
 822
 823 /**
 824  * isr_tr_complete_handler: transaction complete interrupt handler
 825  * @ci: UDC descriptor
 826  *
 827  * This function handles traffic events
 828  */
 829 static void isr_tr_complete_handler(struct ci13xxx *ci)
 830 __releases(ci->lock)
 831 __acquires(ci->lock)
 832 {
 833         unsigned i;
 834         u8 tmode = 0;
 835
 836         for (i = 0; i < ci->hw_ep_max; i++) {
 837                 struct ci13xxx_ep *mEp  = &ci->ci13xxx_ep[i];
 838                 int type, num, dir, err = -EINVAL;
 839                 struct usb_ctrlrequest req;
 840
 841                 if (mEp->ep.desc == NULL)
 842                         continue;   /* not configured */
 843
 844                 if (hw_test_and_clear_complete(ci, i)) {
 845                         err = isr_tr_complete_low(mEp);
 846                         if (mEp->type == USB_ENDPOINT_XFER_CONTROL) {
 847                                 if (err > 0)   /* needs status phase */
 848                                         err = isr_setup_status_phase(ci);
 849                                 if (err < 0) {
 850                                         dbg_event(_usb_addr(mEp),
 851                                                   "ERROR", err);
 852                                         spin_unlock(&ci->lock);
 853                                         if (usb_ep_set_halt(&mEp->ep))
 854                                                 dev_err(ci->dev,
 855                                                         "error: ep_set_halt\n");
 856                                         spin_lock(&ci->lock);
 857                                 }
 858                         }
 859                 }
 860
 861                 if (mEp->type != USB_ENDPOINT_XFER_CONTROL ||
 862                     !hw_test_and_clear_setup_status(ci, i))
 863                         continue;
 864
 865                 if (i != 0) {
 866                         dev_warn(ci->dev, "ctrl traffic at endpoint %d\n", i);
 867                         continue;
 868                 }
 869
 870                 /*
 871                  * Flush data and handshake transactions of previous
 872                  * setup packet.
 873                  */
 874                 _ep_nuke(ci->ep0out);
 875                 _ep_nuke(ci->ep0in);
 876
 877                 /* read_setup_packet */
 878                 do {
 879                         hw_test_and_set_setup_guard(ci);
 880                         memcpy(&req, &mEp->qh.ptr->setup, sizeof(req));
 881                 } while (!hw_test_and_clear_setup_guard(ci));
 882
 883                 type = req.bRequestType;
 884
 885                 ci->ep0_dir = (type & USB_DIR_IN) ? TX : RX;
 886
 887                 dbg_setup(_usb_addr(mEp), &req);
 888
 889                 switch (req.bRequest) {
 890                 case USB_REQ_CLEAR_FEATURE:
 891                         if (type == (USB_DIR_OUT|USB_RECIP_ENDPOINT) &&
 892                                         le16_to_cpu(req.wValue) ==
 893                                         USB_ENDPOINT_HALT) {
 894                                 if (req.wLength != 0)
 895                                         break;
 896                                 num  = le16_to_cpu(req.wIndex);
 897                                 dir = num & USB_ENDPOINT_DIR_MASK;
 898                                 num &= USB_ENDPOINT_NUMBER_MASK;
 899                                 if (dir) /* TX */
 900                                         num += ci->hw_ep_max/2;
 901                                 if (!ci->ci13xxx_ep[num].wedge) {
 902                                         spin_unlock(&ci->lock);
 903                                         err = usb_ep_clear_halt(
 904                                                 &ci->ci13xxx_ep[num].ep);
 905                                         spin_lock(&ci->lock);
 906                                         if (err)
 907                                                 break;
 908                                 }
 909                                 err = isr_setup_status_phase(ci);
 910                         } else if (type == (USB_DIR_OUT|USB_RECIP_DEVICE) &&
 911                                         le16_to_cpu(req.wValue) ==
 912                                         USB_DEVICE_REMOTE_WAKEUP) {
 913                                 if (req.wLength != 0)
 914                                         break;
 915                                 ci->remote_wakeup = 0;
 916                                 err = isr_setup_status_phase(ci);
 917                         } else {
 918                                 goto delegate;
 919                         }
 920                         break;
 921                 case USB_REQ_GET_STATUS:
 922                         if (type != (USB_DIR_IN|USB_RECIP_DEVICE)   &&
 923                             type != (USB_DIR_IN|USB_RECIP_ENDPOINT) &&
 924                             type != (USB_DIR_IN|USB_RECIP_INTERFACE))
 925                                 goto delegate;
 926                         if (le16_to_cpu(req.wLength) != 2 ||
 927                             le16_to_cpu(req.wValue)  != 0)
 928                                 break;
 929                         err = isr_get_status_response(ci, &req);
 930                         break;
 931                 case USB_REQ_SET_ADDRESS:
 932                         if (type != (USB_DIR_OUT|USB_RECIP_DEVICE))
 933                                 goto delegate;
 934                         if (le16_to_cpu(req.wLength) != 0 ||
 935                             le16_to_cpu(req.wIndex)  != 0)
 936                                 break;
 937                         ci->address = (u8)le16_to_cpu(req.wValue);
 938                         ci->setaddr = true;
 939                         err = isr_setup_status_phase(ci);
 940                         break;
 941                 case USB_REQ_SET_FEATURE:
 942                         if (type == (USB_DIR_OUT|USB_RECIP_ENDPOINT) &&
 943                                         le16_to_cpu(req.wValue) ==
 944                                         USB_ENDPOINT_HALT) {
 945                                 if (req.wLength != 0)
 946                                         break;
 947                                 num  = le16_to_cpu(req.wIndex);
 948                                 dir = num & USB_ENDPOINT_DIR_MASK;
 949                                 num &= USB_ENDPOINT_NUMBER_MASK;
 950                                 if (dir) /* TX */
 951                                         num += ci->hw_ep_max/2;
 952
 953                                 spin_unlock(&ci->lock);
 954                                 err = usb_ep_set_halt(&ci->ci13xxx_ep[num].ep);
 955                                 spin_lock(&ci->lock);
 956                                 if (!err)
 957                                         isr_setup_status_phase(ci);
 958                         } else if (type == (USB_DIR_OUT|USB_RECIP_DEVICE)) {
 959                                 if (req.wLength != 0)
 960                                         break;
 961                                 switch (le16_to_cpu(req.wValue)) {
 962                                 case USB_DEVICE_REMOTE_WAKEUP:
 963                                         ci->remote_wakeup = 1;
 964                                         err = isr_setup_status_phase(ci);
 965                                         break;
 966                                 case USB_DEVICE_TEST_MODE:
 967                                         tmode = le16_to_cpu(req.wIndex) >> 8;
 968                                         switch (tmode) {
 969                                         case TEST_J:
 970                                         case TEST_K:
 971                                         case TEST_SE0_NAK:
 972                                         case TEST_PACKET:
 973                                         case TEST_FORCE_EN:
 974                                                 ci->test_mode = tmode;
 975                                                 err = isr_setup_status_phase(
 976                                                                 ci);
 977                                                 break;
 978                                         default:
 979                                                 break;
 980                                         }
 981                                 default:
 982                                         goto delegate;
 983                                 }
 984                         } else {
 985                                 goto delegate;
 986                         }
 987                         break;
 988                 default:
 989 delegate:
 990                         if (req.wLength == 0)   /* no data phase */
 991                                 ci->ep0_dir = TX;
 992
 993                         spin_unlock(&ci->lock);
 994                         err = ci->driver->setup(&ci->gadget, &req);
 995                         spin_lock(&ci->lock);
 996                         break;
 997                 }
 998
 999                 if (err < 0) {
1000                         dbg_event(_usb_addr(mEp), "ERROR", err);
1001
1002                         spin_unlock(&ci->lock);
1003                         if (usb_ep_set_halt(&mEp->ep))
1004                                 dev_err(ci->dev, "error: ep_set_halt\n");
1005                         spin_lock(&ci->lock);
1006                 }
1007         }
1008 }
1009
1010 /******************************************************************************
1011  * ENDPT block
1012  *****************************************************************************/
1013 /**
1014  * ep_enable: configure endpoint, making it usable
1015  *
1016  * Check usb_ep_enable() at "usb_gadget.h" for details
1017  */
1018 static int ep_enable(struct usb_ep *ep,
1019                      const struct usb_endpoint_descriptor *desc)
1020 {
1021         struct ci13xxx_ep *mEp = container_of(ep, struct ci13xxx_ep, ep);
1022         int retval = 0;
1023         unsigned long flags;
1024
1025         if (ep == NULL || desc == NULL)
1026                 return -EINVAL;
1027
1028         spin_lock_irqsave(mEp->lock, flags);
1029
1030         /* only internal SW should enable ctrl endpts */
1031
1032         mEp->ep.desc = desc;
1033
1034         if (!list_empty(&mEp->qh.queue))
1035                 dev_warn(mEp->ci->dev, "enabling a non-empty endpoint!\n");
1036
1037         mEp->dir  = usb_endpoint_dir_in(desc) ? TX : RX;
1038         mEp->num  = usb_endpoint_num(desc);
1039         mEp->type = usb_endpoint_type(desc);
1040
1041         mEp->ep.maxpacket = usb_endpoint_maxp(desc);
1042
1043         dbg_event(_usb_addr(mEp), "ENABLE", 0);
1044
1045         mEp->qh.ptr->cap = 0;
1046
1047         if (mEp->type == USB_ENDPOINT_XFER_CONTROL)
1048                 mEp->qh.ptr->cap |=  QH_IOS;
1049         else if (mEp->type == USB_ENDPOINT_XFER_ISOC)
1050                 mEp->qh.ptr->cap &= ~QH_MULT;
1051         else
1052                 mEp->qh.ptr->cap &= ~QH_ZLT;
1053
1054         mEp->qh.ptr->cap |=
1055                 (mEp->ep.maxpacket << ffs_nr(QH_MAX_PKT)) & QH_MAX_PKT;
1056         mEp->qh.ptr->td.next |= TD_TERMINATE;   /* needed? */
1057
1058         /*
1059          * Enable endpoints in the HW other than ep0 as ep0
1060          * is always enabled
1061          */
1062         if (mEp->num)
1063                 retval |= hw_ep_enable(mEp->ci, mEp->num, mEp->dir, mEp->type);
1064
1065         spin_unlock_irqrestore(mEp->lock, flags);
1066         return retval;
1067 }
1068
1069 /**
1070  * ep_disable: endpoint is no longer usable
1071  *
1072  * Check usb_ep_disable() at "usb_gadget.h" for details
1073  */
1074 static int ep_disable(struct usb_ep *ep)
1075 {
1076         struct ci13xxx_ep *mEp = container_of(ep, struct ci13xxx_ep, ep);
1077         int direction, retval = 0;
1078         unsigned long flags;
1079
1080         if (ep == NULL)
1081                 return -EINVAL;
1082         else if (mEp->ep.desc == NULL)
1083                 return -EBUSY;
1084
1085         spin_lock_irqsave(mEp->lock, flags);
1086
1087         /* only internal SW should disable ctrl endpts */
1088
1089         direction = mEp->dir;
1090         do {
1091                 dbg_event(_usb_addr(mEp), "DISABLE", 0);
1092
1093                 retval |= _ep_nuke(mEp);
1094                 retval |= hw_ep_disable(mEp->ci, mEp->num, mEp->dir);
1095
1096                 if (mEp->type == USB_ENDPOINT_XFER_CONTROL)
1097                         mEp->dir = (mEp->dir == TX) ? RX : TX;
1098
1099         } while (mEp->dir != direction);
1100
1101         mEp->ep.desc = NULL;
1102
1103         spin_unlock_irqrestore(mEp->lock, flags);
1104         return retval;
1105 }
1106
1107 /**
1108  * ep_alloc_request: allocate a request object to use with this endpoint
1109  *
1110  * Check usb_ep_alloc_request() at "usb_gadget.h" for details
1111  */
1112 static struct usb_request *ep_alloc_request(struct usb_ep *ep, gfp_t gfp_flags)
1113 {
1114         struct ci13xxx_ep  *mEp  = container_of(ep, struct ci13xxx_ep, ep);
1115         struct ci13xxx_req *mReq = NULL;
1116
1117         if (ep == NULL)
1118                 return NULL;
1119
1120         mReq = kzalloc(sizeof(struct ci13xxx_req), gfp_flags);
1121         if (mReq != NULL) {
1122                 INIT_LIST_HEAD(&mReq->queue);
1123
1124                 mReq->ptr = dma_pool_alloc(mEp->td_pool, gfp_flags,
1125                                            &mReq->dma);
1126                 if (mReq->ptr == NULL) {
1127                         kfree(mReq);
1128                         mReq = NULL;
1129                 }
1130         }
1131
1132         dbg_event(_usb_addr(mEp), "ALLOC", mReq == NULL);
1133
1134         return (mReq == NULL) ? NULL : &mReq->req;
1135 }
1136
1137 /**
1138  * ep_free_request: frees a request object
1139  *
1140  * Check usb_ep_free_request() at "usb_gadget.h" for details
1141  */
1142 static void ep_free_request(struct usb_ep *ep, struct usb_request *req)
1143 {
1144         struct ci13xxx_ep  *mEp  = container_of(ep,  struct ci13xxx_ep, ep);
1145         struct ci13xxx_req *mReq = container_of(req, struct ci13xxx_req, req);
1146         unsigned long flags;
1147
1148         if (ep == NULL || req == NULL) {
1149                 return;
1150         } else if (!list_empty(&mReq->queue)) {
1151                 dev_err(mEp->ci->dev, "freeing queued request\n");
1152                 return;
1153         }
1154
1155         spin_lock_irqsave(mEp->lock, flags);
1156
1157         if (mReq->ptr)
1158                 dma_pool_free(mEp->td_pool, mReq->ptr, mReq->dma);
1159         kfree(mReq);
1160
1161         dbg_event(_usb_addr(mEp), "FREE", 0);
1162
1163         spin_unlock_irqrestore(mEp->lock, flags);
1164 }
1165
1166 /**
1167  * ep_queue: queues (submits) an I/O request to an endpoint
1168  *
1169  * Check usb_ep_queue()* at usb_gadget.h" for details
1170  */
1171 static int ep_queue(struct usb_ep *ep, struct usb_request *req,
1172                     gfp_t __maybe_unused gfp_flags)
1173 {
1174         struct ci13xxx_ep  *mEp  = container_of(ep,  struct ci13xxx_ep, ep);
1175         struct ci13xxx_req *mReq = container_of(req, struct ci13xxx_req, req);
1176         struct ci13xxx *ci = mEp->ci;
1177         int retval = 0;
1178         unsigned long flags;
1179
1180         if (ep == NULL || req == NULL || mEp->ep.desc == NULL)
1181                 return -EINVAL;
1182
1183         spin_lock_irqsave(mEp->lock, flags);
1184
1185         if (mEp->type == USB_ENDPOINT_XFER_CONTROL) {
1186                 if (req->length)
1187                         mEp = (ci->ep0_dir == RX) ?
1188                                ci->ep0out : ci->ep0in;
1189                 if (!list_empty(&mEp->qh.queue)) {
1190                         _ep_nuke(mEp);
1191                         retval = -EOVERFLOW;
1192                         dev_warn(mEp->ci->dev, "endpoint ctrl %X nuked\n",
1193                                  _usb_addr(mEp));
1194                 }
1195         }
1196
1197         /* first nuke then test link, e.g. previous status has not sent */
1198         if (!list_empty(&mReq->queue)) {
1199                 retval = -EBUSY;
1200                 dev_err(mEp->ci->dev, "request already in queue\n");
1201                 goto done;
1202         }
1203
1204         if (req->length > 4 * CI13XXX_PAGE_SIZE) {
1205                 req->length = 4 * CI13XXX_PAGE_SIZE;
1206                 retval = -EMSGSIZE;
1207                 dev_warn(mEp->ci->dev, "request length truncated\n");
1208         }
1209
1210         dbg_queue(_usb_addr(mEp), req, retval);
1211
1212         /* push request */
1213         mReq->req.status = -EINPROGRESS;
1214         mReq->req.actual = 0;
1215
1216         retval = _hardware_enqueue(mEp, mReq);
1217
1218         if (retval == -EALREADY) {
1219                 dbg_event(_usb_addr(mEp), "QUEUE", retval);
1220                 retval = 0;
1221         }
1222         if (!retval)
1223                 list_add_tail(&mReq->queue, &mEp->qh.queue);
1224
1225  done:
1226         spin_unlock_irqrestore(mEp->lock, flags);
1227         return retval;
1228 }
1229
1230 /**
1231  * ep_dequeue: dequeues (cancels, unlinks) an I/O request from an endpoint
1232  *
1233  * Check usb_ep_dequeue() at "usb_gadget.h" for details
1234  */
1235 static int ep_dequeue(struct usb_ep *ep, struct usb_request *req)
1236 {
1237         struct ci13xxx_ep  *mEp  = container_of(ep,  struct ci13xxx_ep, ep);
1238         struct ci13xxx_req *mReq = container_of(req, struct ci13xxx_req, req);
1239         unsigned long flags;
1240
1241         if (ep == NULL || req == NULL || mReq->req.status != -EALREADY ||
1242                 mEp->ep.desc == NULL || list_empty(&mReq->queue) ||
1243                 list_empty(&mEp->qh.queue))
1244                 return -EINVAL;
1245
1246         spin_lock_irqsave(mEp->lock, flags);
1247
1248         dbg_event(_usb_addr(mEp), "DEQUEUE", 0);
1249
1250         hw_ep_flush(mEp->ci, mEp->num, mEp->dir);
1251
1252         /* pop request */
1253         list_del_init(&mReq->queue);
1254
1255         usb_gadget_unmap_request(&mEp->ci->gadget, req, mEp->dir);
1256
1257         req->status = -ECONNRESET;
1258
1259         if (mReq->req.complete != NULL) {
1260                 spin_unlock(mEp->lock);
1261                 mReq->req.complete(&mEp->ep, &mReq->req);
1262                 spin_lock(mEp->lock);
1263         }
1264
1265         spin_unlock_irqrestore(mEp->lock, flags);
1266         return 0;
1267 }
1268
1269 /**
1270  * ep_set_halt: sets the endpoint halt feature
1271  *
1272  * Check usb_ep_set_halt() at "usb_gadget.h" for details
1273  */
1274 static int ep_set_halt(struct usb_ep *ep, int value)
1275 {
1276         struct ci13xxx_ep *mEp = container_of(ep, struct ci13xxx_ep, ep);
1277         int direction, retval = 0;
1278         unsigned long flags;
1279
1280         if (ep == NULL || mEp->ep.desc == NULL)
1281                 return -EINVAL;
1282
1283         spin_lock_irqsave(mEp->lock, flags);
1284
1285 #ifndef STALL_IN
1286         /* g_file_storage MS compliant but g_zero fails chapter 9 compliance */
1287         if (value && mEp->type == USB_ENDPOINT_XFER_BULK && mEp->dir == TX &&
1288             !list_empty(&mEp->qh.queue)) {
1289                 spin_unlock_irqrestore(mEp->lock, flags);
1290                 return -EAGAIN;
1291         }
1292 #endif
1293
1294         direction = mEp->dir;
1295         do {
1296                 dbg_event(_usb_addr(mEp), "HALT", value);
1297                 retval |= hw_ep_set_halt(mEp->ci, mEp->num, mEp->dir, value);
1298
1299                 if (!value)
1300                         mEp->wedge = 0;
1301
1302                 if (mEp->type == USB_ENDPOINT_XFER_CONTROL)
1303                         mEp->dir = (mEp->dir == TX) ? RX : TX;
1304
1305         } while (mEp->dir != direction);
1306
1307         spin_unlock_irqrestore(mEp->lock, flags);
1308         return retval;
1309 }
1310
1311 /**
1312  * ep_set_wedge: sets the halt feature and ignores clear requests
1313  *
1314  * Check usb_ep_set_wedge() at "usb_gadget.h" for details
1315  */
1316 static int ep_set_wedge(struct usb_ep *ep)
1317 {
1318         struct ci13xxx_ep *mEp = container_of(ep, struct ci13xxx_ep, ep);
1319         unsigned long flags;
1320
1321         if (ep == NULL || mEp->ep.desc == NULL)
1322                 return -EINVAL;
1323
1324         spin_lock_irqsave(mEp->lock, flags);
1325
1326         dbg_event(_usb_addr(mEp), "WEDGE", 0);
1327         mEp->wedge = 1;
1328
1329         spin_unlock_irqrestore(mEp->lock, flags);
1330
1331         return usb_ep_set_halt(ep);
1332 }
1333
1334 /**
1335  * ep_fifo_flush: flushes contents of a fifo
1336  *
1337  * Check usb_ep_fifo_flush() at "usb_gadget.h" for details
1338  */
1339 static void ep_fifo_flush(struct usb_ep *ep)
1340 {
1341         struct ci13xxx_ep *mEp = container_of(ep, struct ci13xxx_ep, ep);
1342         unsigned long flags;
1343
1344         if (ep == NULL) {
1345                 dev_err(mEp->ci->dev, "%02X: -EINVAL\n", _usb_addr(mEp));
1346                 return;
1347         }
1348
1349         spin_lock_irqsave(mEp->lock, flags);
1350
1351         dbg_event(_usb_addr(mEp), "FFLUSH", 0);
1352         hw_ep_flush(mEp->ci, mEp->num, mEp->dir);
1353
1354         spin_unlock_irqrestore(mEp->lock, flags);
1355 }
1356
1357 /**
1358  * Endpoint-specific part of the API to the USB controller hardware
1359  * Check "usb_gadget.h" for details
1360  */
1361 static const struct usb_ep_ops usb_ep_ops = {
1362         .enable        = ep_enable,
1363         .disable       = ep_disable,
1364         .alloc_request = ep_alloc_request,
1365         .free_request  = ep_free_request,
1366         .queue         = ep_queue,
1367         .dequeue       = ep_dequeue,
1368         .set_halt      = ep_set_halt,
1369         .set_wedge     = ep_set_wedge,
1370         .fifo_flush    = ep_fifo_flush,
1371 };
1372
1373 /******************************************************************************
1374  * GADGET block
1375  *****************************************************************************/
1376 static int ci13xxx_vbus_session(struct usb_gadget *_gadget, int is_active)
1377 {
1378         struct ci13xxx *ci = container_of(_gadget, struct ci13xxx, gadget);
1379         unsigned long flags;
1380         int gadget_ready = 0;
1381
1382         if (!(ci->platdata->flags & CI13XXX_PULLUP_ON_VBUS))
1383                 return -EOPNOTSUPP;
1384
1385         spin_lock_irqsave(&ci->lock, flags);
1386         ci->vbus_active = is_active;
1387         if (ci->driver)
1388                 gadget_ready = 1;
1389         spin_unlock_irqrestore(&ci->lock, flags);
1390
1391         if (gadget_ready) {
1392                 if (is_active) {
1393                         pm_runtime_get_sync(&_gadget->dev);
1394                         hw_device_reset(ci, USBMODE_CM_DC);
1395                         hw_enable_vbus_intr(ci);
1396                         hw_device_state(ci, ci->ep0out->qh.dma);
1397                 } else {
1398                         hw_device_state(ci, 0);
1399                         if (ci->platdata->notify_event)
1400                                 ci->platdata->notify_event(ci,
1401                                 CI13XXX_CONTROLLER_STOPPED_EVENT);
1402                         _gadget_stop_activity(&ci->gadget);
1403                         pm_runtime_put_sync(&_gadget->dev);
1404                 }
1405         }
1406
1407         return 0;
1408 }
1409
1410 static int ci13xxx_wakeup(struct usb_gadget *_gadget)
1411 {
1412         struct ci13xxx *ci = container_of(_gadget, struct ci13xxx, gadget);
1413         unsigned long flags;
1414         int ret = 0;
1415
1416         spin_lock_irqsave(&ci->lock, flags);
1417         if (!ci->remote_wakeup) {
1418                 ret = -EOPNOTSUPP;
1419                 goto out;
1420         }
1421         if (!hw_read(ci, OP_PORTSC, PORTSC_SUSP)) {
1422                 ret = -EINVAL;
1423                 goto out;
1424         }
1425         hw_write(ci, OP_PORTSC, PORTSC_FPR, PORTSC_FPR);
1426 out:
1427         spin_unlock_irqrestore(&ci->lock, flags);
1428         return ret;
1429 }
1430
1431 static int ci13xxx_vbus_draw(struct usb_gadget *_gadget, unsigned mA)
1432 {
1433         struct ci13xxx *ci = container_of(_gadget, struct ci13xxx, gadget);
1434
1435         if (ci->transceiver)
1436                 return usb_phy_set_power(ci->transceiver, mA);
1437         return -ENOTSUPP;
1438 }
1439
1440 /* Change Data+ pullup status
1441  * this func is used by usb_gadget_connect/disconnet
1442  */
1443 static int ci13xxx_pullup(struct usb_gadget *_gadget, int is_on)
1444 {
1445         struct ci13xxx *ci = container_of(_gadget, struct ci13xxx, gadget);
1446
1447         if (is_on)
1448                 hw_write(ci, OP_USBCMD, USBCMD_RS, USBCMD_RS);
1449         else
1450                 hw_write(ci, OP_USBCMD, USBCMD_RS, 0);
1451
1452         return 0;
1453 }
1454
1455 static int ci13xxx_start(struct usb_gadget *gadget,
1456                          struct usb_gadget_driver *driver);
1457 static int ci13xxx_stop(struct usb_gadget *gadget,
1458                         struct usb_gadget_driver *driver);
1459 /**
1460  * Device operations part of the API to the USB controller hardware,
1461  * which don't involve endpoints (or i/o)
1462  * Check  "usb_gadget.h" for details
1463  */
1464 static const struct usb_gadget_ops usb_gadget_ops = {
1465         .vbus_session   = ci13xxx_vbus_session,
1466         .wakeup         = ci13xxx_wakeup,
1467         .pullup         = ci13xxx_pullup,
1468         .vbus_draw      = ci13xxx_vbus_draw,
1469         .udc_start      = ci13xxx_start,
1470         .udc_stop       = ci13xxx_stop,
1471 };
1472
1473 static int init_eps(struct ci13xxx *ci)
1474 {
1475         int retval = 0, i, j;
1476
1477         for (i = 0; i < ci->hw_ep_max/2; i++)
1478                 for (j = RX; j <= TX; j++) {
1479                         int k = i + j * ci->hw_ep_max/2;
1480                         struct ci13xxx_ep *mEp = &ci->ci13xxx_ep[k];
1481
1482                         scnprintf(mEp->name, sizeof(mEp->name), "ep%i%s", i,
1483                                         (j == TX)  ? "in" : "out");
1484
1485                         mEp->ci          = ci;
1486                         mEp->lock         = &ci->lock;
1487                         mEp->td_pool      = ci->td_pool;
1488
1489                         mEp->ep.name      = mEp->name;
1490                         mEp->ep.ops       = &usb_ep_ops;
1491                         /*
1492                          * for ep0: maxP defined in desc, for other
1493                          * eps, maxP is set by epautoconfig() called
1494                          * by gadget layer
1495                          */
1496                         mEp->ep.maxpacket = (unsigned short)~0;
1497
1498                         INIT_LIST_HEAD(&mEp->qh.queue);
1499                         mEp->qh.ptr = dma_pool_alloc(ci->qh_pool, GFP_KERNEL,
1500                                                      &mEp->qh.dma);
1501                         if (mEp->qh.ptr == NULL)
1502                                 retval = -ENOMEM;
1503                         else
1504                                 memset(mEp->qh.ptr, 0, sizeof(*mEp->qh.ptr));
1505
1506                         /*
1507                          * set up shorthands for ep0 out and in endpoints,
1508                          * don't add to gadget's ep_list
1509                          */
1510                         if (i == 0) {
1511                                 if (j == RX)
1512                                         ci->ep0out = mEp;
1513                                 else
1514                                         ci->ep0in = mEp;
1515
1516                                 mEp->ep.maxpacket = CTRL_PAYLOAD_MAX;
1517                                 continue;
1518                         }
1519
1520                         list_add_tail(&mEp->ep.ep_list, &ci->gadget.ep_list);
1521                 }
1522
1523         return retval;
1524 }
1525
1526 static void destroy_eps(struct ci13xxx *ci)
1527 {
1528         int i;
1529
1530         for (i = 0; i < ci->hw_ep_max; i++) {
1531                 struct ci13xxx_ep *mEp = &ci->ci13xxx_ep[i];
1532
1533                 dma_pool_free(ci->qh_pool, mEp->qh.ptr, mEp->qh.dma);
1534         }
1535 }
1536
1537 /**
1538  * ci13xxx_start: register a gadget driver
1539  * @gadget: our gadget
1540  * @driver: the driver being registered
1541  *
1542  * Interrupts are enabled here.
1543  */
1544 static int ci13xxx_start(struct usb_gadget *gadget,
1545                          struct usb_gadget_driver *driver)
1546 {
1547         struct ci13xxx *ci = container_of(gadget, struct ci13xxx, gadget);
1548         unsigned long flags;
1549         int retval = -ENOMEM;
1550
1551         if (driver->disconnect == NULL)
1552                 return -EINVAL;
1553
1554
1555         ci->ep0out->ep.desc = &ctrl_endpt_out_desc;
1556         retval = usb_ep_enable(&ci->ep0out->ep);
1557         if (retval)
1558                 return retval;
1559
1560         ci->ep0in->ep.desc = &ctrl_endpt_in_desc;
1561         retval = usb_ep_enable(&ci->ep0in->ep);
1562         if (retval)
1563                 return retval;
1564         spin_lock_irqsave(&ci->lock, flags);
1565
1566         ci->driver = driver;
1567         pm_runtime_get_sync(&ci->gadget.dev);
1568         if (ci->platdata->flags & CI13XXX_PULLUP_ON_VBUS) {
1569                 if (ci->vbus_active) {
1570                         if (ci->platdata->flags & CI13XXX_REGS_SHARED) {
1571                                 hw_device_reset(ci, USBMODE_CM_DC);
1572                                 hw_enable_vbus_intr(ci);
1573                         }
1574                 } else {
1575                         pm_runtime_put_sync(&ci->gadget.dev);
1576                         goto done;
1577                 }
1578         }
1579
1580         retval = hw_device_state(ci, ci->ep0out->qh.dma);
1581         if (retval)
1582                 pm_runtime_put_sync(&ci->gadget.dev);
1583
1584  done:
1585         spin_unlock_irqrestore(&ci->lock, flags);
1586         return retval;
1587 }
1588
1589 /**
1590  * ci13xxx_stop: unregister a gadget driver
1591  */
1592 static int ci13xxx_stop(struct usb_gadget *gadget,
1593                         struct usb_gadget_driver *driver)
1594 {
1595         struct ci13xxx *ci = container_of(gadget, struct ci13xxx, gadget);
1596         unsigned long flags;
1597
1598         spin_lock_irqsave(&ci->lock, flags);
1599
1600         if (!(ci->platdata->flags & CI13XXX_PULLUP_ON_VBUS) ||
1601                         ci->vbus_active) {
1602                 hw_device_state(ci, 0);
1603                 if (ci->platdata->notify_event)
1604                         ci->platdata->notify_event(ci,
1605                         CI13XXX_CONTROLLER_STOPPED_EVENT);
1606                 ci->driver = NULL;
1607                 spin_unlock_irqrestore(&ci->lock, flags);
1608                 _gadget_stop_activity(&ci->gadget);
1609                 spin_lock_irqsave(&ci->lock, flags);
1610                 pm_runtime_put(&ci->gadget.dev);
1611         }
1612
1613         spin_unlock_irqrestore(&ci->lock, flags);
1614
1615         return 0;
1616 }
1617
1618 /******************************************************************************
1619  * BUS block
1620  *****************************************************************************/
1621 /**
1622  * udc_irq: ci interrupt handler
1623  *
1624  * This function returns IRQ_HANDLED if the IRQ has been handled
1625  * It locks access to registers
1626  */
1627 static irqreturn_t udc_irq(struct ci13xxx *ci)
1628 {
1629         irqreturn_t retval;
1630         u32 intr;
1631
1632         if (ci == NULL)
1633                 return IRQ_HANDLED;
1634
1635         spin_lock(&ci->lock);
1636
1637         if (ci->platdata->flags & CI13XXX_REGS_SHARED) {
1638                 if (hw_read(ci, OP_USBMODE, USBMODE_CM) !=
1639                                 USBMODE_CM_DC) {
1640                         spin_unlock(&ci->lock);
1641                         return IRQ_NONE;
1642                 }
1643         }
1644         intr = hw_test_and_clear_intr_active(ci);
1645         dbg_interrupt(intr);
1646
1647         if (intr) {
1648                 /* order defines priority - do NOT change it */
1649                 if (USBi_URI & intr)
1650                         isr_reset_handler(ci);
1651
1652                 if (USBi_PCI & intr) {
1653                         ci->gadget.speed = hw_port_is_high_speed(ci) ?
1654                                 USB_SPEED_HIGH : USB_SPEED_FULL;
1655                         if (ci->suspended && ci->driver->resume) {
1656                                 spin_unlock(&ci->lock);
1657                                 ci->driver->resume(&ci->gadget);
1658                                 spin_lock(&ci->lock);
1659                                 ci->suspended = 0;
1660                         }
1661                 }
1662
1663                 if (USBi_UI  & intr)
1664                         isr_tr_complete_handler(ci);
1665
1666                 if (USBi_SLI & intr) {
1667                         if (ci->gadget.speed != USB_SPEED_UNKNOWN &&
1668                             ci->driver->suspend) {
1669                                 ci->suspended = 1;
1670                                 spin_unlock(&ci->lock);
1671                                 ci->driver->suspend(&ci->gadget);
1672                                 spin_lock(&ci->lock);
1673                         }
1674                 }
1675                 retval = IRQ_HANDLED;
1676         } else {
1677                 retval = IRQ_NONE;
1678         }
1679
1680         intr = hw_read(ci, OP_OTGSC, ~0);
1681         hw_write(ci, OP_OTGSC, ~0, intr);
1682
1683         if (intr & (OTGSC_AVVIE & OTGSC_AVVIS))
1684                 queue_work(ci->wq, &ci->vbus_work);
1685
1686         spin_unlock(&ci->lock);
1687
1688         return retval;
1689 }
1690
1691 /**
1692  * udc_release: driver release function
1693  * @dev: device
1694  *
1695  * Currently does nothing
1696  */
1697 static void udc_release(struct device *dev)
1698 {
1699 }
1700
1701 /**
1702  * udc_start: initialize gadget role
1703  * @ci: chipidea controller
1704  */
1705 static int udc_start(struct ci13xxx *ci)
1706 {
1707         struct device *dev = ci->dev;
1708         int retval = 0;
1709
1710         spin_lock_init(&ci->lock);
1711
1712         ci->gadget.ops          = &usb_gadget_ops;
1713         ci->gadget.speed        = USB_SPEED_UNKNOWN;
1714         ci->gadget.max_speed    = USB_SPEED_HIGH;
1715         ci->gadget.is_otg       = 0;
1716         ci->gadget.name         = ci->platdata->name;
1717
1718         INIT_LIST_HEAD(&ci->gadget.ep_list);
1719
1720         dev_set_name(&ci->gadget.dev, "gadget");
1721         ci->gadget.dev.dma_mask = dev->dma_mask;
1722         ci->gadget.dev.coherent_dma_mask = dev->coherent_dma_mask;
1723         ci->gadget.dev.parent   = dev;
1724         ci->gadget.dev.release  = udc_release;
1725
1726         /* alloc resources */
1727         ci->qh_pool = dma_pool_create("ci13xxx_qh", dev,
1728                                        sizeof(struct ci13xxx_qh),
1729                                        64, CI13XXX_PAGE_SIZE);
1730         if (ci->qh_pool == NULL)
1731                 return -ENOMEM;
1732
1733         ci->td_pool = dma_pool_create("ci13xxx_td", dev,
1734                                        sizeof(struct ci13xxx_td),
1735                                        64, CI13XXX_PAGE_SIZE);
1736         if (ci->td_pool == NULL) {
1737                 retval = -ENOMEM;
1738                 goto free_qh_pool;
1739         }
1740
1741         retval = init_eps(ci);
1742         if (retval)
1743                 goto free_pools;
1744
1745         ci->gadget.ep0 = &ci->ep0in->ep;
1746
1747         if (ci->global_phy)
1748                 ci->transceiver = usb_get_phy(USB_PHY_TYPE_USB2);
1749
1750         if (ci->platdata->flags & CI13XXX_REQUIRE_TRANSCEIVER) {
1751                 if (ci->transceiver == NULL) {
1752                         retval = -ENODEV;
1753                         goto destroy_eps;
1754                 }
1755         }
1756
1757         if (!(ci->platdata->flags & CI13XXX_REGS_SHARED)) {
1758                 retval = hw_device_reset(ci, USBMODE_CM_DC);
1759                 if (retval)
1760                         goto put_transceiver;
1761                 hw_enable_vbus_intr(ci);
1762         }
1763
1764         retval = device_register(&ci->gadget.dev);
1765         if (retval) {
1766                 put_device(&ci->gadget.dev);
1767                 goto put_transceiver;
1768         }
1769
1770         retval = dbg_create_files(ci->dev);
1771         if (retval)
1772                 goto unreg_device;
1773
1774         if (!IS_ERR_OR_NULL(ci->transceiver)) {
1775                 retval = otg_set_peripheral(ci->transceiver->otg,
1776                                                 &ci->gadget);
1777                 if (retval)
1778                         goto remove_dbg;
1779         }
1780
1781         retval = usb_add_gadget_udc(dev, &ci->gadget);
1782         if (retval)
1783                 goto remove_trans;
1784
1785         pm_runtime_no_callbacks(&ci->gadget.dev);
1786         pm_runtime_enable(&ci->gadget.dev);
1787
1788         return retval;
1789
1790 remove_trans:
1791         if (!IS_ERR_OR_NULL(ci->transceiver)) {
1792                 otg_set_peripheral(ci->transceiver->otg, NULL);
1793                 if (ci->global_phy)
1794                         usb_put_phy(ci->transceiver);
1795         }
1796
1797         dev_err(dev, "error = %i\n", retval);
1798 remove_dbg:
1799         dbg_remove_files(ci->dev);
1800 unreg_device:
1801         device_unregister(&ci->gadget.dev);
1802 put_transceiver:
1803         if (!IS_ERR_OR_NULL(ci->transceiver) && ci->global_phy)
1804                 usb_put_phy(ci->transceiver);
1805 destroy_eps:
1806         destroy_eps(ci);
1807 free_pools:
1808         dma_pool_destroy(ci->td_pool);
1809 free_qh_pool:
1810         dma_pool_destroy(ci->qh_pool);
1811         return retval;
1812 }
1813
1814 /**
1815  * udc_remove: parent remove must call this to remove UDC
1816  *
1817  * No interrupts active, the IRQ has been released
1818  */
1819 static void udc_stop(struct ci13xxx *ci)
1820 {
1821         if (ci == NULL)
1822                 return;
1823
1824         hw_disable_vbus_intr(ci);
1825         cancel_work_sync(&ci->vbus_work);
1826
1827         usb_del_gadget_udc(&ci->gadget);
1828
1829         destroy_eps(ci);
1830
1831         dma_pool_destroy(ci->td_pool);
1832         dma_pool_destroy(ci->qh_pool);
1833
1834         if (!IS_ERR_OR_NULL(ci->transceiver)) {
1835                 otg_set_peripheral(ci->transceiver->otg, NULL);
1836                 if (ci->global_phy)
1837                         usb_put_phy(ci->transceiver);
1838         }
1839         dbg_remove_files(ci->dev);
1840         device_unregister(&ci->gadget.dev);
1841         /* my kobject is dynamic, I swear! */
1842         memset(&ci->gadget, 0, sizeof(ci->gadget));
1843 }
1844
1845 /**
1846  * ci_hdrc_gadget_init - initialize device related bits
1847  * ci: the controller
1848  *
1849  * This function enables the gadget role, if the device is "device capable".
1850  */
1851 int ci_hdrc_gadget_init(struct ci13xxx *ci)
1852 {
1853         struct ci_role_driver *rdrv;
1854
1855         if (!hw_read(ci, CAP_DCCPARAMS, DCCPARAMS_DC))
1856                 return -ENXIO;
1857
1858         rdrv = devm_kzalloc(ci->dev, sizeof(struct ci_role_driver), GFP_KERNEL);
1859         if (!rdrv)
1860                 return -ENOMEM;
1861
1862         rdrv->start     = udc_start;
1863         rdrv->stop      = udc_stop;
1864         rdrv->irq       = udc_irq;
1865         rdrv->name      = "gadget";
1866         ci->roles[CI_ROLE_GADGET] = rdrv;
1867         INIT_WORK(&ci->vbus_work, vbus_work);
1868
1869         return 0;
1870 }