drivers/usb/dwc3/gadget.c

   1 /**
   2  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   3  *
   4  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   5  *
   6  * Authors: Felipe Balbi <balbi@ti.com>,
   7  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   8  *
   9  * This program is free software: you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2  of
  11  * the License as published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  */
  18
  19 #include <linux/kernel.h>
  20 #include <linux/delay.h>
  21 #include <linux/slab.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/pm_runtime.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/io.h>
  27 #include <linux/list.h>
  28 #include <linux/dma-mapping.h>
  29
  30 #include <linux/usb/ch9.h>
  31 #include <linux/usb/gadget.h>
  32
  33 #include "debug.h"
  34 #include "core.h"
  35 #include "gadget.h"
  36 #include "io.h"
  37
  38 /**
  39  * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
  40  * @dwc: pointer to our context structure
  41  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  42  *
  43  * Caller should take care of locking. This function will
  44  * return 0 on success or -EINVAL if wrong Test Selector
  45  * is passed
  46  */
  47 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  48 {
  49         u32             reg;
  50
  51         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  52         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  53
  54         switch (mode) {
  55         case TEST_J:
  56         case TEST_K:
  57         case TEST_SE0_NAK:
  58         case TEST_PACKET:
  59         case TEST_FORCE_EN:
  60                 reg |= mode << 1;
  61                 break;
  62         default:
  63                 return -EINVAL;
  64         }
  65
  66         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  67
  68         return 0;
  69 }
  70
  71 /**
  72  * dwc3_gadget_get_link_state - Gets current state of USB Link
  73  * @dwc: pointer to our context structure
  74  *
  75  * Caller should take care of locking. This function will
  76  * return the link state on success (>= 0) or -ETIMEDOUT.
  77  */
  78 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  79 {
  80         u32             reg;
  81
  82         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  83
  84         return DWC3_DSTS_USBLNKST(reg);
  85 }
  86
  87 /**
  88  * dwc3_gadget_set_link_state - Sets USB Link to a particular State
  89  * @dwc: pointer to our context structure
  90  * @state: the state to put link into
  91  *
  92  * Caller should take care of locking. This function will
  93  * return 0 on success or -ETIMEDOUT.
  94  */
  95 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  96 {
  97         int             retries = 10000;
  98         u32             reg;
  99
 100         /*
 101          * Wait until device controller is ready. Only applies to 1.94a and
 102          * later RTL.
 103          */
 104         if (dwc->revision >= DWC3_REVISION_194A) {
 105                 while (--retries) {
 106                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 107                         if (reg & DWC3_DSTS_DCNRD)
 108                                 udelay(5);
 109                         else
 110                                 break;
 111                 }
 112
 113                 if (retries <= 0)
 114                         return -ETIMEDOUT;
 115         }
 116
 117         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 118         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 119
 120         /* set requested state */
 121         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 122         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 123
 124         /*
 125          * The following code is racy when called from dwc3_gadget_wakeup,
 126          * and is not needed, at least on newer versions
 127          */
 128         if (dwc->revision >= DWC3_REVISION_194A)
 129                 return 0;
 130
 131         /* wait for a change in DSTS */
 132         retries = 10000;
 133         while (--retries) {
 134                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 135
 136                 if (DWC3_DSTS_USBLNKST(reg) == state)
 137                         return 0;
 138
 139                 udelay(5);
 140         }
 141
 142         dwc3_trace(trace_dwc3_gadget,
 143                         "link state change request timed out");
 144
 145         return -ETIMEDOUT;
 146 }
 147
 148 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 149                 int status)
 150 {
 151         struct dwc3                     *dwc = dep->dwc;
 152         int                             i;
 153
 154         if (req->queued) {
 155                 i = 0;
 156                 do {
 157                         dep->busy_slot++;
 158                         /*
 159                          * Skip LINK TRB. We can't use req->trb and check for
 160                          * DWC3_TRBCTL_LINK_TRB because it points the TRB we
 161                          * just completed (not the LINK TRB).
 162                          */
 163                         if (((dep->busy_slot & DWC3_TRB_MASK) ==
 164                                 DWC3_TRB_NUM- 1) &&
 165                                 usb_endpoint_xfer_isoc(dep->endpoint.desc))
 166                                 dep->busy_slot++;
 167                 } while(++i < req->request.num_mapped_sgs);
 168                 req->queued = false;
 169         }
 170         list_del(&req->list);
 171         req->trb = NULL;
 172
 173         if (req->request.status == -EINPROGRESS)
 174                 req->request.status = status;
 175
 176         if (dwc->ep0_bounced && dep->number == 0)
 177                 dwc->ep0_bounced = false;
 178         else
 179                 usb_gadget_unmap_request(&dwc->gadget, &req->request,
 180                                 req->direction);
 181
 182         trace_dwc3_gadget_giveback(req);
 183
 184         spin_unlock(&dwc->lock);
 185         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 186         spin_lock(&dwc->lock);
 187 }
 188
 189 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 190 {
 191         u32             timeout = 500;
 192         u32             reg;
 193
 194         trace_dwc3_gadget_generic_cmd(cmd, param);
 195
 196         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 197         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 198
 199         do {
 200                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 201                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 202                         dwc3_trace(trace_dwc3_gadget,
 203                                         "Command Complete --> %d",
 204                                         DWC3_DGCMD_STATUS(reg));
 205                         if (DWC3_DGCMD_STATUS(reg))
 206                                 return -EINVAL;
 207                         return 0;
 208                 }
 209
 210                 /*
 211                  * We can't sleep here, because it's also called from
 212                  * interrupt context.
 213                  */
 214                 timeout--;
 215                 if (!timeout) {
 216                         dwc3_trace(trace_dwc3_gadget,
 217                                         "Command Timed Out");
 218                         return -ETIMEDOUT;
 219                 }
 220                 udelay(1);
 221         } while (1);
 222 }
 223
 224 int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
 225                 unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
 226 {
 227         struct dwc3_ep          *dep = dwc->eps[ep];
 228         u32                     timeout = 500;
 229         u32                     reg;
 230
 231         trace_dwc3_gadget_ep_cmd(dep, cmd, params);
 232
 233         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0);
 234         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1);
 235         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2);
 236
 237         dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
 238         do {
 239                 reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
 240                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 241                         dwc3_trace(trace_dwc3_gadget,
 242                                         "Command Complete --> %d",
 243                                         DWC3_DEPCMD_STATUS(reg));
 244                         if (DWC3_DEPCMD_STATUS(reg))
 245                                 return -EINVAL;
 246                         return 0;
 247                 }
 248
 249                 /*
 250                  * We can't sleep here, because it is also called from
 251                  * interrupt context.
 252                  */
 253                 timeout--;
 254                 if (!timeout) {
 255                         dwc3_trace(trace_dwc3_gadget,
 256                                         "Command Timed Out");
 257                         return -ETIMEDOUT;
 258                 }
 259
 260                 udelay(1);
 261         } while (1);
 262 }
 263
 264 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 265                 struct dwc3_trb *trb)
 266 {
 267         u32             offset = (char *) trb - (char *) dep->trb_pool;
 268
 269         return dep->trb_pool_dma + offset;
 270 }
 271
 272 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 273 {
 274         struct dwc3             *dwc = dep->dwc;
 275
 276         if (dep->trb_pool)
 277                 return 0;
 278
 279         dep->trb_pool = dma_alloc_coherent(dwc->dev,
 280                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 281                         &dep->trb_pool_dma, GFP_KERNEL);
 282         if (!dep->trb_pool) {
 283                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 284                                 dep->name);
 285                 return -ENOMEM;
 286         }
 287
 288         return 0;
 289 }
 290
 291 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 292 {
 293         struct dwc3             *dwc = dep->dwc;
 294
 295         dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 296                         dep->trb_pool, dep->trb_pool_dma);
 297
 298         dep->trb_pool = NULL;
 299         dep->trb_pool_dma = 0;
 300 }
 301
 302 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
 303
 304 /**
 305  * dwc3_gadget_start_config - Configure EP resources
 306  * @dwc: pointer to our controller context structure
 307  * @dep: endpoint that is being enabled
 308  *
 309  * The assignment of transfer resources cannot perfectly follow the
 310  * data book due to the fact that the controller driver does not have
 311  * all knowledge of the configuration in advance. It is given this
 312  * information piecemeal by the composite gadget framework after every
 313  * SET_CONFIGURATION and SET_INTERFACE. Trying to follow the databook
 314  * programming model in this scenario can cause errors. For two
 315  * reasons:
 316  *
 317  * 1) The databook says to do DEPSTARTCFG for every SET_CONFIGURATION
 318  * and SET_INTERFACE (8.1.5). This is incorrect in the scenario of
 319  * multiple interfaces.
 320  *
 321  * 2) The databook does not mention doing more DEPXFERCFG for new
 322  * endpoint on alt setting (8.1.6).
 323  *
 324  * The following simplified method is used instead:
 325  *
 326  * All hardware endpoints can be assigned a transfer resource and this
 327  * setting will stay persistent until either a core reset or
 328  * hibernation. So whenever we do a DEPSTARTCFG(0) we can go ahead and
 329  * do DEPXFERCFG for every hardware endpoint as well. We are
 330  * guaranteed that there are as many transfer resources as endpoints.
 331  *
 332  * This function is called for each endpoint when it is being enabled
 333  * but is triggered only when called for EP0-out, which always happens
 334  * first, and which should only happen in one of the above conditions.
 335  */
 336 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 337 {
 338         struct dwc3_gadget_ep_cmd_params params;
 339         u32                     cmd;
 340         int                     i;
 341         int                     ret;
 342
 343         if (dep->number)
 344                 return 0;
 345
 346         memset(&params, 0x00, sizeof(params));
 347         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 348
 349         ret = dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
 350         if (ret)
 351                 return ret;
 352
 353         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 354                 struct dwc3_ep *dep = dwc->eps[i];
 355
 356                 if (!dep)
 357                         continue;
 358
 359                 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 360                 if (ret)
 361                         return ret;
 362         }
 363
 364         return 0;
 365 }
 366
 367 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 368                 const struct usb_endpoint_descriptor *desc,
 369                 const struct usb_ss_ep_comp_descriptor *comp_desc,
 370                 bool ignore, bool restore)
 371 {
 372         struct dwc3_gadget_ep_cmd_params params;
 373
 374         memset(&params, 0x00, sizeof(params));
 375
 376         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 377                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 378
 379         /* Burst size is only needed in SuperSpeed mode */
 380         if (dwc->gadget.speed == USB_SPEED_SUPER) {
 381                 u32 burst = dep->endpoint.maxburst - 1;
 382
 383                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst);
 384         }
 385
 386         if (ignore)
 387                 params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM;
 388
 389         if (restore) {
 390                 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
 391                 params.param2 |= dep->saved_state;
 392         }
 393
 394         params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
 395                 | DWC3_DEPCFG_XFER_NOT_READY_EN;
 396
 397         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 398                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 399                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 400                 dep->stream_capable = true;
 401         }
 402
 403         if (!usb_endpoint_xfer_control(desc))
 404                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 405
 406         /*
 407          * We are doing 1:1 mapping for endpoints, meaning
 408          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 409          * so on. We consider the direction bit as part of the physical
 410          * endpoint number. So USB endpoint 0x81 is 0x03.
 411          */
 412         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 413
 414         /*
 415          * We must use the lower 16 TX FIFOs even though
 416          * HW might have more
 417          */
 418         if (dep->direction)
 419                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 420
 421         if (desc->bInterval) {
 422                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 423                 dep->interval = 1 << (desc->bInterval - 1);
 424         }
 425
 426         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 427                         DWC3_DEPCMD_SETEPCONFIG, &params);
 428 }
 429
 430 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
 431 {
 432         struct dwc3_gadget_ep_cmd_params params;
 433
 434         memset(&params, 0x00, sizeof(params));
 435
 436         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 437
 438         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 439                         DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
 440 }
 441
 442 /**
 443  * __dwc3_gadget_ep_enable - Initializes a HW endpoint
 444  * @dep: endpoint to be initialized
 445  * @desc: USB Endpoint Descriptor
 446  *
 447  * Caller should take care of locking
 448  */
 449 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 450                 const struct usb_endpoint_descriptor *desc,
 451                 const struct usb_ss_ep_comp_descriptor *comp_desc,
 452                 bool ignore, bool restore)
 453 {
 454         struct dwc3             *dwc = dep->dwc;
 455         u32                     reg;
 456         int                     ret;
 457
 458         dwc3_trace(trace_dwc3_gadget, "Enabling %s", dep->name);
 459
 460         if (!(dep->flags & DWC3_EP_ENABLED)) {
 461                 ret = dwc3_gadget_start_config(dwc, dep);
 462                 if (ret)
 463                         return ret;
 464         }
 465
 466         ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore,
 467                         restore);
 468         if (ret)
 469                 return ret;
 470
 471         if (!(dep->flags & DWC3_EP_ENABLED)) {
 472                 struct dwc3_trb *trb_st_hw;
 473                 struct dwc3_trb *trb_link;
 474
 475                 dep->endpoint.desc = desc;
 476                 dep->comp_desc = comp_desc;
 477                 dep->type = usb_endpoint_type(desc);
 478                 dep->flags |= DWC3_EP_ENABLED;
 479
 480                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 481                 reg |= DWC3_DALEPENA_EP(dep->number);
 482                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 483
 484                 if (!usb_endpoint_xfer_isoc(desc))
 485                         return 0;
 486
 487                 /* Link TRB for ISOC. The HWO bit is never reset */
 488                 trb_st_hw = &dep->trb_pool[0];
 489
 490                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 491                 memset(trb_link, 0, sizeof(*trb_link));
 492
 493                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 494                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 495                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 496                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 497         }
 498
 499         switch (usb_endpoint_type(desc)) {
 500         case USB_ENDPOINT_XFER_CONTROL:
 501                 strlcat(dep->name, "-control", sizeof(dep->name));
 502                 break;
 503         case USB_ENDPOINT_XFER_ISOC:
 504                 strlcat(dep->name, "-isoc", sizeof(dep->name));
 505                 break;
 506         case USB_ENDPOINT_XFER_BULK:
 507                 strlcat(dep->name, "-bulk", sizeof(dep->name));
 508                 break;
 509         case USB_ENDPOINT_XFER_INT:
 510                 strlcat(dep->name, "-int", sizeof(dep->name));
 511                 break;
 512         default:
 513                 dev_err(dwc->dev, "invalid endpoint transfer type\n");
 514         }
 515
 516         return 0;
 517 }
 518
 519 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
 520 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 521 {
 522         struct dwc3_request             *req;
 523
 524         if (!list_empty(&dep->req_queued)) {
 525                 dwc3_stop_active_transfer(dwc, dep->number, true);
 526
 527                 /* - giveback all requests to gadget driver */
 528                 while (!list_empty(&dep->req_queued)) {
 529                         req = next_request(&dep->req_queued);
 530
 531                         dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 532                 }
 533         }
 534
 535         while (!list_empty(&dep->request_list)) {
 536                 req = next_request(&dep->request_list);
 537
 538                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 539         }
 540 }
 541
 542 /**
 543  * __dwc3_gadget_ep_disable - Disables a HW endpoint
 544  * @dep: the endpoint to disable
 545  *
 546  * This function also removes requests which are currently processed ny the
 547  * hardware and those which are not yet scheduled.
 548  * Caller should take care of locking.
 549  */
 550 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 551 {
 552         struct dwc3             *dwc = dep->dwc;
 553         u32                     reg;
 554
 555         dwc3_trace(trace_dwc3_gadget, "Disabling %s", dep->name);
 556
 557         dwc3_remove_requests(dwc, dep);
 558
 559         /* make sure HW endpoint isn't stalled */
 560         if (dep->flags & DWC3_EP_STALL)
 561                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 562
 563         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 564         reg &= ~DWC3_DALEPENA_EP(dep->number);
 565         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 566
 567         dep->stream_capable = false;
 568         dep->endpoint.desc = NULL;
 569         dep->comp_desc = NULL;
 570         dep->type = 0;
 571         dep->flags = 0;
 572
 573         snprintf(dep->name, sizeof(dep->name), "ep%d%s",
 574                         dep->number >> 1,
 575                         (dep->number & 1) ? "in" : "out");
 576
 577         return 0;
 578 }
 579
 580 /* -------------------------------------------------------------------------- */
 581
 582 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 583                 const struct usb_endpoint_descriptor *desc)
 584 {
 585         return -EINVAL;
 586 }
 587
 588 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 589 {
 590         return -EINVAL;
 591 }
 592
 593 /* -------------------------------------------------------------------------- */
 594
 595 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 596                 const struct usb_endpoint_descriptor *desc)
 597 {
 598         struct dwc3_ep                  *dep;
 599         struct dwc3                     *dwc;
 600         unsigned long                   flags;
 601         int                             ret;
 602
 603         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 604                 pr_debug("dwc3: invalid parameters\n");
 605                 return -EINVAL;
 606         }
 607
 608         if (!desc->wMaxPacketSize) {
 609                 pr_debug("dwc3: missing wMaxPacketSize\n");
 610                 return -EINVAL;
 611         }
 612
 613         dep = to_dwc3_ep(ep);
 614         dwc = dep->dwc;
 615
 616         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 617                                         "%s is already enabled\n",
 618                                         dep->name))
 619                 return 0;
 620
 621         spin_lock_irqsave(&dwc->lock, flags);
 622         ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false);
 623         spin_unlock_irqrestore(&dwc->lock, flags);
 624
 625         return ret;
 626 }
 627
 628 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 629 {
 630         struct dwc3_ep                  *dep;
 631         struct dwc3                     *dwc;
 632         unsigned long                   flags;
 633         int                             ret;
 634
 635         if (!ep) {
 636                 pr_debug("dwc3: invalid parameters\n");
 637                 return -EINVAL;
 638         }
 639
 640         dep = to_dwc3_ep(ep);
 641         dwc = dep->dwc;
 642
 643         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 644                                         "%s is already disabled\n",
 645                                         dep->name))
 646                 return 0;
 647
 648         spin_lock_irqsave(&dwc->lock, flags);
 649         ret = __dwc3_gadget_ep_disable(dep);
 650         spin_unlock_irqrestore(&dwc->lock, flags);
 651
 652         return ret;
 653 }
 654
 655 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 656         gfp_t gfp_flags)
 657 {
 658         struct dwc3_request             *req;
 659         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 660
 661         req = kzalloc(sizeof(*req), gfp_flags);
 662         if (!req)
 663                 return NULL;
 664
 665         req->epnum      = dep->number;
 666         req->dep        = dep;
 667
 668         trace_dwc3_alloc_request(req);
 669
 670         return &req->request;
 671 }
 672
 673 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 674                 struct usb_request *request)
 675 {
 676         struct dwc3_request             *req = to_dwc3_request(request);
 677
 678         trace_dwc3_free_request(req);
 679         kfree(req);
 680 }
 681
 682 /**
 683  * dwc3_prepare_one_trb - setup one TRB from one request
 684  * @dep: endpoint for which this request is prepared
 685  * @req: dwc3_request pointer
 686  */
 687 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 688                 struct dwc3_request *req, dma_addr_t dma,
 689                 unsigned length, unsigned last, unsigned chain, unsigned node)
 690 {
 691         struct dwc3_trb         *trb;
 692
 693         dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s%s",
 694                         dep->name, req, (unsigned long long) dma,
 695                         length, last ? " last" : "",
 696                         chain ? " chain" : "");
 697
 698
 699         trb = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
 700
 701         if (!req->trb) {
 702                 dwc3_gadget_move_request_queued(req);
 703                 req->trb = trb;
 704                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
 705                 req->start_slot = dep->free_slot & DWC3_TRB_MASK;
 706         }
 707
 708         dep->free_slot++;
 709         /* Skip the LINK-TRB on ISOC */
 710         if (((dep->free_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
 711                         usb_endpoint_xfer_isoc(dep->endpoint.desc))
 712                 dep->free_slot++;
 713
 714         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 715         trb->bpl = lower_32_bits(dma);
 716         trb->bph = upper_32_bits(dma);
 717
 718         switch (usb_endpoint_type(dep->endpoint.desc)) {
 719         case USB_ENDPOINT_XFER_CONTROL:
 720                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 721                 break;
 722
 723         case USB_ENDPOINT_XFER_ISOC:
 724                 if (!node)
 725                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 726                 else
 727                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 728                 break;
 729
 730         case USB_ENDPOINT_XFER_BULK:
 731         case USB_ENDPOINT_XFER_INT:
 732                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 733                 break;
 734         default:
 735                 /*
 736                  * This is only possible with faulty memory because we
 737                  * checked it already :)
 738                  */
 739                 BUG();
 740         }
 741
 742         if (!req->request.no_interrupt && !chain)
 743                 trb->ctrl |= DWC3_TRB_CTRL_IOC;
 744
 745         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
 746                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 747                 trb->ctrl |= DWC3_TRB_CTRL_CSP;
 748         } else if (last) {
 749                 trb->ctrl |= DWC3_TRB_CTRL_LST;
 750         }
 751
 752         if (chain)
 753                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
 754
 755         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
 756                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
 757
 758         trb->ctrl |= DWC3_TRB_CTRL_HWO;
 759
 760         trace_dwc3_prepare_trb(dep, trb);
 761 }
 762
 763 /*
 764  * dwc3_prepare_trbs - setup TRBs from requests
 765  * @dep: endpoint for which requests are being prepared
 766  * @starting: true if the endpoint is idle and no requests are queued.
 767  *
 768  * The function goes through the requests list and sets up TRBs for the
 769  * transfers. The function returns once there are no more TRBs available or
 770  * it runs out of requests.
 771  */
 772 static void dwc3_prepare_trbs(struct dwc3_ep *dep, bool starting)
 773 {
 774         struct dwc3_request     *req, *n;
 775         u32                     trbs_left;
 776         u32                     max;
 777         unsigned int            last_one = 0;
 778
 779         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
 780
 781         /* the first request must not be queued */
 782         trbs_left = (dep->busy_slot - dep->free_slot) & DWC3_TRB_MASK;
 783
 784         /* Can't wrap around on a non-isoc EP since there's no link TRB */
 785         if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
 786                 max = DWC3_TRB_NUM - (dep->free_slot & DWC3_TRB_MASK);
 787                 if (trbs_left > max)
 788                         trbs_left = max;
 789         }
 790
 791         /*
 792          * If busy & slot are equal than it is either full or empty. If we are
 793          * starting to process requests then we are empty. Otherwise we are
 794          * full and don't do anything
 795          */
 796         if (!trbs_left) {
 797                 if (!starting)
 798                         return;
 799                 trbs_left = DWC3_TRB_NUM;
 800                 /*
 801                  * In case we start from scratch, we queue the ISOC requests
 802                  * starting from slot 1. This is done because we use ring
 803                  * buffer and have no LST bit to stop us. Instead, we place
 804                  * IOC bit every TRB_NUM/4. We try to avoid having an interrupt
 805                  * after the first request so we start at slot 1 and have
 806                  * 7 requests proceed before we hit the first IOC.
 807                  * Other transfer types don't use the ring buffer and are
 808                  * processed from the first TRB until the last one. Since we
 809                  * don't wrap around we have to start at the beginning.
 810                  */
 811                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
 812                         dep->busy_slot = 1;
 813                         dep->free_slot = 1;
 814                 } else {
 815                         dep->busy_slot = 0;
 816                         dep->free_slot = 0;
 817                 }
 818         }
 819
 820         /* The last TRB is a link TRB, not used for xfer */
 821         if ((trbs_left <= 1) && usb_endpoint_xfer_isoc(dep->endpoint.desc))
 822                 return;
 823
 824         list_for_each_entry_safe(req, n, &dep->request_list, list) {
 825                 unsigned        length;
 826                 dma_addr_t      dma;
 827                 last_one = false;
 828
 829                 if (req->request.num_mapped_sgs > 0) {
 830                         struct usb_request *request = &req->request;
 831                         struct scatterlist *sg = request->sg;
 832                         struct scatterlist *s;
 833                         int             i;
 834
 835                         for_each_sg(sg, s, request->num_mapped_sgs, i) {
 836                                 unsigned chain = true;
 837
 838                                 length = sg_dma_len(s);
 839                                 dma = sg_dma_address(s);
 840
 841                                 if (i == (request->num_mapped_sgs - 1) ||
 842                                                 sg_is_last(s)) {
 843                                         if (list_empty(&dep->request_list))
 844                                                 last_one = true;
 845                                         chain = false;
 846                                 }
 847
 848                                 trbs_left--;
 849                                 if (!trbs_left)
 850                                         last_one = true;
 851
 852                                 if (last_one)
 853                                         chain = false;
 854
 855                                 dwc3_prepare_one_trb(dep, req, dma, length,
 856                                                 last_one, chain, i);
 857
 858                                 if (last_one)
 859                                         break;
 860                         }
 861
 862                         if (last_one)
 863                                 break;
 864                 } else {
 865                         dma = req->request.dma;
 866                         length = req->request.length;
 867                         trbs_left--;
 868
 869                         if (!trbs_left)
 870                                 last_one = 1;
 871
 872                         /* Is this the last request? */
 873                         if (list_is_last(&req->list, &dep->request_list))
 874                                 last_one = 1;
 875
 876                         dwc3_prepare_one_trb(dep, req, dma, length,
 877                                         last_one, false, 0);
 878
 879                         if (last_one)
 880                                 break;
 881                 }
 882         }
 883 }
 884
 885 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,
 886                 int start_new)
 887 {
 888         struct dwc3_gadget_ep_cmd_params params;
 889         struct dwc3_request             *req;
 890         struct dwc3                     *dwc = dep->dwc;
 891         int                             ret;
 892         u32                             cmd;
 893
 894         if (start_new && (dep->flags & DWC3_EP_BUSY)) {
 895                 dwc3_trace(trace_dwc3_gadget, "%s: endpoint busy", dep->name);
 896                 return -EBUSY;
 897         }
 898
 899         /*
 900          * If we are getting here after a short-out-packet we don't enqueue any
 901          * new requests as we try to set the IOC bit only on the last request.
 902          */
 903         if (start_new) {
 904                 if (list_empty(&dep->req_queued))
 905                         dwc3_prepare_trbs(dep, start_new);
 906
 907                 /* req points to the first request which will be sent */
 908                 req = next_request(&dep->req_queued);
 909         } else {
 910                 dwc3_prepare_trbs(dep, start_new);
 911
 912                 /*
 913                  * req points to the first request where HWO changed from 0 to 1
 914                  */
 915                 req = next_request(&dep->req_queued);
 916         }
 917         if (!req) {
 918                 dep->flags |= DWC3_EP_PENDING_REQUEST;
 919                 return 0;
 920         }
 921
 922         memset(&params, 0, sizeof(params));
 923
 924         if (start_new) {
 925                 params.param0 = upper_32_bits(req->trb_dma);
 926                 params.param1 = lower_32_bits(req->trb_dma);
 927                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 928         } else {
 929                 cmd = DWC3_DEPCMD_UPDATETRANSFER;
 930         }
 931
 932         cmd |= DWC3_DEPCMD_PARAM(cmd_param);
 933         ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
 934         if (ret < 0) {
 935                 /*
 936                  * FIXME we need to iterate over the list of requests
 937                  * here and stop, unmap, free and del each of the linked
 938                  * requests instead of what we do now.
 939                  */
 940                 usb_gadget_unmap_request(&dwc->gadget, &req->request,
 941                                 req->direction);
 942                 list_del(&req->list);
 943                 return ret;
 944         }
 945
 946         dep->flags |= DWC3_EP_BUSY;
 947
 948         if (start_new) {
 949                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dwc,
 950                                 dep->number);
 951                 WARN_ON_ONCE(!dep->resource_index);
 952         }
 953
 954         return 0;
 955 }
 956
 957 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
 958                 struct dwc3_ep *dep, u32 cur_uf)
 959 {
 960         u32 uf;
 961
 962         if (list_empty(&dep->request_list)) {
 963                 dwc3_trace(trace_dwc3_gadget,
 964                                 "ISOC ep %s run out for requests",
 965                                 dep->name);
 966                 dep->flags |= DWC3_EP_PENDING_REQUEST;
 967                 return;
 968         }
 969
 970         /* 4 micro frames in the future */
 971         uf = cur_uf + dep->interval * 4;
 972
 973         __dwc3_gadget_kick_transfer(dep, uf, 1);
 974 }
 975
 976 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
 977                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
 978 {
 979         u32 cur_uf, mask;
 980
 981         mask = ~(dep->interval - 1);
 982         cur_uf = event->parameters & mask;
 983
 984         __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
 985 }
 986
 987 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
 988 {
 989         struct dwc3             *dwc = dep->dwc;
 990         int                     ret;
 991
 992         if (!dep->endpoint.desc) {
 993                 dwc3_trace(trace_dwc3_gadget,
 994                                 "trying to queue request %p to disabled %s\n",
 995                                 &req->request, dep->endpoint.name);
 996                 return -ESHUTDOWN;
 997         }
 998
 999         if (WARN(req->dep != dep, "request %p belongs to '%s'\n",
1000                                 &req->request, req->dep->name)) {
1001                 dwc3_trace(trace_dwc3_gadget, "request %p belongs to '%s'\n",
1002                                 &req->request, req->dep->name);
1003                 return -EINVAL;
1004         }
1005
1006         req->request.actual     = 0;
1007         req->request.status     = -EINPROGRESS;
1008         req->direction          = dep->direction;
1009         req->epnum              = dep->number;
1010
1011         trace_dwc3_ep_queue(req);
1012
1013         /*
1014          * Per databook, the total size of buffer must be a multiple
1015          * of MaxPacketSize for OUT endpoints. And MaxPacketSize is
1016          * configed for endpoints in dwc3_gadget_set_ep_config(),
1017          * set to usb_endpoint_descriptor->wMaxPacketSize.
1018          */
1019         if (dep->direction == 0 &&
1020             req->request.length % dep->endpoint.desc->wMaxPacketSize)
1021                 req->request.length = roundup(req->request.length,
1022                                         dep->endpoint.desc->wMaxPacketSize);
1023
1024         /*
1025          * We only add to our list of requests now and
1026          * start consuming the list once we get XferNotReady
1027          * IRQ.
1028          *
1029          * That way, we avoid doing anything that we don't need
1030          * to do now and defer it until the point we receive a
1031          * particular token from the Host side.
1032          *
1033          * This will also avoid Host cancelling URBs due to too
1034          * many NAKs.
1035          */
1036         ret = usb_gadget_map_request(&dwc->gadget, &req->request,
1037                         dep->direction);
1038         if (ret)
1039                 return ret;
1040
1041         list_add_tail(&req->list, &dep->request_list);
1042
1043         /*
1044          * If there are no pending requests and the endpoint isn't already
1045          * busy, we will just start the request straight away.
1046          *
1047          * This will save one IRQ (XFER_NOT_READY) and possibly make it a
1048          * little bit faster.
1049          */
1050         if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1051                         !usb_endpoint_xfer_int(dep->endpoint.desc) &&
1052                         !(dep->flags & DWC3_EP_BUSY)) {
1053                 ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1054                 goto out;
1055         }
1056
1057         /*
1058          * There are a few special cases:
1059          *
1060          * 1. XferNotReady with empty list of requests. We need to kick the
1061          *    transfer here in that situation, otherwise we will be NAKing
1062          *    forever. If we get XferNotReady before gadget driver has a
1063          *    chance to queue a request, we will ACK the IRQ but won't be
1064          *    able to receive the data until the next request is queued.
1065          *    The following code is handling exactly that.
1066          *
1067          */
1068         if (dep->flags & DWC3_EP_PENDING_REQUEST) {
1069                 /*
1070                  * If xfernotready is already elapsed and it is a case
1071                  * of isoc transfer, then issue END TRANSFER, so that
1072                  * you can receive xfernotready again and can have
1073                  * notion of current microframe.
1074                  */
1075                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1076                         if (list_empty(&dep->req_queued)) {
1077                                 dwc3_stop_active_transfer(dwc, dep->number, true);
1078                                 dep->flags = DWC3_EP_ENABLED;
1079                         }
1080                         return 0;
1081                 }
1082
1083                 ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1084                 if (!ret)
1085                         dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1086
1087                 goto out;
1088         }
1089
1090         /*
1091          * 2. XferInProgress on Isoc EP with an active transfer. We need to
1092          *    kick the transfer here after queuing a request, otherwise the
1093          *    core may not see the modified TRB(s).
1094          */
1095         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1096                         (dep->flags & DWC3_EP_BUSY) &&
1097                         !(dep->flags & DWC3_EP_MISSED_ISOC)) {
1098                 WARN_ON_ONCE(!dep->resource_index);
1099                 ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index,
1100                                 false);
1101                 goto out;
1102         }
1103
1104         /*
1105          * 4. Stream Capable Bulk Endpoints. We need to start the transfer
1106          * right away, otherwise host will not know we have streams to be
1107          * handled.
1108          */
1109         if (dep->stream_capable)
1110                 ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1111
1112 out:
1113         if (ret && ret != -EBUSY)
1114                 dwc3_trace(trace_dwc3_gadget,
1115                                 "%s: failed to kick transfers\n",
1116                                 dep->name);
1117         if (ret == -EBUSY)
1118                 ret = 0;
1119
1120         return ret;
1121 }
1122
1123 static void __dwc3_gadget_ep_zlp_complete(struct usb_ep *ep,
1124                 struct usb_request *request)
1125 {
1126         dwc3_gadget_ep_free_request(ep, request);
1127 }
1128
1129 static int __dwc3_gadget_ep_queue_zlp(struct dwc3 *dwc, struct dwc3_ep *dep)
1130 {
1131         struct dwc3_request             *req;
1132         struct usb_request              *request;
1133         struct usb_ep                   *ep = &dep->endpoint;
1134
1135         dwc3_trace(trace_dwc3_gadget, "queueing ZLP\n");
1136         request = dwc3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
1137         if (!request)
1138                 return -ENOMEM;
1139
1140         request->length = 0;
1141         request->buf = dwc->zlp_buf;
1142         request->complete = __dwc3_gadget_ep_zlp_complete;
1143
1144         req = to_dwc3_request(request);
1145
1146         return __dwc3_gadget_ep_queue(dep, req);
1147 }
1148
1149 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1150         gfp_t gfp_flags)
1151 {
1152         struct dwc3_request             *req = to_dwc3_request(request);
1153         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1154         struct dwc3                     *dwc = dep->dwc;
1155
1156         unsigned long                   flags;
1157
1158         int                             ret;
1159
1160         spin_lock_irqsave(&dwc->lock, flags);
1161         ret = __dwc3_gadget_ep_queue(dep, req);
1162
1163         /*
1164          * Okay, here's the thing, if gadget driver has requested for a ZLP by
1165          * setting request->zero, instead of doing magic, we will just queue an
1166          * extra usb_request ourselves so that it gets handled the same way as
1167          * any other request.
1168          */
1169         if (ret == 0 && request->zero && request->length &&
1170             (request->length % ep->desc->wMaxPacketSize == 0))
1171                 ret = __dwc3_gadget_ep_queue_zlp(dwc, dep);
1172
1173         spin_unlock_irqrestore(&dwc->lock, flags);
1174
1175         return ret;
1176 }
1177
1178 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1179                 struct usb_request *request)
1180 {
1181         struct dwc3_request             *req = to_dwc3_request(request);
1182         struct dwc3_request             *r = NULL;
1183
1184         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1185         struct dwc3                     *dwc = dep->dwc;
1186
1187         unsigned long                   flags;
1188         int                             ret = 0;
1189
1190         trace_dwc3_ep_dequeue(req);
1191
1192         spin_lock_irqsave(&dwc->lock, flags);
1193
1194         list_for_each_entry(r, &dep->request_list, list) {
1195                 if (r == req)
1196                         break;
1197         }
1198
1199         if (r != req) {
1200                 list_for_each_entry(r, &dep->req_queued, list) {
1201                         if (r == req)
1202                                 break;
1203                 }
1204                 if (r == req) {
1205                         /* wait until it is processed */
1206                         dwc3_stop_active_transfer(dwc, dep->number, true);
1207                         goto out1;
1208                 }
1209                 dev_err(dwc->dev, "request %p was not queued to %s\n",
1210                                 request, ep->name);
1211                 ret = -EINVAL;
1212                 goto out0;
1213         }
1214
1215 out1:
1216         /* giveback the request */
1217         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1218
1219 out0:
1220         spin_unlock_irqrestore(&dwc->lock, flags);
1221
1222         return ret;
1223 }
1224
1225 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1226 {
1227         struct dwc3_gadget_ep_cmd_params        params;
1228         struct dwc3                             *dwc = dep->dwc;
1229         int                                     ret;
1230
1231         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1232                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1233                 return -EINVAL;
1234         }
1235
1236         memset(&params, 0x00, sizeof(params));
1237
1238         if (value) {
1239                 if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) ||
1240                                 (!list_empty(&dep->req_queued) ||
1241                                  !list_empty(&dep->request_list)))) {
1242                         dwc3_trace(trace_dwc3_gadget,
1243                                         "%s: pending request, cannot halt\n",
1244                                         dep->name);
1245                         return -EAGAIN;
1246                 }
1247
1248                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1249                         DWC3_DEPCMD_SETSTALL, &params);
1250                 if (ret)
1251                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1252                                         dep->name);
1253                 else
1254                         dep->flags |= DWC3_EP_STALL;
1255         } else {
1256                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1257                         DWC3_DEPCMD_CLEARSTALL, &params);
1258                 if (ret)
1259                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1260                                         dep->name);
1261                 else
1262                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1263         }
1264
1265         return ret;
1266 }
1267
1268 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1269 {
1270         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1271         struct dwc3                     *dwc = dep->dwc;
1272
1273         unsigned long                   flags;
1274
1275         int                             ret;
1276
1277         spin_lock_irqsave(&dwc->lock, flags);
1278         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1279         spin_unlock_irqrestore(&dwc->lock, flags);
1280
1281         return ret;
1282 }
1283
1284 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1285 {
1286         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1287         struct dwc3                     *dwc = dep->dwc;
1288         unsigned long                   flags;
1289         int                             ret;
1290
1291         spin_lock_irqsave(&dwc->lock, flags);
1292         dep->flags |= DWC3_EP_WEDGE;
1293
1294         if (dep->number == 0 || dep->number == 1)
1295                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1296         else
1297                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1298         spin_unlock_irqrestore(&dwc->lock, flags);
1299
1300         return ret;
1301 }
1302
1303 /* -------------------------------------------------------------------------- */
1304
1305 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1306         .bLength        = USB_DT_ENDPOINT_SIZE,
1307         .bDescriptorType = USB_DT_ENDPOINT,
1308         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1309 };
1310
1311 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1312         .enable         = dwc3_gadget_ep0_enable,
1313         .disable        = dwc3_gadget_ep0_disable,
1314         .alloc_request  = dwc3_gadget_ep_alloc_request,
1315         .free_request   = dwc3_gadget_ep_free_request,
1316         .queue          = dwc3_gadget_ep0_queue,
1317         .dequeue        = dwc3_gadget_ep_dequeue,
1318         .set_halt       = dwc3_gadget_ep0_set_halt,
1319         .set_wedge      = dwc3_gadget_ep_set_wedge,
1320 };
1321
1322 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1323         .enable         = dwc3_gadget_ep_enable,
1324         .disable        = dwc3_gadget_ep_disable,
1325         .alloc_request  = dwc3_gadget_ep_alloc_request,
1326         .free_request   = dwc3_gadget_ep_free_request,
1327         .queue          = dwc3_gadget_ep_queue,
1328         .dequeue        = dwc3_gadget_ep_dequeue,
1329         .set_halt       = dwc3_gadget_ep_set_halt,
1330         .set_wedge      = dwc3_gadget_ep_set_wedge,
1331 };
1332
1333 /* -------------------------------------------------------------------------- */
1334
1335 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1336 {
1337         struct dwc3             *dwc = gadget_to_dwc(g);
1338         u32                     reg;
1339
1340         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1341         return DWC3_DSTS_SOFFN(reg);
1342 }
1343
1344 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1345 {
1346         struct dwc3             *dwc = gadget_to_dwc(g);
1347
1348         unsigned long           timeout;
1349         unsigned long           flags;
1350
1351         u32                     reg;
1352
1353         int                     ret = 0;
1354
1355         u8                      link_state;
1356         u8                      speed;
1357
1358         spin_lock_irqsave(&dwc->lock, flags);
1359
1360         /*
1361          * According to the Databook Remote wakeup request should
1362          * be issued only when the device is in early suspend state.
1363          *
1364          * We can check that via USB Link State bits in DSTS register.
1365          */
1366         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1367
1368         speed = reg & DWC3_DSTS_CONNECTSPD;
1369         if (speed == DWC3_DSTS_SUPERSPEED) {
1370                 dwc3_trace(trace_dwc3_gadget, "no wakeup on SuperSpeed\n");
1371                 ret = -EINVAL;
1372                 goto out;
1373         }
1374
1375         link_state = DWC3_DSTS_USBLNKST(reg);
1376
1377         switch (link_state) {
1378         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1379         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1380                 break;
1381         default:
1382                 dwc3_trace(trace_dwc3_gadget,
1383                                 "can't wakeup from '%s'\n",
1384                                 dwc3_gadget_link_string(link_state));
1385                 ret = -EINVAL;
1386                 goto out;
1387         }
1388
1389         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1390         if (ret < 0) {
1391                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1392                 goto out;
1393         }
1394
1395         /* Recent versions do this automatically */
1396         if (dwc->revision < DWC3_REVISION_194A) {
1397                 /* write zeroes to Link Change Request */
1398                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1399                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1400                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1401         }
1402
1403         /* poll until Link State changes to ON */
1404         timeout = jiffies + msecs_to_jiffies(100);
1405
1406         while (!time_after(jiffies, timeout)) {
1407                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1408
1409                 /* in HS, means ON */
1410                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1411                         break;
1412         }
1413
1414         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1415                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1416                 ret = -EINVAL;
1417         }
1418
1419 out:
1420         spin_unlock_irqrestore(&dwc->lock, flags);
1421
1422         return ret;
1423 }
1424
1425 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1426                 int is_selfpowered)
1427 {
1428         struct dwc3             *dwc = gadget_to_dwc(g);
1429         unsigned long           flags;
1430
1431         spin_lock_irqsave(&dwc->lock, flags);
1432         g->is_selfpowered = !!is_selfpowered;
1433         spin_unlock_irqrestore(&dwc->lock, flags);
1434
1435         return 0;
1436 }
1437
1438 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1439 {
1440         u32                     reg;
1441         u32                     timeout = 500;
1442
1443         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1444         if (is_on) {
1445                 if (dwc->revision <= DWC3_REVISION_187A) {
1446                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1447                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1448                 }
1449
1450                 if (dwc->revision >= DWC3_REVISION_194A)
1451                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1452                 reg |= DWC3_DCTL_RUN_STOP;
1453
1454                 if (dwc->has_hibernation)
1455                         reg |= DWC3_DCTL_KEEP_CONNECT;
1456
1457                 dwc->pullups_connected = true;
1458         } else {
1459                 reg &= ~DWC3_DCTL_RUN_STOP;
1460
1461                 if (dwc->has_hibernation && !suspend)
1462                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1463
1464                 dwc->pullups_connected = false;
1465         }
1466
1467         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1468
1469         do {
1470                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1471                 if (is_on) {
1472                         if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1473                                 break;
1474                 } else {
1475                         if (reg & DWC3_DSTS_DEVCTRLHLT)
1476                                 break;
1477                 }
1478                 timeout--;
1479                 if (!timeout)
1480                         return -ETIMEDOUT;
1481                 udelay(1);
1482         } while (1);
1483
1484         dwc3_trace(trace_dwc3_gadget, "gadget %s data soft-%s",
1485                         dwc->gadget_driver
1486                         ? dwc->gadget_driver->function : "no-function",
1487                         is_on ? "connect" : "disconnect");
1488
1489         return 0;
1490 }
1491
1492 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1493 {
1494         struct dwc3             *dwc = gadget_to_dwc(g);
1495         unsigned long           flags;
1496         int                     ret = 0;
1497         u32                     reg;
1498
1499         is_on = !!is_on;
1500
1501         spin_lock_irqsave(&dwc->lock, flags);
1502
1503         dwc->enabled = is_on;
1504
1505         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
1506
1507         if (DWC3_GCTL_PRTCAP(reg) == DWC3_GCTL_PRTCAP_DEVICE)
1508                 ret = dwc3_gadget_run_stop(dwc, is_on, false);
1509
1510         spin_unlock_irqrestore(&dwc->lock, flags);
1511
1512         return ret;
1513 }
1514
1515 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1516 {
1517         u32                     reg;
1518
1519         /* Enable all but Start and End of Frame IRQs */
1520         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1521                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1522                         DWC3_DEVTEN_CMDCMPLTEN |
1523                         DWC3_DEVTEN_ERRTICERREN |
1524                         DWC3_DEVTEN_WKUPEVTEN |
1525                         DWC3_DEVTEN_ULSTCNGEN |
1526                         DWC3_DEVTEN_CONNECTDONEEN |
1527                         DWC3_DEVTEN_USBRSTEN |
1528                         DWC3_DEVTEN_DISCONNEVTEN);
1529
1530         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1531 }
1532
1533 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1534 {
1535         /* mask all interrupts */
1536         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1537 }
1538
1539 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1540 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1541
1542 static int dwc3_gadget_start(struct usb_gadget *g,
1543                 struct usb_gadget_driver *driver)
1544 {
1545         struct dwc3             *dwc = gadget_to_dwc(g);
1546         struct dwc3_ep          *dep;
1547         unsigned long           flags;
1548         int                     ret = 0;
1549         int                     irq;
1550         u32                     reg;
1551
1552         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1553         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1554                         IRQF_SHARED, "dwc3", dwc);
1555         if (ret) {
1556                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1557                                 irq, ret);
1558                 goto err0;
1559         }
1560
1561         spin_lock_irqsave(&dwc->lock, flags);
1562
1563         if (dwc->gadget_driver) {
1564                 dev_err(dwc->dev, "%s is already bound to %s\n",
1565                                 dwc->gadget.name,
1566                                 dwc->gadget_driver->driver.name);
1567                 ret = -EBUSY;
1568                 goto err1;
1569         }
1570
1571         dwc->gadget_driver      = driver;
1572
1573         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
1574         if (DWC3_GCTL_PRTCAP(reg) != DWC3_GCTL_PRTCAP_DEVICE)
1575                 goto mode_mismatch;
1576
1577         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1578         reg &= ~(DWC3_DCFG_SPEED_MASK);
1579
1580         /**
1581          * WORKAROUND: DWC3 revision < 2.20a have an issue
1582          * which would cause metastability state on Run/Stop
1583          * bit if we try to force the IP to USB2-only mode.
1584          *
1585          * Because of that, we cannot configure the IP to any
1586          * speed other than the SuperSpeed
1587          *
1588          * Refers to:
1589          *
1590          * STAR#9000525659: Clock Domain Crossing on DCTL in
1591          * USB 2.0 Mode
1592          */
1593         if (dwc->revision < DWC3_REVISION_220A) {
1594                 reg |= DWC3_DCFG_SUPERSPEED;
1595         } else {
1596                 switch (dwc->maximum_speed) {
1597                 case USB_SPEED_LOW:
1598                         reg |= DWC3_DSTS_LOWSPEED;
1599                         break;
1600                 case USB_SPEED_FULL:
1601                         reg |= DWC3_DSTS_FULLSPEED1;
1602                         break;
1603                 case USB_SPEED_HIGH:
1604                         reg |= DWC3_DSTS_HIGHSPEED;
1605                         break;
1606                 case USB_SPEED_SUPER:   /* FALLTHROUGH */
1607                 case USB_SPEED_UNKNOWN: /* FALTHROUGH */
1608                 default:
1609                         reg |= DWC3_DSTS_SUPERSPEED;
1610                 }
1611         }
1612         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1613
1614         /* Start with SuperSpeed Default */
1615         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1616
1617         dep = dwc->eps[0];
1618         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1619                         false);
1620         if (ret) {
1621                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1622                 goto err2;
1623         }
1624
1625         dep = dwc->eps[1];
1626         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1627                         false);
1628         if (ret) {
1629                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1630                 goto err3;
1631         }
1632
1633         /* begin to receive SETUP packets */
1634         dwc->ep0state = EP0_SETUP_PHASE;
1635         dwc3_ep0_out_start(dwc);
1636
1637         dwc3_gadget_enable_irq(dwc);
1638
1639 mode_mismatch:
1640         spin_unlock_irqrestore(&dwc->lock, flags);
1641
1642         return 0;
1643
1644 err3:
1645         __dwc3_gadget_ep_disable(dwc->eps[0]);
1646
1647 err2:
1648         dwc->gadget_driver = NULL;
1649
1650 err1:
1651         spin_unlock_irqrestore(&dwc->lock, flags);
1652
1653         free_irq(irq, dwc);
1654
1655 err0:
1656         return ret;
1657 }
1658
1659 static int dwc3_gadget_stop(struct usb_gadget *g)
1660 {
1661         struct dwc3             *dwc = gadget_to_dwc(g);
1662         unsigned long           flags;
1663         int                     irq;
1664         u32                     reg;
1665
1666         spin_lock_irqsave(&dwc->lock, flags);
1667
1668         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
1669
1670         if (DWC3_GCTL_PRTCAP(reg) == DWC3_GCTL_PRTCAP_DEVICE) {
1671                 dwc3_gadget_disable_irq(dwc);
1672                 __dwc3_gadget_ep_disable(dwc->eps[0]);
1673                 __dwc3_gadget_ep_disable(dwc->eps[1]);
1674         }
1675
1676         dwc->gadget_driver      = NULL;
1677
1678         spin_unlock_irqrestore(&dwc->lock, flags);
1679
1680         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1681         free_irq(irq, dwc);
1682
1683         return 0;
1684 }
1685
1686 static const struct usb_gadget_ops dwc3_gadget_ops = {
1687         .get_frame              = dwc3_gadget_get_frame,
1688         .wakeup                 = dwc3_gadget_wakeup,
1689         .set_selfpowered        = dwc3_gadget_set_selfpowered,
1690         .pullup                 = dwc3_gadget_pullup,
1691         .udc_start              = dwc3_gadget_start,
1692         .udc_stop               = dwc3_gadget_stop,
1693 };
1694
1695 /* -------------------------------------------------------------------------- */
1696
1697 static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
1698                 u8 num, u32 direction)
1699 {
1700         struct dwc3_ep                  *dep;
1701         u8                              i;
1702
1703         for (i = 0; i < num; i++) {
1704                 u8 epnum = (i << 1) | (!!direction);
1705
1706                 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1707                 if (!dep)
1708                         return -ENOMEM;
1709
1710                 dep->dwc = dwc;
1711                 dep->number = epnum;
1712                 dep->direction = !!direction;
1713                 dwc->eps[epnum] = dep;
1714
1715                 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1716                                 (epnum & 1) ? "in" : "out");
1717
1718                 dep->endpoint.name = dep->name;
1719
1720                 dwc3_trace(trace_dwc3_gadget, "initializing %s", dep->name);
1721
1722                 if (epnum == 0 || epnum == 1) {
1723                         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
1724                         dep->endpoint.maxburst = 1;
1725                         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1726                         if (!epnum)
1727                                 dwc->gadget.ep0 = &dep->endpoint;
1728                 } else {
1729                         int             ret;
1730
1731                         usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
1732                         dep->endpoint.max_streams = 15;
1733                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
1734                         list_add_tail(&dep->endpoint.ep_list,
1735                                         &dwc->gadget.ep_list);
1736
1737                         ret = dwc3_alloc_trb_pool(dep);
1738                         if (ret)
1739                                 return ret;
1740                 }
1741
1742                 if (epnum == 0 || epnum == 1) {
1743                         dep->endpoint.caps.type_control = true;
1744                 } else {
1745                         dep->endpoint.caps.type_iso = true;
1746                         dep->endpoint.caps.type_bulk = true;
1747                         dep->endpoint.caps.type_int = true;
1748                 }
1749
1750                 dep->endpoint.caps.dir_in = !!direction;
1751                 dep->endpoint.caps.dir_out = !direction;
1752
1753                 INIT_LIST_HEAD(&dep->request_list);
1754                 INIT_LIST_HEAD(&dep->req_queued);
1755         }
1756
1757         return 0;
1758 }
1759
1760 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1761 {
1762         int                             ret;
1763
1764         INIT_LIST_HEAD(&dwc->gadget.ep_list);
1765
1766         ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
1767         if (ret < 0) {
1768                 dwc3_trace(trace_dwc3_gadget,
1769                                 "failed to allocate OUT endpoints");
1770                 return ret;
1771         }
1772
1773         ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
1774         if (ret < 0) {
1775                 dwc3_trace(trace_dwc3_gadget,
1776                                 "failed to allocate IN endpoints");
1777                 return ret;
1778         }
1779
1780         return 0;
1781 }
1782
1783 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1784 {
1785         struct dwc3_ep                  *dep;
1786         u8                              epnum;
1787
1788         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1789                 dep = dwc->eps[epnum];
1790                 if (!dep)
1791                         continue;
1792                 /*
1793                  * Physical endpoints 0 and 1 are special; they form the
1794                  * bi-directional USB endpoint 0.
1795                  *
1796                  * For those two physical endpoints, we don't allocate a TRB
1797                  * pool nor do we add them the endpoints list. Due to that, we
1798                  * shouldn't do these two operations otherwise we would end up
1799                  * with all sorts of bugs when removing dwc3.ko.
1800                  */
1801                 if (epnum != 0 && epnum != 1) {
1802                         dwc3_free_trb_pool(dep);
1803                         list_del(&dep->endpoint.ep_list);
1804                 }
1805
1806                 kfree(dep);
1807         }
1808 }
1809
1810 /* -------------------------------------------------------------------------- */
1811
1812 static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
1813                 struct dwc3_request *req, struct dwc3_trb *trb,
1814                 const struct dwc3_event_depevt *event, int status)
1815 {
1816         unsigned int            count;
1817         unsigned int            s_pkt = 0;
1818         unsigned int            trb_status;
1819
1820         trace_dwc3_complete_trb(dep, trb);
1821
1822         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
1823                 /*
1824                  * We continue despite the error. There is not much we
1825                  * can do. If we don't clean it up we loop forever. If
1826                  * we skip the TRB then it gets overwritten after a
1827                  * while since we use them in a ring buffer. A BUG()
1828                  * would help. Lets hope that if this occurs, someone
1829                  * fixes the root cause instead of looking away :)
1830                  */
1831                 dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1832                                 dep->name, trb);
1833         count = trb->size & DWC3_TRB_SIZE_MASK;
1834
1835         if (dep->direction) {
1836                 if (count) {
1837                         trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
1838                         if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
1839                                 dwc3_trace(trace_dwc3_gadget,
1840                                                 "%s: incomplete IN transfer\n",
1841                                                 dep->name);
1842                                 /*
1843                                  * If missed isoc occurred and there is
1844                                  * no request queued then issue END
1845                                  * TRANSFER, so that core generates
1846                                  * next xfernotready and we will issue
1847                                  * a fresh START TRANSFER.
1848                                  * If there are still queued request
1849                                  * then wait, do not issue either END
1850                                  * or UPDATE TRANSFER, just attach next
1851                                  * request in request_list during
1852                                  * giveback.If any future queued request
1853                                  * is successfully transferred then we
1854                                  * will issue UPDATE TRANSFER for all
1855                                  * request in the request_list.
1856                                  */
1857                                 dep->flags |= DWC3_EP_MISSED_ISOC;
1858                         } else {
1859                                 dev_err(dwc->dev, "incomplete IN transfer %s\n",
1860                                                 dep->name);
1861                                 status = -ECONNRESET;
1862                         }
1863                 } else {
1864                         dep->flags &= ~DWC3_EP_MISSED_ISOC;
1865                 }
1866         } else {
1867                 if (count && (event->status & DEPEVT_STATUS_SHORT))
1868                         s_pkt = 1;
1869         }
1870
1871         /*
1872          * We assume here we will always receive the entire data block
1873          * which we should receive. Meaning, if we program RX to
1874          * receive 4K but we receive only 2K, we assume that's all we
1875          * should receive and we simply bounce the request back to the
1876          * gadget driver for further processing.
1877          */
1878         req->request.actual += req->request.length - count;
1879         if (s_pkt)
1880                 return 1;
1881         if ((event->status & DEPEVT_STATUS_LST) &&
1882                         (trb->ctrl & (DWC3_TRB_CTRL_LST |
1883                                 DWC3_TRB_CTRL_HWO)))
1884                 return 1;
1885         if ((event->status & DEPEVT_STATUS_IOC) &&
1886                         (trb->ctrl & DWC3_TRB_CTRL_IOC))
1887                 return 1;
1888         return 0;
1889 }
1890
1891 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1892                 const struct dwc3_event_depevt *event, int status)
1893 {
1894         struct dwc3_request     *req;
1895         struct dwc3_trb         *trb;
1896         unsigned int            slot;
1897         unsigned int            i;
1898         int                     ret;
1899
1900         do {
1901                 req = next_request(&dep->req_queued);
1902                 if (WARN_ON_ONCE(!req))
1903                         return 1;
1904
1905                 i = 0;
1906                 do {
1907                         slot = req->start_slot + i;
1908                         if ((slot == DWC3_TRB_NUM - 1) &&
1909                                 usb_endpoint_xfer_isoc(dep->endpoint.desc))
1910                                 slot++;
1911                         slot %= DWC3_TRB_NUM;
1912                         trb = &dep->trb_pool[slot];
1913
1914                         ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
1915                                         event, status);
1916                         if (ret)
1917                                 break;
1918                 } while (++i < req->request.num_mapped_sgs);
1919
1920                 dwc3_gadget_giveback(dep, req, status);
1921
1922                 if (ret)
1923                         break;
1924         } while (1);
1925
1926         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1927                         list_empty(&dep->req_queued)) {
1928                 if (list_empty(&dep->request_list)) {
1929                         /*
1930                          * If there is no entry in request list then do
1931                          * not issue END TRANSFER now. Just set PENDING
1932                          * flag, so that END TRANSFER is issued when an
1933                          * entry is added into request list.
1934                          */
1935                         dep->flags = DWC3_EP_PENDING_REQUEST;
1936                 } else {
1937                         dwc3_stop_active_transfer(dwc, dep->number, true);
1938                         dep->flags = DWC3_EP_ENABLED;
1939                 }
1940                 return 1;
1941         }
1942
1943         return 1;
1944 }
1945
1946 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
1947                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1948 {
1949         unsigned                status = 0;
1950         int                     clean_busy;
1951         u32                     is_xfer_complete;
1952
1953         is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE);
1954
1955         if (event->status & DEPEVT_STATUS_BUSERR)
1956                 status = -ECONNRESET;
1957
1958         clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
1959         if (clean_busy && (is_xfer_complete ||
1960                                 usb_endpoint_xfer_isoc(dep->endpoint.desc)))
1961                 dep->flags &= ~DWC3_EP_BUSY;
1962
1963         /*
1964          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
1965          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
1966          */
1967         if (dwc->revision < DWC3_REVISION_183A) {
1968                 u32             reg;
1969                 int             i;
1970
1971                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
1972                         dep = dwc->eps[i];
1973
1974                         if (!(dep->flags & DWC3_EP_ENABLED))
1975                                 continue;
1976
1977                         if (!list_empty(&dep->req_queued))
1978                                 return;
1979                 }
1980
1981                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1982                 reg |= dwc->u1u2;
1983                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1984
1985                 dwc->u1u2 = 0;
1986         }
1987
1988         if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1989                 int ret;
1990
1991                 ret = __dwc3_gadget_kick_transfer(dep, 0, is_xfer_complete);
1992                 if (!ret || ret == -EBUSY)
1993                         return;
1994         }
1995 }
1996
1997 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
1998                 const struct dwc3_event_depevt *event)
1999 {
2000         struct dwc3_ep          *dep;
2001         u8                      epnum = event->endpoint_number;
2002
2003         dep = dwc->eps[epnum];
2004
2005         if (!(dep->flags & DWC3_EP_ENABLED))
2006                 return;
2007
2008         if (epnum == 0 || epnum == 1) {
2009                 dwc3_ep0_interrupt(dwc, event);
2010                 return;
2011         }
2012
2013         switch (event->endpoint_event) {
2014         case DWC3_DEPEVT_XFERCOMPLETE:
2015                 dep->resource_index = 0;
2016
2017                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2018                         dwc3_trace(trace_dwc3_gadget,
2019                                         "%s is an Isochronous endpoint\n",
2020                                         dep->name);
2021                         return;
2022                 }
2023
2024                 dwc3_endpoint_transfer_complete(dwc, dep, event);
2025                 break;
2026         case DWC3_DEPEVT_XFERINPROGRESS:
2027                 dwc3_endpoint_transfer_complete(dwc, dep, event);
2028                 break;
2029         case DWC3_DEPEVT_XFERNOTREADY:
2030                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2031                         dwc3_gadget_start_isoc(dwc, dep, event);
2032                 } else {
2033                         int active;
2034                         int ret;
2035
2036                         active = event->status & DEPEVT_STATUS_TRANSFER_ACTIVE;
2037
2038                         dwc3_trace(trace_dwc3_gadget, "%s: reason %s",
2039                                         dep->name, active ? "Transfer Active"
2040                                         : "Transfer Not Active");
2041
2042                         ret = __dwc3_gadget_kick_transfer(dep, 0, !active);
2043                         if (!ret || ret == -EBUSY)
2044                                 return;
2045
2046                         dwc3_trace(trace_dwc3_gadget,
2047                                         "%s: failed to kick transfers\n",
2048                                         dep->name);
2049                 }
2050
2051                 break;
2052         case DWC3_DEPEVT_STREAMEVT:
2053                 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
2054                         dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
2055                                         dep->name);
2056                         return;
2057                 }
2058
2059                 switch (event->status) {
2060                 case DEPEVT_STREAMEVT_FOUND:
2061                         dwc3_trace(trace_dwc3_gadget,
2062                                         "Stream %d found and started",
2063                                         event->parameters);
2064
2065                         break;
2066                 case DEPEVT_STREAMEVT_NOTFOUND:
2067                         /* FALLTHROUGH */
2068                 default:
2069                         dwc3_trace(trace_dwc3_gadget,
2070                                         "unable to find suitable stream\n");
2071                 }
2072                 break;
2073         case DWC3_DEPEVT_RXTXFIFOEVT:
2074                 dwc3_trace(trace_dwc3_gadget, "%s FIFO Overrun\n", dep->name);
2075                 break;
2076         case DWC3_DEPEVT_EPCMDCMPLT:
2077                 dwc3_trace(trace_dwc3_gadget, "Endpoint Command Complete");
2078                 break;
2079         }
2080 }
2081
2082 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2083 {
2084         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2085                 spin_unlock(&dwc->lock);
2086                 dwc->gadget_driver->disconnect(&dwc->gadget);
2087                 spin_lock(&dwc->lock);
2088         }
2089 }
2090
2091 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2092 {
2093         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2094                 spin_unlock(&dwc->lock);
2095                 dwc->gadget_driver->suspend(&dwc->gadget);
2096                 spin_lock(&dwc->lock);
2097         }
2098 }
2099
2100 static void dwc3_resume_gadget(struct dwc3 *dwc)
2101 {
2102         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2103                 spin_unlock(&dwc->lock);
2104                 dwc->gadget_driver->resume(&dwc->gadget);
2105                 spin_lock(&dwc->lock);
2106         }
2107 }
2108
2109 static void dwc3_reset_gadget(struct dwc3 *dwc)
2110 {
2111         if (!dwc->gadget_driver)
2112                 return;
2113
2114         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2115                 spin_unlock(&dwc->lock);
2116                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2117                 spin_lock(&dwc->lock);
2118         }
2119 }
2120
2121 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
2122 {
2123         struct dwc3_ep *dep;
2124         struct dwc3_gadget_ep_cmd_params params;
2125         u32 cmd;
2126         int ret;
2127
2128         dep = dwc->eps[epnum];
2129
2130         if (!dep->resource_index)
2131                 return;
2132
2133         /*
2134          * NOTICE: We are violating what the Databook says about the
2135          * EndTransfer command. Ideally we would _always_ wait for the
2136          * EndTransfer Command Completion IRQ, but that's causing too
2137          * much trouble synchronizing between us and gadget driver.
2138          *
2139          * We have discussed this with the IP Provider and it was
2140          * suggested to giveback all requests here, but give HW some
2141          * extra time to synchronize with the interconnect. We're using
2142          * an arbitrary 100us delay for that.
2143          *
2144          * Note also that a similar handling was tested by Synopsys
2145          * (thanks a lot Paul) and nothing bad has come out of it.
2146          * In short, what we're doing is:
2147          *
2148          * - Issue EndTransfer WITH CMDIOC bit set
2149          * - Wait 100us
2150          */
2151
2152         cmd = DWC3_DEPCMD_ENDTRANSFER;
2153         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2154         cmd |= DWC3_DEPCMD_CMDIOC;
2155         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2156         memset(&params, 0, sizeof(params));
2157         ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
2158         WARN_ON_ONCE(ret);
2159         dep->resource_index = 0;
2160         dep->flags &= ~DWC3_EP_BUSY;
2161         udelay(100);
2162 }
2163
2164 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2165 {
2166         u32 epnum;
2167
2168         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2169                 struct dwc3_ep *dep;
2170
2171                 dep = dwc->eps[epnum];
2172                 if (!dep)
2173                         continue;
2174
2175                 if (!(dep->flags & DWC3_EP_ENABLED))
2176                         continue;
2177
2178                 dwc3_remove_requests(dwc, dep);
2179         }
2180 }
2181
2182 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2183 {
2184         u32 epnum;
2185
2186         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2187                 struct dwc3_ep *dep;
2188                 struct dwc3_gadget_ep_cmd_params params;
2189                 int ret;
2190
2191                 dep = dwc->eps[epnum];
2192                 if (!dep)
2193                         continue;
2194
2195                 if (!(dep->flags & DWC3_EP_STALL))
2196                         continue;
2197
2198                 dep->flags &= ~DWC3_EP_STALL;
2199
2200                 memset(&params, 0, sizeof(params));
2201                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
2202                                 DWC3_DEPCMD_CLEARSTALL, &params);
2203                 WARN_ON_ONCE(ret);
2204         }
2205 }
2206
2207 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2208 {
2209         int                     reg;
2210
2211         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2212         reg &= ~DWC3_DCTL_INITU1ENA;
2213         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2214
2215         reg &= ~DWC3_DCTL_INITU2ENA;
2216         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2217
2218         dwc3_disconnect_gadget(dwc);
2219
2220         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2221         dwc->setup_packet_pending = false;
2222         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2223 }
2224
2225 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2226 {
2227         u32                     reg;
2228
2229         /*
2230          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2231          * would cause a missing Disconnect Event if there's a
2232          * pending Setup Packet in the FIFO.
2233          *
2234          * There's no suggested workaround on the official Bug
2235          * report, which states that "unless the driver/application
2236          * is doing any special handling of a disconnect event,
2237          * there is no functional issue".
2238          *
2239          * Unfortunately, it turns out that we _do_ some special
2240          * handling of a disconnect event, namely complete all
2241          * pending transfers, notify gadget driver of the
2242          * disconnection, and so on.
2243          *
2244          * Our suggested workaround is to follow the Disconnect
2245          * Event steps here, instead, based on a setup_packet_pending
2246          * flag. Such flag gets set whenever we have a SETUP_PENDING
2247          * status for EP0 TRBs and gets cleared on XferComplete for the
2248          * same endpoint.
2249          *
2250          * Refers to:
2251          *
2252          * STAR#9000466709: RTL: Device : Disconnect event not
2253          * generated if setup packet pending in FIFO
2254          */
2255         if (dwc->revision < DWC3_REVISION_188A) {
2256                 if (dwc->setup_packet_pending)
2257                         dwc3_gadget_disconnect_interrupt(dwc);
2258         }
2259
2260         dwc3_reset_gadget(dwc);
2261
2262         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2263         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2264         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2265         dwc->test_mode = false;
2266
2267         dwc3_stop_active_transfers(dwc);
2268         dwc3_clear_stall_all_ep(dwc);
2269
2270         /* Reset device address to zero */
2271         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2272         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2273         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2274 }
2275
2276 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
2277 {
2278         u32 reg;
2279         u32 usb30_clock = DWC3_GCTL_CLK_BUS;
2280
2281         /*
2282          * We change the clock only at SS but I dunno why I would want to do
2283          * this. Maybe it becomes part of the power saving plan.
2284          */
2285
2286         if (speed != DWC3_DSTS_SUPERSPEED)
2287                 return;
2288
2289         /*
2290          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2291          * each time on Connect Done.
2292          */
2293         if (!usb30_clock)
2294                 return;
2295
2296         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2297         reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
2298         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
2299 }
2300
2301 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2302 {
2303         struct dwc3_ep          *dep;
2304         int                     ret;
2305         u32                     reg;
2306         u8                      speed;
2307
2308         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2309         speed = reg & DWC3_DSTS_CONNECTSPD;
2310         dwc->speed = speed;
2311
2312         dwc3_update_ram_clk_sel(dwc, speed);
2313
2314         switch (speed) {
2315         case DWC3_DCFG_SUPERSPEED:
2316                 /*
2317                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2318                  * would cause a missing USB3 Reset event.
2319                  *
2320                  * In such situations, we should force a USB3 Reset
2321                  * event by calling our dwc3_gadget_reset_interrupt()
2322                  * routine.
2323                  *
2324                  * Refers to:
2325                  *
2326                  * STAR#9000483510: RTL: SS : USB3 reset event may
2327                  * not be generated always when the link enters poll
2328                  */
2329                 if (dwc->revision < DWC3_REVISION_190A)
2330                         dwc3_gadget_reset_interrupt(dwc);
2331
2332                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2333                 dwc->gadget.ep0->maxpacket = 512;
2334                 dwc->gadget.speed = USB_SPEED_SUPER;
2335                 break;
2336         case DWC3_DCFG_HIGHSPEED:
2337                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2338                 dwc->gadget.ep0->maxpacket = 64;
2339                 dwc->gadget.speed = USB_SPEED_HIGH;
2340                 break;
2341         case DWC3_DCFG_FULLSPEED2:
2342         case DWC3_DCFG_FULLSPEED1:
2343                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2344                 dwc->gadget.ep0->maxpacket = 64;
2345                 dwc->gadget.speed = USB_SPEED_FULL;
2346                 break;
2347         case DWC3_DCFG_LOWSPEED:
2348                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2349                 dwc->gadget.ep0->maxpacket = 8;
2350                 dwc->gadget.speed = USB_SPEED_LOW;
2351                 break;
2352         }
2353
2354         /* Enable USB2 LPM Capability */
2355
2356         if ((dwc->revision > DWC3_REVISION_194A)
2357                         && (speed != DWC3_DCFG_SUPERSPEED)) {
2358                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2359                 reg |= DWC3_DCFG_LPM_CAP;
2360                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2361
2362                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2363                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2364
2365                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2366
2367                 /*
2368                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2369                  * DCFG.LPMCap is set, core responses with an ACK and the
2370                  * BESL value in the LPM token is less than or equal to LPM
2371                  * NYET threshold.
2372                  */
2373                 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2374                                 && dwc->has_lpm_erratum,
2375                                 "LPM Erratum not available on dwc3 revisisions < 2.40a\n");
2376
2377                 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2378                         reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2379
2380                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2381         } else {
2382                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2383                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2384                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2385         }
2386
2387         dep = dwc->eps[0];
2388         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2389                         false);
2390         if (ret) {
2391                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2392                 return;
2393         }
2394
2395         dep = dwc->eps[1];
2396         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2397                         false);
2398         if (ret) {
2399                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2400                 return;
2401         }
2402
2403         /*
2404          * Configure PHY via GUSB3PIPECTLn if required.
2405          *
2406          * Update GTXFIFOSIZn
2407          *
2408          * In both cases reset values should be sufficient.
2409          */
2410 }
2411
2412 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2413 {
2414         /*
2415          * TODO take core out of low power mode when that's
2416          * implemented.
2417          */
2418
2419         dwc->gadget_driver->resume(&dwc->gadget);
2420 }
2421
2422 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2423                 unsigned int evtinfo)
2424 {
2425         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2426         unsigned int            pwropt;
2427
2428         /*
2429          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2430          * Hibernation mode enabled which would show up when device detects
2431          * host-initiated U3 exit.
2432          *
2433          * In that case, device will generate a Link State Change Interrupt
2434          * from U3 to RESUME which is only necessary if Hibernation is
2435          * configured in.
2436          *
2437          * There are no functional changes due to such spurious event and we
2438          * just need to ignore it.
2439          *
2440          * Refers to:
2441          *
2442          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2443          * operational mode
2444          */
2445         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2446         if ((dwc->revision < DWC3_REVISION_250A) &&
2447                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2448                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2449                                 (next == DWC3_LINK_STATE_RESUME)) {
2450                         dwc3_trace(trace_dwc3_gadget,
2451                                         "ignoring transition U3 -> Resume");
2452                         return;
2453                 }
2454         }
2455
2456         /*
2457          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2458          * on the link partner, the USB session might do multiple entry/exit
2459          * of low power states before a transfer takes place.
2460          *
2461          * Due to this problem, we might experience lower throughput. The
2462          * suggested workaround is to disable DCTL[12:9] bits if we're
2463          * transitioning from U1/U2 to U0 and enable those bits again
2464          * after a transfer completes and there are no pending transfers
2465          * on any of the enabled endpoints.
2466          *
2467          * This is the first half of that workaround.
2468          *
2469          * Refers to:
2470          *
2471          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2472          * core send LGO_Ux entering U0
2473          */
2474         if (dwc->revision < DWC3_REVISION_183A) {
2475                 if (next == DWC3_LINK_STATE_U0) {
2476                         u32     u1u2;
2477                         u32     reg;
2478
2479                         switch (dwc->link_state) {
2480                         case DWC3_LINK_STATE_U1:
2481                         case DWC3_LINK_STATE_U2:
2482                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2483                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2484                                                 | DWC3_DCTL_ACCEPTU2ENA
2485                                                 | DWC3_DCTL_INITU1ENA
2486                                                 | DWC3_DCTL_ACCEPTU1ENA);
2487
2488                                 if (!dwc->u1u2)
2489                                         dwc->u1u2 = reg & u1u2;
2490
2491                                 reg &= ~u1u2;
2492
2493                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2494                                 break;
2495                         default:
2496                                 /* do nothing */
2497                                 break;
2498                         }
2499                 }
2500         }
2501
2502         switch (next) {
2503         case DWC3_LINK_STATE_U1:
2504                 if (dwc->speed == USB_SPEED_SUPER)
2505                         dwc3_suspend_gadget(dwc);
2506                 break;
2507         case DWC3_LINK_STATE_U2:
2508         case DWC3_LINK_STATE_U3:
2509                 dwc3_suspend_gadget(dwc);
2510                 break;
2511         case DWC3_LINK_STATE_RESUME:
2512                 dwc3_resume_gadget(dwc);
2513                 break;
2514         default:
2515                 /* do nothing */
2516                 break;
2517         }
2518
2519         dwc->link_state = next;
2520 }
2521
2522 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2523                 unsigned int evtinfo)
2524 {
2525         unsigned int is_ss = evtinfo & BIT(4);
2526
2527         /**
2528          * WORKAROUND: DWC3 revison 2.20a with hibernation support
2529          * have a known issue which can cause USB CV TD.9.23 to fail
2530          * randomly.
2531          *
2532          * Because of this issue, core could generate bogus hibernation
2533          * events which SW needs to ignore.
2534          *
2535          * Refers to:
2536          *
2537          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2538          * Device Fallback from SuperSpeed
2539          */
2540         if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2541                 return;
2542
2543         /* enter hibernation here */
2544 }
2545
2546 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2547                 const struct dwc3_event_devt *event)
2548 {
2549         switch (event->type) {
2550         case DWC3_DEVICE_EVENT_DISCONNECT:
2551                 dwc3_gadget_disconnect_interrupt(dwc);
2552                 break;
2553         case DWC3_DEVICE_EVENT_RESET:
2554                 dwc3_gadget_reset_interrupt(dwc);
2555                 break;
2556         case DWC3_DEVICE_EVENT_CONNECT_DONE:
2557                 dwc3_gadget_conndone_interrupt(dwc);
2558                 break;
2559         case DWC3_DEVICE_EVENT_WAKEUP:
2560                 dwc3_gadget_wakeup_interrupt(dwc);
2561                 break;
2562         case DWC3_DEVICE_EVENT_HIBER_REQ:
2563                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
2564                                         "unexpected hibernation event\n"))
2565                         break;
2566
2567                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2568                 break;
2569         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2570                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2571                 break;
2572         case DWC3_DEVICE_EVENT_EOPF:
2573                 dwc3_trace(trace_dwc3_gadget, "End of Periodic Frame");
2574                 break;
2575         case DWC3_DEVICE_EVENT_SOF:
2576                 dwc3_trace(trace_dwc3_gadget, "Start of Periodic Frame");
2577                 break;
2578         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2579                 dwc3_trace(trace_dwc3_gadget, "Erratic Error");
2580                 break;
2581         case DWC3_DEVICE_EVENT_CMD_CMPL:
2582                 dwc3_trace(trace_dwc3_gadget, "Command Complete");
2583                 break;
2584         case DWC3_DEVICE_EVENT_OVERFLOW:
2585                 dwc3_trace(trace_dwc3_gadget, "Overflow");
2586                 break;
2587         default:
2588                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2589         }
2590 }
2591
2592 static void dwc3_process_event_entry(struct dwc3 *dwc,
2593                 const union dwc3_event *event)
2594 {
2595         trace_dwc3_event(event->raw);
2596
2597         /* Endpoint IRQ, handle it and return early */
2598         if (event->type.is_devspec == 0) {
2599                 /* depevt */
2600                 return dwc3_endpoint_interrupt(dwc, &event->depevt);
2601         }
2602
2603         switch (event->type.type) {
2604         case DWC3_EVENT_TYPE_DEV:
2605                 dwc3_gadget_interrupt(dwc, &event->devt);
2606                 break;
2607         /* REVISIT what to do with Carkit and I2C events ? */
2608         default:
2609                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2610         }
2611 }
2612
2613 static irqreturn_t dwc3_process_event_buf(struct dwc3 *dwc, u32 buf)
2614 {
2615         struct dwc3_event_buffer *evt;
2616         irqreturn_t ret = IRQ_NONE;
2617         int left;
2618         u32 reg;
2619
2620         evt = dwc->ev_buffs[buf];
2621         left = evt->count;
2622
2623         if (!(evt->flags & DWC3_EVENT_PENDING))
2624                 return IRQ_NONE;
2625
2626         while (left > 0) {
2627                 union dwc3_event event;
2628
2629                 event.raw = *(u32 *) (evt->buf + evt->lpos);
2630
2631                 dwc3_process_event_entry(dwc, &event);
2632
2633                 /*
2634                  * FIXME we wrap around correctly to the next entry as
2635                  * almost all entries are 4 bytes in size. There is one
2636                  * entry which has 12 bytes which is a regular entry
2637                  * followed by 8 bytes data. ATM I don't know how
2638                  * things are organized if we get next to the a
2639                  * boundary so I worry about that once we try to handle
2640                  * that.
2641                  */
2642                 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2643                 left -= 4;
2644
2645                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(buf), 4);
2646         }
2647
2648         evt->count = 0;
2649         evt->flags &= ~DWC3_EVENT_PENDING;
2650         ret = IRQ_HANDLED;
2651
2652         /* Unmask interrupt */
2653         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(buf));
2654         reg &= ~DWC3_GEVNTSIZ_INTMASK;
2655         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(buf), reg);
2656
2657         return ret;
2658 }
2659
2660 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc)
2661 {
2662         struct dwc3 *dwc = _dwc;
2663         unsigned long flags;
2664         irqreturn_t ret = IRQ_NONE;
2665         int i;
2666
2667         spin_lock_irqsave(&dwc->lock, flags);
2668
2669         for (i = 0; i < dwc->num_event_buffers; i++)
2670                 ret |= dwc3_process_event_buf(dwc, i);
2671
2672         spin_unlock_irqrestore(&dwc->lock, flags);
2673
2674         return ret;
2675 }
2676
2677 static irqreturn_t dwc3_check_event_buf(struct dwc3 *dwc, u32 buf)
2678 {
2679         struct dwc3_event_buffer *evt;
2680         u32 count;
2681         u32 reg;
2682
2683         evt = dwc->ev_buffs[buf];
2684
2685         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(buf));
2686         count &= DWC3_GEVNTCOUNT_MASK;
2687         if (!count)
2688                 return IRQ_NONE;
2689
2690         evt->count = count;
2691         evt->flags |= DWC3_EVENT_PENDING;
2692
2693         /* Mask interrupt */
2694         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(buf));
2695         reg |= DWC3_GEVNTSIZ_INTMASK;
2696         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(buf), reg);
2697
2698         return IRQ_WAKE_THREAD;
2699 }
2700
2701 static irqreturn_t dwc3_interrupt(int irq, void *_dwc)
2702 {
2703         struct dwc3                     *dwc = _dwc;
2704         int                             i;
2705         irqreturn_t                     ret = IRQ_NONE;
2706
2707         for (i = 0; i < dwc->num_event_buffers; i++) {
2708                 irqreturn_t status;
2709
2710                 status = dwc3_check_event_buf(dwc, i);
2711                 if (status == IRQ_WAKE_THREAD)
2712                         ret = status;
2713         }
2714
2715         return ret;
2716 }
2717
2718 /**
2719  * dwc3_gadget_init - Initializes gadget related registers
2720  * @dwc: pointer to our controller context structure
2721  *
2722  * Returns 0 on success otherwise negative errno.
2723  */
2724 int dwc3_gadget_init(struct dwc3 *dwc)
2725 {
2726         int                                     ret;
2727
2728         dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2729                         &dwc->ctrl_req_addr, GFP_KERNEL);
2730         if (!dwc->ctrl_req) {
2731                 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2732                 ret = -ENOMEM;
2733                 goto err0;
2734         }
2735
2736         dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
2737                         &dwc->ep0_trb_addr, GFP_KERNEL);
2738         if (!dwc->ep0_trb) {
2739                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2740                 ret = -ENOMEM;
2741                 goto err1;
2742         }
2743
2744         dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL);
2745         if (!dwc->setup_buf) {
2746                 ret = -ENOMEM;
2747                 goto err2;
2748         }
2749
2750         dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
2751                         DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
2752                         GFP_KERNEL);
2753         if (!dwc->ep0_bounce) {
2754                 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2755                 ret = -ENOMEM;
2756                 goto err3;
2757         }
2758
2759         dwc->zlp_buf = kzalloc(DWC3_ZLP_BUF_SIZE, GFP_KERNEL);
2760         if (!dwc->zlp_buf) {
2761                 ret = -ENOMEM;
2762                 goto err4;
2763         }
2764
2765         dwc->gadget.ops                 = &dwc3_gadget_ops;
2766         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
2767         dwc->gadget.sg_supported        = true;
2768         dwc->gadget.name                = "dwc3-gadget";
2769         dwc->gadget.is_otg              = dwc->dr_mode == USB_DR_MODE_OTG;
2770
2771         /*
2772          * FIXME We might be setting max_speed to <SUPER, however versions
2773          * <2.20a of dwc3 have an issue with metastability (documented
2774          * elsewhere in this driver) which tells us we can't set max speed to
2775          * anything lower than SUPER.
2776          *
2777          * Because gadget.max_speed is only used by composite.c and function
2778          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
2779          * to happen so we avoid sending SuperSpeed Capability descriptor
2780          * together with our BOS descriptor as that could confuse host into
2781          * thinking we can handle super speed.
2782          *
2783          * Note that, in fact, we won't even support GetBOS requests when speed
2784          * is less than super speed because we don't have means, yet, to tell
2785          * composite.c that we are USB 2.0 + LPM ECN.
2786          */
2787         if (dwc->revision < DWC3_REVISION_220A)
2788                 dwc3_trace(trace_dwc3_gadget,
2789                                 "Changing max_speed on rev %08x\n",
2790                                 dwc->revision);
2791
2792         dwc->gadget.max_speed           = dwc->maximum_speed;
2793
2794         /*
2795          * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize
2796          * on ep out.
2797          */
2798         dwc->gadget.quirk_ep_out_aligned_size = true;
2799
2800         /*
2801          * REVISIT: Here we should clear all pending IRQs to be
2802          * sure we're starting from a well known location.
2803          */
2804
2805         ret = dwc3_gadget_init_endpoints(dwc);
2806         if (ret)
2807                 goto err5;
2808
2809         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2810         if (ret) {
2811                 dev_err(dwc->dev, "failed to register udc\n");
2812                 goto err5;
2813         }
2814
2815         return 0;
2816
2817 err5:
2818         kfree(dwc->zlp_buf);
2819
2820 err4:
2821         dwc3_gadget_free_endpoints(dwc);
2822         dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2823                         dwc->ep0_bounce, dwc->ep0_bounce_addr);
2824
2825 err3:
2826         kfree(dwc->setup_buf);
2827
2828 err2:
2829         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2830                         dwc->ep0_trb, dwc->ep0_trb_addr);
2831
2832 err1:
2833         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2834                         dwc->ctrl_req, dwc->ctrl_req_addr);
2835
2836 err0:
2837         return ret;
2838 }
2839
2840 /* -------------------------------------------------------------------------- */
2841
2842 void dwc3_gadget_exit(struct dwc3 *dwc)
2843 {
2844         usb_del_gadget_udc(&dwc->gadget);
2845
2846         dwc3_gadget_free_endpoints(dwc);
2847
2848         dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2849                         dwc->ep0_bounce, dwc->ep0_bounce_addr);
2850
2851         kfree(dwc->setup_buf);
2852         kfree(dwc->zlp_buf);
2853
2854         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2855                         dwc->ep0_trb, dwc->ep0_trb_addr);
2856
2857         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2858                         dwc->ctrl_req, dwc->ctrl_req_addr);
2859 }
2860
2861 int dwc3_gadget_suspend(struct dwc3 *dwc)
2862 {
2863         if (dwc->pullups_connected) {
2864                 dwc3_gadget_disable_irq(dwc);
2865                 dwc3_gadget_run_stop(dwc, true, true);
2866         }
2867
2868         __dwc3_gadget_ep_disable(dwc->eps[0]);
2869         __dwc3_gadget_ep_disable(dwc->eps[1]);
2870
2871         dwc->dcfg = dwc3_readl(dwc->regs, DWC3_DCFG);
2872
2873         return 0;
2874 }
2875
2876 int dwc3_gadget_resume(struct dwc3 *dwc)
2877 {
2878         struct dwc3_ep          *dep;
2879         int                     ret;
2880
2881         /* Start with SuperSpeed Default */
2882         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2883
2884         dep = dwc->eps[0];
2885         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
2886                         false);
2887         if (ret)
2888                 goto err0;
2889
2890         dep = dwc->eps[1];
2891         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
2892                         false);
2893         if (ret)
2894                 goto err1;
2895
2896         /* begin to receive SETUP packets */
2897         dwc->ep0state = EP0_SETUP_PHASE;
2898         dwc3_ep0_out_start(dwc);
2899
2900         dwc3_writel(dwc->regs, DWC3_DCFG, dwc->dcfg);
2901
2902         if (dwc->pullups_connected) {
2903                 dwc3_gadget_enable_irq(dwc);
2904                 dwc3_gadget_run_stop(dwc, true, false);
2905         }
2906
2907         return 0;
2908
2909 err1:
2910         __dwc3_gadget_ep_disable(dwc->eps[0]);
2911
2912 err0:
2913         return ret;
2914 }
2915
2916 static int dwc3_gadget_reinit(struct dwc3 *dwc)
2917 {
2918         u32                     hwparams4 = dwc->hwparams.hwparams4;
2919         u32                     reg;
2920         int                     ret;
2921         struct dwc3_ep          *dep = NULL;
2922
2923         reg = dwc3_readl(dwc->regs, DWC3_GSNPSID);
2924         /* This should read as U3 followed by revision number */
2925         if ((reg & DWC3_GSNPSID_MASK) == 0x55330000) {
2926                 /* Detected DWC_usb3 IP */
2927                 dwc->revision = reg;
2928         } else if ((reg & DWC3_GSNPSID_MASK) == 0x33310000) {
2929                 /* Detected DWC_usb31 IP */
2930                 dwc->revision = dwc3_readl(dwc->regs, DWC3_VER_NUMBER);
2931                 dwc->revision |= DWC3_REVISION_IS_DWC31;
2932         } else {
2933                 dev_err(dwc->dev, "this is not a DesignWare USB3 DRD Core\n");
2934                 ret = -ENODEV;
2935                 goto err0;
2936         }
2937
2938         /*
2939          * Write Linux Version Code to our GUID register so it's easy to figure
2940          * out which kernel version a bug was found.
2941          */
2942         dwc3_writel(dwc->regs, DWC3_GUID, LINUX_VERSION_CODE);
2943
2944         /* Handle USB2.0-only core configuration */
2945         if (DWC3_GHWPARAMS3_SSPHY_IFC(dwc->hwparams.hwparams3) ==
2946                         DWC3_GHWPARAMS3_SSPHY_IFC_DIS) {
2947                 if (dwc->maximum_speed == USB_SPEED_SUPER)
2948                         dwc->maximum_speed = USB_SPEED_HIGH;
2949         }
2950
2951         /* issue device SoftReset too */
2952         ret = dwc3_soft_reset(dwc);
2953         if (ret)
2954                 goto err0;
2955
2956         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2957         reg &= ~DWC3_GCTL_SCALEDOWN_MASK;
2958
2959         switch (DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1)) {
2960         case DWC3_GHWPARAMS1_EN_PWROPT_CLK:
2961                 /**
2962                  * WORKAROUND: DWC3 revisions between 2.10a and 2.50a have an
2963                  * issue which would cause xHCI compliance tests to fail.
2964                  *
2965                  * Because of that we cannot enable clock gating on such
2966                  * configurations.
2967                  *
2968                  * Refers to:
2969                  *
2970                  * STAR#9000588375: Clock Gating, SOF Issues when ref_clk-Based
2971                  * SOF/ITP Mode Used
2972                  */
2973                 if ((dwc->dr_mode == USB_DR_MODE_HOST ||
2974                      dwc->dr_mode == USB_DR_MODE_OTG) &&
2975                      (dwc->revision >= DWC3_REVISION_210A &&
2976                      dwc->revision <= DWC3_REVISION_250A))
2977                         reg |= DWC3_GCTL_DSBLCLKGTNG | DWC3_GCTL_SOFITPSYNC;
2978                 else
2979                         reg &= ~DWC3_GCTL_DSBLCLKGTNG;
2980                 break;
2981         case DWC3_GHWPARAMS1_EN_PWROPT_HIB:
2982                 /* enable hibernation here */
2983                 dwc->nr_scratch = DWC3_GHWPARAMS4_HIBER_SCRATCHBUFS(hwparams4);
2984
2985                 /*
2986                  * REVISIT Enabling this bit so that host-mode hibernation
2987                  * will work. Device-mode hibernation is not yet implemented.
2988                  */
2989                 reg |= DWC3_GCTL_GBLHIBERNATIONEN;
2990                 break;
2991         default:
2992                 dwc3_trace(trace_dwc3_core,
2993                            "No power optimization available\n");
2994         }
2995
2996         /* check if current dwc3 is on simulation board */
2997         if (dwc->hwparams.hwparams6 & DWC3_GHWPARAMS6_EN_FPGA) {
2998                 dwc3_trace(trace_dwc3_core,
2999                            "running on FPGA platform\n");
3000                 dwc->is_fpga = true;
3001         }
3002
3003         WARN_ONCE(dwc->disable_scramble_quirk && !dwc->is_fpga,
3004                   "disable_scramble cannot be used on non-FPGA builds\n");
3005
3006         if (dwc->disable_scramble_quirk && dwc->is_fpga)
3007                 reg |= DWC3_GCTL_DISSCRAMBLE;
3008         else
3009                 reg &= ~DWC3_GCTL_DISSCRAMBLE;
3010
3011         if (dwc->u2exit_lfps_quirk)
3012                 reg |= DWC3_GCTL_U2EXIT_LFPS;
3013
3014         /*
3015          * WORKAROUND: DWC3 revisions <1.90a have a bug
3016          * where the device can fail to connect at SuperSpeed
3017          * and falls back to high-speed mode which causes
3018          * the device to enter a Connect/Disconnect loop
3019          */
3020         if (dwc->revision < DWC3_REVISION_190A)
3021                 reg |= DWC3_GCTL_U2RSTECN;
3022         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
3023
3024         ret = dwc3_event_buffers_setup(dwc);
3025
3026         if (ret) {
3027                 dev_err(dwc->dev, "failed to setup event buffers\n");
3028                 goto err1;
3029         }
3030
3031         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3032         reg |= DWC3_DCFG_LPM_CAP;
3033         reg &= ~(DWC3_DCFG_SPEED_MASK);
3034
3035         /**
3036          * WORKAROUND: DWC3 revision < 2.20a have an issue
3037          * which would cause metastability state on Run/Stop
3038          * bit if we try to force the IP to USB2-only mode.
3039          *
3040          * Because of that, we cannot configure the IP to any
3041          * speed other than the SuperSpeed
3042          *
3043          * Refers to:
3044          *
3045          * STAR#9000525659: Clock Domain Crossing on DCTL in
3046          * USB 2.0 Mode
3047          */
3048         if (dwc->revision < DWC3_REVISION_220A) {
3049                 reg |= DWC3_DCFG_SUPERSPEED;
3050         } else {
3051                 switch (dwc->maximum_speed) {
3052                 case USB_SPEED_LOW:
3053                         reg |= DWC3_DSTS_LOWSPEED;
3054                         break;
3055                 case USB_SPEED_FULL:
3056                         reg |= DWC3_DSTS_FULLSPEED1;
3057                         break;
3058                 case USB_SPEED_HIGH:
3059                         reg |= DWC3_DSTS_HIGHSPEED;
3060                         break;
3061                 case USB_SPEED_SUPER:   /* FALLTHROUGH */
3062                 case USB_SPEED_UNKNOWN: /* FALTHROUGH */
3063                 default:
3064                         reg |= DWC3_DSTS_SUPERSPEED;
3065                 }
3066         }
3067         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3068
3069         /* Start with SuperSpeed Default */
3070         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3071
3072         dep = dwc->eps[0];
3073         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
3074                                       false);
3075         if (ret) {
3076                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3077                 goto err1;
3078         }
3079
3080         dep = dwc->eps[1];
3081         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
3082                                       false);
3083         if (ret) {
3084                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3085                 goto err2;
3086         }
3087
3088         /* begin to receive SETUP packets */
3089         dwc->ep0state = EP0_SETUP_PHASE;
3090         dwc3_ep0_out_start(dwc);
3091
3092         return 0;
3093 err2:
3094         __dwc3_gadget_ep_disable(dwc->eps[0]);
3095 err1:
3096         dwc3_event_buffers_cleanup(dwc);
3097 err0:
3098         return ret;
3099 }
3100
3101 int dwc3_gadget_restart(struct dwc3 *dwc, bool start)
3102 {
3103         struct dwc3_event_buffer        *evt;
3104         int                             ret = 0;
3105         int                             i;
3106         u32                             reg;
3107
3108         if (start) {
3109                 ret = dwc3_gadget_reinit(dwc);
3110                 if (ret < 0) {
3111                         dev_err(dwc->dev,
3112                                 "dwc3 gadget reinit error = %d\n", ret);
3113                         goto err;
3114                 }
3115
3116                 if (dwc->enabled) {
3117                         ret = dwc3_gadget_run_stop(dwc, start, false);
3118                         if (ret < 0) {
3119                                 dev_err(dwc->dev,
3120                                         "dwc3 gadget run stop err = %d\n", ret);
3121                                 goto err;
3122                         }
3123                 }
3124                 dwc3_gadget_enable_irq(dwc);
3125         } else {
3126                 /*
3127                  * Per databook, DEVCTRLHLT bit setting requires
3128                  * interrupts to be acknowledged. so acknowledge
3129                  * the events that are generated (by writing to
3130                  * GEVNTCOUNTn) first. And we also mask interrupts
3131                  * and clear SW states to avoid generating other
3132                  * interrupts after do gadget disconnnect operation.
3133                  */
3134                 dwc3_gadget_disable_irq(dwc);
3135
3136                 for (i = 0; i < dwc->num_event_buffers; i++) {
3137                         evt = dwc->ev_buffs[i];
3138                         evt->count = 0;
3139                         evt->flags &= ~DWC3_EVENT_PENDING;
3140                         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(i));
3141                         reg |= DWC3_GEVNTSIZ_INTMASK;
3142                         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(i), reg);
3143                         reg = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(i));
3144                         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(i), reg);
3145                 }
3146
3147                 /*
3148                  * DEVCTRLHLT bit sometimes does not get set
3149                  * even when GEVNTCOUNT is acked so do not
3150                  * care run stop function return value.
3151                  */
3152                 dwc3_gadget_run_stop(dwc, start, false);
3153
3154                 if (dwc->gadget.state != USB_STATE_NOTATTACHED)
3155                         dwc3_gadget_disconnect_interrupt(dwc);
3156
3157                 __dwc3_gadget_ep_disable(dwc->eps[0]);
3158                 __dwc3_gadget_ep_disable(dwc->eps[1]);
3159         }
3160
3161 err:
3162         return ret;
3163 }