drivers/usb/dwc3/gadget.c

   1 /**
   2  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   3  *
   4  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   5  *
   6  * Authors: Felipe Balbi <balbi@ti.com>,
   7  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   8  *
   9  * This program is free software: you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2  of
  11  * the License as published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  */
  18
  19 #include <linux/kernel.h>
  20 #include <linux/delay.h>
  21 #include <linux/slab.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/pm_runtime.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/io.h>
  27 #include <linux/list.h>
  28 #include <linux/dma-mapping.h>
  29
  30 #include <linux/usb/ch9.h>
  31 #include <linux/usb/gadget.h>
  32
  33 #include "debug.h"
  34 #include "core.h"
  35 #include "gadget.h"
  36 #include "io.h"
  37
  38 /**
  39  * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
  40  * @dwc: pointer to our context structure
  41  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  42  *
  43  * Caller should take care of locking. This function will
  44  * return 0 on success or -EINVAL if wrong Test Selector
  45  * is passed
  46  */
  47 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  48 {
  49         u32             reg;
  50
  51         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  52         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  53
  54         switch (mode) {
  55         case TEST_J:
  56         case TEST_K:
  57         case TEST_SE0_NAK:
  58         case TEST_PACKET:
  59         case TEST_FORCE_EN:
  60                 reg |= mode << 1;
  61                 break;
  62         default:
  63                 return -EINVAL;
  64         }
  65
  66         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  67
  68         return 0;
  69 }
  70
  71 /**
  72  * dwc3_gadget_get_link_state - Gets current state of USB Link
  73  * @dwc: pointer to our context structure
  74  *
  75  * Caller should take care of locking. This function will
  76  * return the link state on success (>= 0) or -ETIMEDOUT.
  77  */
  78 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  79 {
  80         u32             reg;
  81
  82         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  83
  84         return DWC3_DSTS_USBLNKST(reg);
  85 }
  86
  87 /**
  88  * dwc3_gadget_set_link_state - Sets USB Link to a particular State
  89  * @dwc: pointer to our context structure
  90  * @state: the state to put link into
  91  *
  92  * Caller should take care of locking. This function will
  93  * return 0 on success or -ETIMEDOUT.
  94  */
  95 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  96 {
  97         int             retries = 10000;
  98         u32             reg;
  99
 100         /*
 101          * Wait until device controller is ready. Only applies to 1.94a and
 102          * later RTL.
 103          */
 104         if (dwc->revision >= DWC3_REVISION_194A) {
 105                 while (--retries) {
 106                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 107                         if (reg & DWC3_DSTS_DCNRD)
 108                                 udelay(5);
 109                         else
 110                                 break;
 111                 }
 112
 113                 if (retries <= 0)
 114                         return -ETIMEDOUT;
 115         }
 116
 117         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 118         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 119
 120         /* set requested state */
 121         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 122         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 123
 124         /*
 125          * The following code is racy when called from dwc3_gadget_wakeup,
 126          * and is not needed, at least on newer versions
 127          */
 128         if (dwc->revision >= DWC3_REVISION_194A)
 129                 return 0;
 130
 131         /* wait for a change in DSTS */
 132         retries = 10000;
 133         while (--retries) {
 134                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 135
 136                 if (DWC3_DSTS_USBLNKST(reg) == state)
 137                         return 0;
 138
 139                 udelay(5);
 140         }
 141
 142         dwc3_trace(trace_dwc3_gadget,
 143                         "link state change request timed out");
 144
 145         return -ETIMEDOUT;
 146 }
 147
 148 /**
 149  * dwc3_ep_inc_trb() - Increment a TRB index.
 150  * @index - Pointer to the TRB index to increment.
 151  *
 152  * The index should never point to the link TRB. After incrementing,
 153  * if it is point to the link TRB, wrap around to the beginning. The
 154  * link TRB is always at the last TRB entry.
 155  */
 156 static void dwc3_ep_inc_trb(u8 *index)
 157 {
 158         (*index)++;
 159         if (*index == (DWC3_TRB_NUM - 1))
 160                 *index = 0;
 161 }
 162
 163 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 164 {
 165         dwc3_ep_inc_trb(&dep->trb_enqueue);
 166 }
 167
 168 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 169 {
 170         dwc3_ep_inc_trb(&dep->trb_dequeue);
 171 }
 172
 173 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 174                 int status)
 175 {
 176         struct dwc3                     *dwc = dep->dwc;
 177         int                             i;
 178
 179         if (req->started) {
 180                 i = 0;
 181                 do {
 182                         dwc3_ep_inc_deq(dep);
 183                 } while(++i < req->request.num_mapped_sgs);
 184                 req->started = false;
 185         }
 186         list_del(&req->list);
 187         req->trb = NULL;
 188
 189         if (req->request.status == -EINPROGRESS)
 190                 req->request.status = status;
 191
 192         if (dwc->ep0_bounced && dep->number == 0)
 193                 dwc->ep0_bounced = false;
 194         else
 195                 usb_gadget_unmap_request(&dwc->gadget, &req->request,
 196                                 req->direction);
 197
 198         trace_dwc3_gadget_giveback(req);
 199
 200         spin_unlock(&dwc->lock);
 201         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 202         spin_lock(&dwc->lock);
 203
 204         if (dep->number > 1)
 205                 pm_runtime_put(dwc->dev);
 206 }
 207
 208 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 209 {
 210         u32             timeout = 500;
 211         int             status = 0;
 212         int             ret = 0;
 213         u32             reg;
 214
 215         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 216         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 217
 218         do {
 219                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 220                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 221                         status = DWC3_DGCMD_STATUS(reg);
 222                         if (status)
 223                                 ret = -EINVAL;
 224                         break;
 225                 }
 226         } while (timeout--);
 227
 228         if (!timeout) {
 229                 ret = -ETIMEDOUT;
 230                 status = -ETIMEDOUT;
 231         }
 232
 233         trace_dwc3_gadget_generic_cmd(cmd, param, status);
 234
 235         return ret;
 236 }
 237
 238 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 239
 240 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 241                 struct dwc3_gadget_ep_cmd_params *params)
 242 {
 243         struct dwc3             *dwc = dep->dwc;
 244         u32                     timeout = 500;
 245         u32                     reg;
 246
 247         int                     cmd_status = 0;
 248         int                     susphy = false;
 249         int                     ret = -EINVAL;
 250
 251         /*
 252          * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
 253          * we're issuing an endpoint command, we must check if
 254          * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
 255          *
 256          * We will also set SUSPHY bit to what it was before returning as stated
 257          * by the same section on Synopsys databook.
 258          */
 259         if (dwc->gadget.speed <= USB_SPEED_HIGH) {
 260                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 261                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 262                         susphy = true;
 263                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 264                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 265                 }
 266         }
 267
 268         if (cmd == DWC3_DEPCMD_STARTTRANSFER) {
 269                 int             needs_wakeup;
 270
 271                 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
 272                                 dwc->link_state == DWC3_LINK_STATE_U2 ||
 273                                 dwc->link_state == DWC3_LINK_STATE_U3);
 274
 275                 if (unlikely(needs_wakeup)) {
 276                         ret = __dwc3_gadget_wakeup(dwc);
 277                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 278                                         ret);
 279                 }
 280         }
 281
 282         dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 283         dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 284         dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 285
 286         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd | DWC3_DEPCMD_CMDACT);
 287         do {
 288                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 289                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 290                         cmd_status = DWC3_DEPCMD_STATUS(reg);
 291
 292                         dwc3_trace(trace_dwc3_gadget,
 293                                         "Command Complete --> %d",
 294                                         cmd_status);
 295
 296                         switch (cmd_status) {
 297                         case 0:
 298                                 ret = 0;
 299                                 break;
 300                         case DEPEVT_TRANSFER_NO_RESOURCE:
 301                                 dwc3_trace(trace_dwc3_gadget, "no resource available");
 302                                 ret = -EINVAL;
 303                                 break;
 304                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 305                                 /*
 306                                  * SW issues START TRANSFER command to
 307                                  * isochronous ep with future frame interval. If
 308                                  * future interval time has already passed when
 309                                  * core receives the command, it will respond
 310                                  * with an error status of 'Bus Expiry'.
 311                                  *
 312                                  * Instead of always returning -EINVAL, let's
 313                                  * give a hint to the gadget driver that this is
 314                                  * the case by returning -EAGAIN.
 315                                  */
 316                                 dwc3_trace(trace_dwc3_gadget, "bus expiry");
 317                                 ret = -EAGAIN;
 318                                 break;
 319                         default:
 320                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 321                         }
 322
 323                         break;
 324                 }
 325         } while (--timeout);
 326
 327         if (timeout == 0) {
 328                 dwc3_trace(trace_dwc3_gadget,
 329                                 "Command Timed Out");
 330                 ret = -ETIMEDOUT;
 331                 cmd_status = -ETIMEDOUT;
 332         }
 333
 334         trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 335
 336         if (unlikely(susphy)) {
 337                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 338                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
 339                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 340         }
 341
 342         return ret;
 343 }
 344
 345 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 346 {
 347         struct dwc3 *dwc = dep->dwc;
 348         struct dwc3_gadget_ep_cmd_params params;
 349         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 350
 351         /*
 352          * As of core revision 2.60a the recommended programming model
 353          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 354          * command for IN endpoints. This is to prevent an issue where
 355          * some (non-compliant) hosts may not send ACK TPs for pending
 356          * IN transfers due to a mishandled error condition. Synopsys
 357          * STAR 9000614252.
 358          */
 359         if (dep->direction && (dwc->revision >= DWC3_REVISION_260A))
 360                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 361
 362         memset(&params, 0, sizeof(params));
 363
 364         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 365 }
 366
 367 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 368                 struct dwc3_trb *trb)
 369 {
 370         u32             offset = (char *) trb - (char *) dep->trb_pool;
 371
 372         return dep->trb_pool_dma + offset;
 373 }
 374
 375 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 376 {
 377         struct dwc3             *dwc = dep->dwc;
 378
 379         if (dep->trb_pool)
 380                 return 0;
 381
 382         dep->trb_pool = dma_alloc_coherent(dwc->dev,
 383                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 384                         &dep->trb_pool_dma, GFP_KERNEL);
 385         if (!dep->trb_pool) {
 386                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 387                                 dep->name);
 388                 return -ENOMEM;
 389         }
 390
 391         return 0;
 392 }
 393
 394 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 395 {
 396         struct dwc3             *dwc = dep->dwc;
 397
 398         dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 399                         dep->trb_pool, dep->trb_pool_dma);
 400
 401         dep->trb_pool = NULL;
 402         dep->trb_pool_dma = 0;
 403 }
 404
 405 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
 406
 407 /**
 408  * dwc3_gadget_start_config - Configure EP resources
 409  * @dwc: pointer to our controller context structure
 410  * @dep: endpoint that is being enabled
 411  *
 412  * The assignment of transfer resources cannot perfectly follow the
 413  * data book due to the fact that the controller driver does not have
 414  * all knowledge of the configuration in advance. It is given this
 415  * information piecemeal by the composite gadget framework after every
 416  * SET_CONFIGURATION and SET_INTERFACE. Trying to follow the databook
 417  * programming model in this scenario can cause errors. For two
 418  * reasons:
 419  *
 420  * 1) The databook says to do DEPSTARTCFG for every SET_CONFIGURATION
 421  * and SET_INTERFACE (8.1.5). This is incorrect in the scenario of
 422  * multiple interfaces.
 423  *
 424  * 2) The databook does not mention doing more DEPXFERCFG for new
 425  * endpoint on alt setting (8.1.6).
 426  *
 427  * The following simplified method is used instead:
 428  *
 429  * All hardware endpoints can be assigned a transfer resource and this
 430  * setting will stay persistent until either a core reset or
 431  * hibernation. So whenever we do a DEPSTARTCFG(0) we can go ahead and
 432  * do DEPXFERCFG for every hardware endpoint as well. We are
 433  * guaranteed that there are as many transfer resources as endpoints.
 434  *
 435  * This function is called for each endpoint when it is being enabled
 436  * but is triggered only when called for EP0-out, which always happens
 437  * first, and which should only happen in one of the above conditions.
 438  */
 439 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 440 {
 441         struct dwc3_gadget_ep_cmd_params params;
 442         u32                     cmd;
 443         int                     i;
 444         int                     ret;
 445
 446         if (dep->number)
 447                 return 0;
 448
 449         memset(&params, 0x00, sizeof(params));
 450         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 451
 452         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 453         if (ret)
 454                 return ret;
 455
 456         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 457                 struct dwc3_ep *dep = dwc->eps[i];
 458
 459                 if (!dep)
 460                         continue;
 461
 462                 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 463                 if (ret)
 464                         return ret;
 465         }
 466
 467         return 0;
 468 }
 469
 470 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 471                 const struct usb_endpoint_descriptor *desc,
 472                 const struct usb_ss_ep_comp_descriptor *comp_desc,
 473                 bool ignore, bool restore)
 474 {
 475         struct dwc3_gadget_ep_cmd_params params;
 476
 477         memset(&params, 0x00, sizeof(params));
 478
 479         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 480                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 481
 482         /* Burst size is only needed in SuperSpeed mode */
 483         if (dwc->gadget.speed >= USB_SPEED_SUPER) {
 484                 u32 burst = dep->endpoint.maxburst;
 485                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 486         }
 487
 488         if (ignore)
 489                 params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM;
 490
 491         if (restore) {
 492                 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
 493                 params.param2 |= dep->saved_state;
 494         }
 495
 496         params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
 497                 | DWC3_DEPCFG_XFER_NOT_READY_EN;
 498
 499         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 500                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 501                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 502                 dep->stream_capable = true;
 503         }
 504
 505         if (!usb_endpoint_xfer_control(desc))
 506                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 507
 508         /*
 509          * We are doing 1:1 mapping for endpoints, meaning
 510          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 511          * so on. We consider the direction bit as part of the physical
 512          * endpoint number. So USB endpoint 0x81 is 0x03.
 513          */
 514         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 515
 516         /*
 517          * We must use the lower 16 TX FIFOs even though
 518          * HW might have more
 519          */
 520         if (dep->direction)
 521                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 522
 523         if (desc->bInterval) {
 524                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 525                 dep->interval = 1 << (desc->bInterval - 1);
 526         }
 527
 528         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 529 }
 530
 531 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
 532 {
 533         struct dwc3_gadget_ep_cmd_params params;
 534
 535         memset(&params, 0x00, sizeof(params));
 536
 537         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 538
 539         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 540                         &params);
 541 }
 542
 543 /**
 544  * __dwc3_gadget_ep_enable - Initializes a HW endpoint
 545  * @dep: endpoint to be initialized
 546  * @desc: USB Endpoint Descriptor
 547  *
 548  * Caller should take care of locking
 549  */
 550 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 551                 const struct usb_endpoint_descriptor *desc,
 552                 const struct usb_ss_ep_comp_descriptor *comp_desc,
 553                 bool ignore, bool restore)
 554 {
 555         struct dwc3             *dwc = dep->dwc;
 556         u32                     reg;
 557         int                     ret;
 558
 559         dwc3_trace(trace_dwc3_gadget, "Enabling %s", dep->name);
 560
 561         if (!(dep->flags & DWC3_EP_ENABLED)) {
 562                 ret = dwc3_gadget_start_config(dwc, dep);
 563                 if (ret)
 564                         return ret;
 565         }
 566
 567         ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore,
 568                         restore);
 569         if (ret)
 570                 return ret;
 571
 572         if (!(dep->flags & DWC3_EP_ENABLED)) {
 573                 struct dwc3_trb *trb_st_hw;
 574                 struct dwc3_trb *trb_link;
 575
 576                 dep->endpoint.desc = desc;
 577                 dep->comp_desc = comp_desc;
 578                 dep->type = usb_endpoint_type(desc);
 579                 dep->flags |= DWC3_EP_ENABLED;
 580
 581                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 582                 reg |= DWC3_DALEPENA_EP(dep->number);
 583                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 584
 585                 if (usb_endpoint_xfer_control(desc))
 586                         return 0;
 587
 588                 /* Link TRB. The HWO bit is never reset */
 589                 trb_st_hw = &dep->trb_pool[0];
 590
 591                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 592                 memset(trb_link, 0, sizeof(*trb_link));
 593
 594                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 595                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 596                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 597                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 598         }
 599
 600         return 0;
 601 }
 602
 603 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
 604 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 605 {
 606         struct dwc3_request             *req;
 607
 608         if (!list_empty(&dep->started_list)) {
 609                 dwc3_stop_active_transfer(dwc, dep->number, true);
 610
 611                 /* - giveback all requests to gadget driver */
 612                 while (!list_empty(&dep->started_list)) {
 613                         req = next_request(&dep->started_list);
 614
 615                         dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 616                 }
 617         }
 618
 619         while (!list_empty(&dep->pending_list)) {
 620                 req = next_request(&dep->pending_list);
 621
 622                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 623         }
 624 }
 625
 626 /**
 627  * __dwc3_gadget_ep_disable - Disables a HW endpoint
 628  * @dep: the endpoint to disable
 629  *
 630  * This function also removes requests which are currently processed ny the
 631  * hardware and those which are not yet scheduled.
 632  * Caller should take care of locking.
 633  */
 634 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 635 {
 636         struct dwc3             *dwc = dep->dwc;
 637         u32                     reg;
 638
 639         dwc3_trace(trace_dwc3_gadget, "Disabling %s", dep->name);
 640
 641         dwc3_remove_requests(dwc, dep);
 642
 643         /* make sure HW endpoint isn't stalled */
 644         if (dep->flags & DWC3_EP_STALL)
 645                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 646
 647         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 648         reg &= ~DWC3_DALEPENA_EP(dep->number);
 649         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 650
 651         dep->stream_capable = false;
 652         dep->endpoint.desc = NULL;
 653         dep->comp_desc = NULL;
 654         dep->type = 0;
 655         dep->flags = 0;
 656
 657         return 0;
 658 }
 659
 660 /* -------------------------------------------------------------------------- */
 661
 662 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 663                 const struct usb_endpoint_descriptor *desc)
 664 {
 665         return -EINVAL;
 666 }
 667
 668 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 669 {
 670         return -EINVAL;
 671 }
 672
 673 /* -------------------------------------------------------------------------- */
 674
 675 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 676                 const struct usb_endpoint_descriptor *desc)
 677 {
 678         struct dwc3_ep                  *dep;
 679         struct dwc3                     *dwc;
 680         unsigned long                   flags;
 681         int                             ret;
 682
 683         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 684                 pr_debug("dwc3: invalid parameters\n");
 685                 return -EINVAL;
 686         }
 687
 688         if (!desc->wMaxPacketSize) {
 689                 pr_debug("dwc3: missing wMaxPacketSize\n");
 690                 return -EINVAL;
 691         }
 692
 693         dep = to_dwc3_ep(ep);
 694         dwc = dep->dwc;
 695
 696         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 697                                         "%s is already enabled\n",
 698                                         dep->name))
 699                 return 0;
 700
 701         spin_lock_irqsave(&dwc->lock, flags);
 702         ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false);
 703         spin_unlock_irqrestore(&dwc->lock, flags);
 704
 705         return ret;
 706 }
 707
 708 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 709 {
 710         struct dwc3_ep                  *dep;
 711         struct dwc3                     *dwc;
 712         unsigned long                   flags;
 713         int                             ret;
 714
 715         if (!ep) {
 716                 pr_debug("dwc3: invalid parameters\n");
 717                 return -EINVAL;
 718         }
 719
 720         dep = to_dwc3_ep(ep);
 721         dwc = dep->dwc;
 722
 723         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 724                                         "%s is already disabled\n",
 725                                         dep->name))
 726                 return 0;
 727
 728         spin_lock_irqsave(&dwc->lock, flags);
 729         ret = __dwc3_gadget_ep_disable(dep);
 730         spin_unlock_irqrestore(&dwc->lock, flags);
 731
 732         return ret;
 733 }
 734
 735 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 736         gfp_t gfp_flags)
 737 {
 738         struct dwc3_request             *req;
 739         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 740
 741         req = kzalloc(sizeof(*req), gfp_flags);
 742         if (!req)
 743                 return NULL;
 744
 745         req->epnum      = dep->number;
 746         req->dep        = dep;
 747
 748         trace_dwc3_alloc_request(req);
 749
 750         return &req->request;
 751 }
 752
 753 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 754                 struct usb_request *request)
 755 {
 756         struct dwc3_request             *req = to_dwc3_request(request);
 757
 758         trace_dwc3_free_request(req);
 759         kfree(req);
 760 }
 761
 762 /**
 763  * dwc3_prepare_one_trb - setup one TRB from one request
 764  * @dep: endpoint for which this request is prepared
 765  * @req: dwc3_request pointer
 766  */
 767 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 768                 struct dwc3_request *req, dma_addr_t dma,
 769                 unsigned length, unsigned last, unsigned chain, unsigned node)
 770 {
 771         struct dwc3_trb         *trb;
 772
 773         dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s%s",
 774                         dep->name, req, (unsigned long long) dma,
 775                         length, last ? " last" : "",
 776                         chain ? " chain" : "");
 777
 778
 779         trb = &dep->trb_pool[dep->trb_enqueue];
 780
 781         if (!req->trb) {
 782                 dwc3_gadget_move_started_request(req);
 783                 req->trb = trb;
 784                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
 785                 req->first_trb_index = dep->trb_enqueue;
 786         }
 787
 788         dwc3_ep_inc_enq(dep);
 789
 790         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 791         trb->bpl = lower_32_bits(dma);
 792         trb->bph = upper_32_bits(dma);
 793
 794         switch (usb_endpoint_type(dep->endpoint.desc)) {
 795         case USB_ENDPOINT_XFER_CONTROL:
 796                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 797                 break;
 798
 799         case USB_ENDPOINT_XFER_ISOC:
 800                 if (!node)
 801                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 802                 else
 803                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 804
 805                 /* always enable Interrupt on Missed ISOC */
 806                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 807                 break;
 808
 809         case USB_ENDPOINT_XFER_BULK:
 810         case USB_ENDPOINT_XFER_INT:
 811                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 812                 break;
 813         default:
 814                 /*
 815                  * This is only possible with faulty memory because we
 816                  * checked it already :)
 817                  */
 818                 BUG();
 819         }
 820
 821         /* always enable Continue on Short Packet */
 822         trb->ctrl |= DWC3_TRB_CTRL_CSP;
 823
 824         if (!req->request.no_interrupt && !chain)
 825                 trb->ctrl |= DWC3_TRB_CTRL_IOC | DWC3_TRB_CTRL_ISP_IMI;
 826
 827         if (last)
 828                 trb->ctrl |= DWC3_TRB_CTRL_LST;
 829
 830         if (chain)
 831                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
 832
 833         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
 834                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
 835
 836         trb->ctrl |= DWC3_TRB_CTRL_HWO;
 837
 838         trace_dwc3_prepare_trb(dep, trb);
 839 }
 840
 841 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 842 {
 843         struct dwc3_trb         *tmp;
 844
 845         /*
 846          * If enqueue & dequeue are equal than it is either full or empty.
 847          *
 848          * One way to know for sure is if the TRB right before us has HWO bit
 849          * set or not. If it has, then we're definitely full and can't fit any
 850          * more transfers in our ring.
 851          */
 852         if (dep->trb_enqueue == dep->trb_dequeue) {
 853                 /* If we're full, enqueue/dequeue are > 0 */
 854                 if (dep->trb_enqueue) {
 855                         tmp = &dep->trb_pool[dep->trb_enqueue - 1];
 856                         if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
 857                                 return 0;
 858                 }
 859
 860                 return DWC3_TRB_NUM - 1;
 861         }
 862
 863         return dep->trb_dequeue - dep->trb_enqueue;
 864 }
 865
 866 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
 867                 struct dwc3_request *req, unsigned int trbs_left)
 868 {
 869         struct usb_request *request = &req->request;
 870         struct scatterlist *sg = request->sg;
 871         struct scatterlist *s;
 872         unsigned int    last = false;
 873         unsigned int    length;
 874         dma_addr_t      dma;
 875         int             i;
 876
 877         for_each_sg(sg, s, request->num_mapped_sgs, i) {
 878                 unsigned chain = true;
 879
 880                 length = sg_dma_len(s);
 881                 dma = sg_dma_address(s);
 882
 883                 if (sg_is_last(s)) {
 884                         if (list_is_last(&req->list, &dep->pending_list))
 885                                 last = true;
 886
 887                         chain = false;
 888                 }
 889
 890                 if (!trbs_left)
 891                         last = true;
 892
 893                 if (last)
 894                         chain = false;
 895
 896                 dwc3_prepare_one_trb(dep, req, dma, length,
 897                                 last, chain, i);
 898
 899                 if (last)
 900                         break;
 901         }
 902 }
 903
 904 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
 905                 struct dwc3_request *req, unsigned int trbs_left)
 906 {
 907         unsigned int    last = false;
 908         unsigned int    length;
 909         dma_addr_t      dma;
 910
 911         dma = req->request.dma;
 912         length = req->request.length;
 913
 914         if (!trbs_left)
 915                 last = true;
 916
 917         /* Is this the last request? */
 918         if (list_is_last(&req->list, &dep->pending_list))
 919                 last = true;
 920
 921         dwc3_prepare_one_trb(dep, req, dma, length,
 922                         last, false, 0);
 923 }
 924
 925 /*
 926  * dwc3_prepare_trbs - setup TRBs from requests
 927  * @dep: endpoint for which requests are being prepared
 928  *
 929  * The function goes through the requests list and sets up TRBs for the
 930  * transfers. The function returns once there are no more TRBs available or
 931  * it runs out of requests.
 932  */
 933 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
 934 {
 935         struct dwc3_request     *req, *n;
 936         u32                     trbs_left;
 937
 938         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
 939
 940         trbs_left = dwc3_calc_trbs_left(dep);
 941
 942         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
 943                 if (req->request.num_mapped_sgs > 0)
 944                         dwc3_prepare_one_trb_sg(dep, req, trbs_left--);
 945                 else
 946                         dwc3_prepare_one_trb_linear(dep, req, trbs_left--);
 947
 948                 if (!trbs_left)
 949                         return;
 950         }
 951 }
 952
 953 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param)
 954 {
 955         struct dwc3_gadget_ep_cmd_params params;
 956         struct dwc3_request             *req;
 957         struct dwc3                     *dwc = dep->dwc;
 958         int                             starting;
 959         int                             ret;
 960         u32                             cmd;
 961
 962         starting = !(dep->flags & DWC3_EP_BUSY);
 963
 964         dwc3_prepare_trbs(dep);
 965         req = next_request(&dep->started_list);
 966         if (!req) {
 967                 dep->flags |= DWC3_EP_PENDING_REQUEST;
 968                 return 0;
 969         }
 970
 971         memset(&params, 0, sizeof(params));
 972
 973         if (starting) {
 974                 params.param0 = upper_32_bits(req->trb_dma);
 975                 params.param1 = lower_32_bits(req->trb_dma);
 976                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 977         } else {
 978                 cmd = DWC3_DEPCMD_UPDATETRANSFER;
 979         }
 980
 981         cmd |= DWC3_DEPCMD_PARAM(cmd_param);
 982         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 983         if (ret < 0) {
 984                 /*
 985                  * FIXME we need to iterate over the list of requests
 986                  * here and stop, unmap, free and del each of the linked
 987                  * requests instead of what we do now.
 988                  */
 989                 usb_gadget_unmap_request(&dwc->gadget, &req->request,
 990                                 req->direction);
 991                 list_del(&req->list);
 992                 return ret;
 993         }
 994
 995         dep->flags |= DWC3_EP_BUSY;
 996
 997         if (starting) {
 998                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
 999                 WARN_ON_ONCE(!dep->resource_index);
1000         }
1001
1002         return 0;
1003 }
1004
1005 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1006                 struct dwc3_ep *dep, u32 cur_uf)
1007 {
1008         u32 uf;
1009
1010         if (list_empty(&dep->pending_list)) {
1011                 dwc3_trace(trace_dwc3_gadget,
1012                                 "ISOC ep %s run out for requests",
1013                                 dep->name);
1014                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1015                 return;
1016         }
1017
1018         /* 4 micro frames in the future */
1019         uf = cur_uf + dep->interval * 4;
1020
1021         __dwc3_gadget_kick_transfer(dep, uf);
1022 }
1023
1024 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1025                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1026 {
1027         u32 cur_uf, mask;
1028
1029         mask = ~(dep->interval - 1);
1030         cur_uf = event->parameters & mask;
1031
1032         __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1033 }
1034
1035 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1036 {
1037         struct dwc3             *dwc = dep->dwc;
1038         int                     ret;
1039
1040         if (!dep->endpoint.desc) {
1041                 dwc3_trace(trace_dwc3_gadget,
1042                                 "trying to queue request %p to disabled %s\n",
1043                                 &req->request, dep->endpoint.name);
1044                 return -ESHUTDOWN;
1045         }
1046
1047         if (WARN(req->dep != dep, "request %p belongs to '%s'\n",
1048                                 &req->request, req->dep->name)) {
1049                 dwc3_trace(trace_dwc3_gadget, "request %p belongs to '%s'\n",
1050                                 &req->request, req->dep->name);
1051                 return -EINVAL;
1052         }
1053
1054         pm_runtime_get(dwc->dev);
1055
1056         req->request.actual     = 0;
1057         req->request.status     = -EINPROGRESS;
1058         req->direction          = dep->direction;
1059         req->epnum              = dep->number;
1060
1061         trace_dwc3_ep_queue(req);
1062
1063         /*
1064          * Per databook, the total size of buffer must be a multiple
1065          * of MaxPacketSize for OUT endpoints. And MaxPacketSize is
1066          * configed for endpoints in dwc3_gadget_set_ep_config(),
1067          * set to usb_endpoint_descriptor->wMaxPacketSize.
1068          */
1069         if (dep->direction == 0 &&
1070             req->request.length % dep->endpoint.desc->wMaxPacketSize)
1071                 req->request.length = roundup(req->request.length,
1072                                         dep->endpoint.desc->wMaxPacketSize);
1073
1074         /*
1075          * We only add to our list of requests now and
1076          * start consuming the list once we get XferNotReady
1077          * IRQ.
1078          *
1079          * That way, we avoid doing anything that we don't need
1080          * to do now and defer it until the point we receive a
1081          * particular token from the Host side.
1082          *
1083          * This will also avoid Host cancelling URBs due to too
1084          * many NAKs.
1085          */
1086         ret = usb_gadget_map_request(&dwc->gadget, &req->request,
1087                         dep->direction);
1088         if (ret)
1089                 return ret;
1090
1091         list_add_tail(&req->list, &dep->pending_list);
1092
1093         /*
1094          * If there are no pending requests and the endpoint isn't already
1095          * busy, we will just start the request straight away.
1096          *
1097          * This will save one IRQ (XFER_NOT_READY) and possibly make it a
1098          * little bit faster.
1099          */
1100         if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1101                         !usb_endpoint_xfer_int(dep->endpoint.desc) &&
1102                         !(dep->flags & DWC3_EP_BUSY)) {
1103                 ret = __dwc3_gadget_kick_transfer(dep, 0);
1104                 goto out;
1105         }
1106
1107         /*
1108          * There are a few special cases:
1109          *
1110          * 1. XferNotReady with empty list of requests. We need to kick the
1111          *    transfer here in that situation, otherwise we will be NAKing
1112          *    forever. If we get XferNotReady before gadget driver has a
1113          *    chance to queue a request, we will ACK the IRQ but won't be
1114          *    able to receive the data until the next request is queued.
1115          *    The following code is handling exactly that.
1116          *
1117          */
1118         if (dep->flags & DWC3_EP_PENDING_REQUEST) {
1119                 /*
1120                  * If xfernotready is already elapsed and it is a case
1121                  * of isoc transfer, then issue END TRANSFER, so that
1122                  * you can receive xfernotready again and can have
1123                  * notion of current microframe.
1124                  */
1125                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1126                         if (list_empty(&dep->started_list)) {
1127                                 dwc3_stop_active_transfer(dwc, dep->number, true);
1128                                 dep->flags = DWC3_EP_ENABLED;
1129                         }
1130                         return 0;
1131                 }
1132
1133                 ret = __dwc3_gadget_kick_transfer(dep, 0);
1134                 if (!ret)
1135                         dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1136
1137                 goto out;
1138         }
1139
1140         /*
1141          * 2. XferInProgress on Isoc EP with an active transfer. We need to
1142          *    kick the transfer here after queuing a request, otherwise the
1143          *    core may not see the modified TRB(s).
1144          */
1145         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1146                         (dep->flags & DWC3_EP_BUSY) &&
1147                         !(dep->flags & DWC3_EP_MISSED_ISOC)) {
1148                 WARN_ON_ONCE(!dep->resource_index);
1149                 ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index);
1150                 goto out;
1151         }
1152
1153         /*
1154          * 4. Stream Capable Bulk Endpoints. We need to start the transfer
1155          * right away, otherwise host will not know we have streams to be
1156          * handled.
1157          */
1158         if (dep->stream_capable)
1159                 ret = __dwc3_gadget_kick_transfer(dep, 0);
1160
1161 out:
1162         if (ret && ret != -EBUSY)
1163                 dwc3_trace(trace_dwc3_gadget,
1164                                 "%s: failed to kick transfers\n",
1165                                 dep->name);
1166         if (ret == -EBUSY)
1167                 ret = 0;
1168
1169         return ret;
1170 }
1171
1172 static void __dwc3_gadget_ep_zlp_complete(struct usb_ep *ep,
1173                 struct usb_request *request)
1174 {
1175         dwc3_gadget_ep_free_request(ep, request);
1176 }
1177
1178 static int __dwc3_gadget_ep_queue_zlp(struct dwc3 *dwc, struct dwc3_ep *dep)
1179 {
1180         struct dwc3_request             *req;
1181         struct usb_request              *request;
1182         struct usb_ep                   *ep = &dep->endpoint;
1183
1184         dwc3_trace(trace_dwc3_gadget, "queueing ZLP\n");
1185         request = dwc3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
1186         if (!request)
1187                 return -ENOMEM;
1188
1189         request->length = 0;
1190         request->buf = dwc->zlp_buf;
1191         request->complete = __dwc3_gadget_ep_zlp_complete;
1192
1193         req = to_dwc3_request(request);
1194
1195         return __dwc3_gadget_ep_queue(dep, req);
1196 }
1197
1198 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1199         gfp_t gfp_flags)
1200 {
1201         struct dwc3_request             *req = to_dwc3_request(request);
1202         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1203         struct dwc3                     *dwc = dep->dwc;
1204
1205         unsigned long                   flags;
1206
1207         int                             ret;
1208
1209         spin_lock_irqsave(&dwc->lock, flags);
1210         ret = __dwc3_gadget_ep_queue(dep, req);
1211
1212         /*
1213          * Okay, here's the thing, if gadget driver has requested for a ZLP by
1214          * setting request->zero, instead of doing magic, we will just queue an
1215          * extra usb_request ourselves so that it gets handled the same way as
1216          * any other request.
1217          */
1218         if (ret == 0 && request->zero && request->length &&
1219             (request->length % ep->desc->wMaxPacketSize == 0))
1220                 ret = __dwc3_gadget_ep_queue_zlp(dwc, dep);
1221
1222         spin_unlock_irqrestore(&dwc->lock, flags);
1223
1224         return ret;
1225 }
1226
1227 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1228                 struct usb_request *request)
1229 {
1230         struct dwc3_request             *req = to_dwc3_request(request);
1231         struct dwc3_request             *r = NULL;
1232
1233         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1234         struct dwc3                     *dwc = dep->dwc;
1235
1236         unsigned long                   flags;
1237         int                             ret = 0;
1238
1239         trace_dwc3_ep_dequeue(req);
1240
1241         spin_lock_irqsave(&dwc->lock, flags);
1242
1243         list_for_each_entry(r, &dep->pending_list, list) {
1244                 if (r == req)
1245                         break;
1246         }
1247
1248         if (r != req) {
1249                 list_for_each_entry(r, &dep->started_list, list) {
1250                         if (r == req)
1251                                 break;
1252                 }
1253                 if (r == req) {
1254                         /* wait until it is processed */
1255                         dwc3_stop_active_transfer(dwc, dep->number, true);
1256                         goto out1;
1257                 }
1258                 dev_err(dwc->dev, "request %p was not queued to %s\n",
1259                                 request, ep->name);
1260                 ret = -EINVAL;
1261                 goto out0;
1262         }
1263
1264 out1:
1265         /* giveback the request */
1266         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1267
1268 out0:
1269         spin_unlock_irqrestore(&dwc->lock, flags);
1270
1271         return ret;
1272 }
1273
1274 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1275 {
1276         struct dwc3_gadget_ep_cmd_params        params;
1277         struct dwc3                             *dwc = dep->dwc;
1278         int                                     ret;
1279
1280         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1281                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1282                 return -EINVAL;
1283         }
1284
1285         memset(&params, 0x00, sizeof(params));
1286
1287         if (value) {
1288                 if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) ||
1289                                 (!list_empty(&dep->started_list) ||
1290                                  !list_empty(&dep->pending_list)))) {
1291                         dwc3_trace(trace_dwc3_gadget,
1292                                         "%s: pending request, cannot halt",
1293                                         dep->name);
1294                         return -EAGAIN;
1295                 }
1296
1297                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1298                                 &params);
1299                 if (ret)
1300                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1301                                         dep->name);
1302                 else
1303                         dep->flags |= DWC3_EP_STALL;
1304         } else {
1305
1306                 ret = dwc3_send_clear_stall_ep_cmd(dep);
1307                 if (ret)
1308                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1309                                         dep->name);
1310                 else
1311                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1312         }
1313
1314         return ret;
1315 }
1316
1317 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1318 {
1319         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1320         struct dwc3                     *dwc = dep->dwc;
1321
1322         unsigned long                   flags;
1323
1324         int                             ret;
1325
1326         spin_lock_irqsave(&dwc->lock, flags);
1327         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1328         spin_unlock_irqrestore(&dwc->lock, flags);
1329
1330         return ret;
1331 }
1332
1333 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1334 {
1335         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1336         struct dwc3                     *dwc = dep->dwc;
1337         unsigned long                   flags;
1338         int                             ret;
1339
1340         spin_lock_irqsave(&dwc->lock, flags);
1341         dep->flags |= DWC3_EP_WEDGE;
1342
1343         if (dep->number == 0 || dep->number == 1)
1344                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1345         else
1346                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1347         spin_unlock_irqrestore(&dwc->lock, flags);
1348
1349         return ret;
1350 }
1351
1352 /* -------------------------------------------------------------------------- */
1353
1354 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1355         .bLength        = USB_DT_ENDPOINT_SIZE,
1356         .bDescriptorType = USB_DT_ENDPOINT,
1357         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1358 };
1359
1360 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1361         .enable         = dwc3_gadget_ep0_enable,
1362         .disable        = dwc3_gadget_ep0_disable,
1363         .alloc_request  = dwc3_gadget_ep_alloc_request,
1364         .free_request   = dwc3_gadget_ep_free_request,
1365         .queue          = dwc3_gadget_ep0_queue,
1366         .dequeue        = dwc3_gadget_ep_dequeue,
1367         .set_halt       = dwc3_gadget_ep0_set_halt,
1368         .set_wedge      = dwc3_gadget_ep_set_wedge,
1369 };
1370
1371 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1372         .enable         = dwc3_gadget_ep_enable,
1373         .disable        = dwc3_gadget_ep_disable,
1374         .alloc_request  = dwc3_gadget_ep_alloc_request,
1375         .free_request   = dwc3_gadget_ep_free_request,
1376         .queue          = dwc3_gadget_ep_queue,
1377         .dequeue        = dwc3_gadget_ep_dequeue,
1378         .set_halt       = dwc3_gadget_ep_set_halt,
1379         .set_wedge      = dwc3_gadget_ep_set_wedge,
1380 };
1381
1382 /* -------------------------------------------------------------------------- */
1383
1384 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1385 {
1386         struct dwc3             *dwc = gadget_to_dwc(g);
1387         u32                     reg;
1388
1389         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1390         return DWC3_DSTS_SOFFN(reg);
1391 }
1392
1393 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1394 {
1395         unsigned long           timeout;
1396
1397         int                     ret;
1398         u32                     reg;
1399
1400         u8                      link_state;
1401         u8                      speed;
1402
1403         /*
1404          * According to the Databook Remote wakeup request should
1405          * be issued only when the device is in early suspend state.
1406          *
1407          * We can check that via USB Link State bits in DSTS register.
1408          */
1409         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1410
1411         speed = reg & DWC3_DSTS_CONNECTSPD;
1412         if (speed == DWC3_DSTS_SUPERSPEED) {
1413                 dwc3_trace(trace_dwc3_gadget, "no wakeup on SuperSpeed\n");
1414                 return 0;
1415         }
1416
1417         link_state = DWC3_DSTS_USBLNKST(reg);
1418
1419         switch (link_state) {
1420         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1421         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1422                 break;
1423         default:
1424                 dwc3_trace(trace_dwc3_gadget,
1425                                 "can't wakeup from '%s'\n",
1426                                 dwc3_gadget_link_string(link_state));
1427                 return -EINVAL;
1428         }
1429
1430         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1431         if (ret < 0) {
1432                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1433                 return ret;
1434         }
1435
1436         /* Recent versions do this automatically */
1437         if (dwc->revision < DWC3_REVISION_194A) {
1438                 /* write zeroes to Link Change Request */
1439                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1440                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1441                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1442         }
1443
1444         /* poll until Link State changes to ON */
1445         timeout = jiffies + msecs_to_jiffies(100);
1446
1447         while (!time_after(jiffies, timeout)) {
1448                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1449
1450                 /* in HS, means ON */
1451                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1452                         break;
1453         }
1454
1455         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1456                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1457                 return -EINVAL;
1458         }
1459
1460         return 0;
1461 }
1462
1463 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1464 {
1465         struct dwc3             *dwc = gadget_to_dwc(g);
1466         unsigned long           flags;
1467         int                     ret;
1468
1469         spin_lock_irqsave(&dwc->lock, flags);
1470         ret = __dwc3_gadget_wakeup(dwc);
1471         spin_unlock_irqrestore(&dwc->lock, flags);
1472
1473         return ret;
1474 }
1475
1476 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1477                 int is_selfpowered)
1478 {
1479         struct dwc3             *dwc = gadget_to_dwc(g);
1480         unsigned long           flags;
1481
1482         spin_lock_irqsave(&dwc->lock, flags);
1483         g->is_selfpowered = !!is_selfpowered;
1484         spin_unlock_irqrestore(&dwc->lock, flags);
1485
1486         return 0;
1487 }
1488
1489 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1490 {
1491         u32                     reg;
1492         u32                     timeout = 500;
1493
1494         if (pm_runtime_suspended(dwc->dev))
1495                 return 0;
1496
1497         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1498         if (is_on) {
1499                 if (dwc->revision <= DWC3_REVISION_187A) {
1500                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1501                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1502                 }
1503
1504                 if (dwc->revision >= DWC3_REVISION_194A)
1505                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1506                 reg |= DWC3_DCTL_RUN_STOP;
1507
1508                 if (dwc->has_hibernation)
1509                         reg |= DWC3_DCTL_KEEP_CONNECT;
1510
1511                 dwc->pullups_connected = true;
1512         } else {
1513                 reg &= ~DWC3_DCTL_RUN_STOP;
1514
1515                 if (dwc->has_hibernation && !suspend)
1516                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1517
1518                 dwc->pullups_connected = false;
1519         }
1520
1521         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1522
1523         do {
1524                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1525                 if (is_on) {
1526                         if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1527                                 break;
1528                 } else {
1529                         if (reg & DWC3_DSTS_DEVCTRLHLT)
1530                                 break;
1531                 }
1532                 timeout--;
1533                 if (!timeout)
1534                         return -ETIMEDOUT;
1535                 udelay(1);
1536         } while (1);
1537
1538         dwc3_trace(trace_dwc3_gadget, "gadget %s data soft-%s",
1539                         dwc->gadget_driver
1540                         ? dwc->gadget_driver->function : "no-function",
1541                         is_on ? "connect" : "disconnect");
1542
1543         return 0;
1544 }
1545
1546 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1547 {
1548         struct dwc3             *dwc = gadget_to_dwc(g);
1549         unsigned long           flags;
1550         int                     ret;
1551
1552         is_on = !!is_on;
1553
1554         spin_lock_irqsave(&dwc->lock, flags);
1555         ret = dwc3_gadget_run_stop(dwc, is_on, false);
1556         spin_unlock_irqrestore(&dwc->lock, flags);
1557
1558         return ret;
1559 }
1560
1561 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1562 {
1563         u32                     reg;
1564
1565         /* Enable all but Start and End of Frame IRQs */
1566         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1567                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1568                         DWC3_DEVTEN_CMDCMPLTEN |
1569                         DWC3_DEVTEN_ERRTICERREN |
1570                         DWC3_DEVTEN_WKUPEVTEN |
1571                         DWC3_DEVTEN_ULSTCNGEN |
1572                         DWC3_DEVTEN_CONNECTDONEEN |
1573                         DWC3_DEVTEN_USBRSTEN |
1574                         DWC3_DEVTEN_DISCONNEVTEN);
1575
1576         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1577 }
1578
1579 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1580 {
1581         /* mask all interrupts */
1582         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1583 }
1584
1585 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1586 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1587
1588 /**
1589  * dwc3_gadget_setup_nump - Calculate and initialize NUMP field of DCFG
1590  * dwc: pointer to our context structure
1591  *
1592  * The following looks like complex but it's actually very simple. In order to
1593  * calculate the number of packets we can burst at once on OUT transfers, we're
1594  * gonna use RxFIFO size.
1595  *
1596  * To calculate RxFIFO size we need two numbers:
1597  * MDWIDTH = size, in bits, of the internal memory bus
1598  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1599  *
1600  * Given these two numbers, the formula is simple:
1601  *
1602  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1603  *
1604  * 24 bytes is for 3x SETUP packets
1605  * 16 bytes is a clock domain crossing tolerance
1606  *
1607  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1608  */
1609 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1610 {
1611         u32 ram2_depth;
1612         u32 mdwidth;
1613         u32 nump;
1614         u32 reg;
1615
1616         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1617         mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1618
1619         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1620         nump = min_t(u32, nump, 16);
1621
1622         /* update NumP */
1623         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1624         reg &= ~DWC3_DCFG_NUMP_MASK;
1625         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1626         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1627 }
1628
1629 static int __dwc3_gadget_start(struct dwc3 *dwc)
1630 {
1631         struct dwc3_ep          *dep;
1632         int                     ret = 0;
1633         u32                     reg;
1634
1635         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1636         reg &= ~(DWC3_DCFG_SPEED_MASK);
1637
1638         /**
1639          * WORKAROUND: DWC3 revision < 2.20a have an issue
1640          * which would cause metastability state on Run/Stop
1641          * bit if we try to force the IP to USB2-only mode.
1642          *
1643          * Because of that, we cannot configure the IP to any
1644          * speed other than the SuperSpeed
1645          *
1646          * Refers to:
1647          *
1648          * STAR#9000525659: Clock Domain Crossing on DCTL in
1649          * USB 2.0 Mode
1650          */
1651         if (dwc->revision < DWC3_REVISION_220A) {
1652                 reg |= DWC3_DCFG_SUPERSPEED;
1653         } else {
1654                 switch (dwc->maximum_speed) {
1655                 case USB_SPEED_LOW:
1656                         reg |= DWC3_DSTS_LOWSPEED;
1657                         break;
1658                 case USB_SPEED_FULL:
1659                         reg |= DWC3_DSTS_FULLSPEED1;
1660                         break;
1661                 case USB_SPEED_HIGH:
1662                         reg |= DWC3_DSTS_HIGHSPEED;
1663                         break;
1664                 case USB_SPEED_SUPER:   /* FALLTHROUGH */
1665                 case USB_SPEED_UNKNOWN: /* FALTHROUGH */
1666                 default:
1667                         reg |= DWC3_DSTS_SUPERSPEED;
1668                 }
1669         }
1670         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1671
1672         /*
1673          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1674          * field instead of letting dwc3 itself calculate that automatically.
1675          *
1676          * This way, we maximize the chances that we'll be able to get several
1677          * bursts of data without going through any sort of endpoint throttling.
1678          */
1679         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1680         reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1681         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1682
1683         dwc3_gadget_setup_nump(dwc);
1684
1685         /* Start with SuperSpeed Default */
1686         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1687
1688         dep = dwc->eps[0];
1689         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1690                         false);
1691         if (ret) {
1692                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1693                 goto err0;
1694         }
1695
1696         dep = dwc->eps[1];
1697         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1698                         false);
1699         if (ret) {
1700                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1701                 goto err1;
1702         }
1703
1704         /* begin to receive SETUP packets */
1705         dwc->ep0state = EP0_SETUP_PHASE;
1706         dwc3_ep0_out_start(dwc);
1707
1708         dwc3_gadget_enable_irq(dwc);
1709
1710         return 0;
1711
1712 err1:
1713         __dwc3_gadget_ep_disable(dwc->eps[0]);
1714
1715 err0:
1716         return ret;
1717 }
1718
1719 static int dwc3_gadget_start(struct usb_gadget *g,
1720                 struct usb_gadget_driver *driver)
1721 {
1722         struct dwc3             *dwc = gadget_to_dwc(g);
1723         unsigned long           flags;
1724         int                     ret = 0;
1725         int                     irq;
1726
1727         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1728         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1729                         IRQF_SHARED, "dwc3", dwc->ev_buf);
1730         if (ret) {
1731                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1732                                 irq, ret);
1733                 goto err0;
1734         }
1735         dwc->irq_gadget = irq;
1736
1737         spin_lock_irqsave(&dwc->lock, flags);
1738         if (dwc->gadget_driver) {
1739                 dev_err(dwc->dev, "%s is already bound to %s\n",
1740                                 dwc->gadget.name,
1741                                 dwc->gadget_driver->driver.name);
1742                 ret = -EBUSY;
1743                 goto err1;
1744         }
1745
1746         dwc->gadget_driver      = driver;
1747
1748         if (pm_runtime_active(dwc->dev))
1749                 __dwc3_gadget_start(dwc);
1750
1751         spin_unlock_irqrestore(&dwc->lock, flags);
1752
1753         return 0;
1754
1755 err1:
1756         spin_unlock_irqrestore(&dwc->lock, flags);
1757         free_irq(irq, dwc);
1758
1759 err0:
1760         return ret;
1761 }
1762
1763 static void __dwc3_gadget_stop(struct dwc3 *dwc)
1764 {
1765         dwc3_gadget_disable_irq(dwc);
1766         __dwc3_gadget_ep_disable(dwc->eps[0]);
1767         __dwc3_gadget_ep_disable(dwc->eps[1]);
1768 }
1769
1770 static int dwc3_gadget_stop(struct usb_gadget *g)
1771 {
1772         struct dwc3             *dwc = gadget_to_dwc(g);
1773         unsigned long           flags;
1774
1775         spin_lock_irqsave(&dwc->lock, flags);
1776         __dwc3_gadget_stop(dwc);
1777         dwc->gadget_driver      = NULL;
1778         spin_unlock_irqrestore(&dwc->lock, flags);
1779
1780         free_irq(dwc->irq_gadget, dwc->ev_buf);
1781
1782         return 0;
1783 }
1784
1785 static const struct usb_gadget_ops dwc3_gadget_ops = {
1786         .get_frame              = dwc3_gadget_get_frame,
1787         .wakeup                 = dwc3_gadget_wakeup,
1788         .set_selfpowered        = dwc3_gadget_set_selfpowered,
1789         .pullup                 = dwc3_gadget_pullup,
1790         .udc_start              = dwc3_gadget_start,
1791         .udc_stop               = dwc3_gadget_stop,
1792 };
1793
1794 /* -------------------------------------------------------------------------- */
1795
1796 static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
1797                 u8 num, u32 direction)
1798 {
1799         struct dwc3_ep                  *dep;
1800         u8                              i;
1801
1802         for (i = 0; i < num; i++) {
1803                 u8 epnum = (i << 1) | (direction ? 1 : 0);
1804
1805                 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1806                 if (!dep)
1807                         return -ENOMEM;
1808
1809                 dep->dwc = dwc;
1810                 dep->number = epnum;
1811                 dep->direction = !!direction;
1812                 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
1813                 dwc->eps[epnum] = dep;
1814
1815                 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1816                                 (epnum & 1) ? "in" : "out");
1817
1818                 dep->endpoint.name = dep->name;
1819                 spin_lock_init(&dep->lock);
1820
1821                 dwc3_trace(trace_dwc3_gadget, "initializing %s", dep->name);
1822
1823                 if (epnum == 0 || epnum == 1) {
1824                         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
1825                         dep->endpoint.maxburst = 1;
1826                         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1827                         if (!epnum)
1828                                 dwc->gadget.ep0 = &dep->endpoint;
1829                 } else {
1830                         int             ret;
1831
1832                         usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
1833                         dep->endpoint.max_streams = 15;
1834                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
1835                         list_add_tail(&dep->endpoint.ep_list,
1836                                         &dwc->gadget.ep_list);
1837
1838                         ret = dwc3_alloc_trb_pool(dep);
1839                         if (ret)
1840                                 return ret;
1841                 }
1842
1843                 if (epnum == 0 || epnum == 1) {
1844                         dep->endpoint.caps.type_control = true;
1845                 } else {
1846                         dep->endpoint.caps.type_iso = true;
1847                         dep->endpoint.caps.type_bulk = true;
1848                         dep->endpoint.caps.type_int = true;
1849                 }
1850
1851                 dep->endpoint.caps.dir_in = !!direction;
1852                 dep->endpoint.caps.dir_out = !direction;
1853
1854                 INIT_LIST_HEAD(&dep->pending_list);
1855                 INIT_LIST_HEAD(&dep->started_list);
1856         }
1857
1858         return 0;
1859 }
1860
1861 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1862 {
1863         int                             ret;
1864
1865         INIT_LIST_HEAD(&dwc->gadget.ep_list);
1866
1867         ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
1868         if (ret < 0) {
1869                 dwc3_trace(trace_dwc3_gadget,
1870                                 "failed to allocate OUT endpoints");
1871                 return ret;
1872         }
1873
1874         ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
1875         if (ret < 0) {
1876                 dwc3_trace(trace_dwc3_gadget,
1877                                 "failed to allocate IN endpoints");
1878                 return ret;
1879         }
1880
1881         return 0;
1882 }
1883
1884 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1885 {
1886         struct dwc3_ep                  *dep;
1887         u8                              epnum;
1888
1889         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1890                 dep = dwc->eps[epnum];
1891                 if (!dep)
1892                         continue;
1893                 /*
1894                  * Physical endpoints 0 and 1 are special; they form the
1895                  * bi-directional USB endpoint 0.
1896                  *
1897                  * For those two physical endpoints, we don't allocate a TRB
1898                  * pool nor do we add them the endpoints list. Due to that, we
1899                  * shouldn't do these two operations otherwise we would end up
1900                  * with all sorts of bugs when removing dwc3.ko.
1901                  */
1902                 if (epnum != 0 && epnum != 1) {
1903                         dwc3_free_trb_pool(dep);
1904                         list_del(&dep->endpoint.ep_list);
1905                 }
1906
1907                 kfree(dep);
1908         }
1909 }
1910
1911 /* -------------------------------------------------------------------------- */
1912
1913 static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
1914                 struct dwc3_request *req, struct dwc3_trb *trb,
1915                 const struct dwc3_event_depevt *event, int status)
1916 {
1917         unsigned int            count;
1918         unsigned int            s_pkt = 0;
1919         unsigned int            trb_status;
1920
1921         trace_dwc3_complete_trb(dep, trb);
1922
1923         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
1924                 /*
1925                  * We continue despite the error. There is not much we
1926                  * can do. If we don't clean it up we loop forever. If
1927                  * we skip the TRB then it gets overwritten after a
1928                  * while since we use them in a ring buffer. A BUG()
1929                  * would help. Lets hope that if this occurs, someone
1930                  * fixes the root cause instead of looking away :)
1931                  */
1932                 dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1933                                 dep->name, trb);
1934         count = trb->size & DWC3_TRB_SIZE_MASK;
1935
1936         if (dep->direction) {
1937                 if (count) {
1938                         trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
1939                         if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
1940                                 dwc3_trace(trace_dwc3_gadget,
1941                                                 "%s: incomplete IN transfer\n",
1942                                                 dep->name);
1943                                 /*
1944                                  * If missed isoc occurred and there is
1945                                  * no request queued then issue END
1946                                  * TRANSFER, so that core generates
1947                                  * next xfernotready and we will issue
1948                                  * a fresh START TRANSFER.
1949                                  * If there are still queued request
1950                                  * then wait, do not issue either END
1951                                  * or UPDATE TRANSFER, just attach next
1952                                  * request in pending_list during
1953                                  * giveback.If any future queued request
1954                                  * is successfully transferred then we
1955                                  * will issue UPDATE TRANSFER for all
1956                                  * request in the pending_list.
1957                                  */
1958                                 dep->flags |= DWC3_EP_MISSED_ISOC;
1959                         } else {
1960                                 dev_err(dwc->dev, "incomplete IN transfer %s\n",
1961                                                 dep->name);
1962                                 status = -ECONNRESET;
1963                         }
1964                 } else {
1965                         dep->flags &= ~DWC3_EP_MISSED_ISOC;
1966                 }
1967         } else {
1968                 if (count && (event->status & DEPEVT_STATUS_SHORT))
1969                         s_pkt = 1;
1970         }
1971
1972         /*
1973          * We assume here we will always receive the entire data block
1974          * which we should receive. Meaning, if we program RX to
1975          * receive 4K but we receive only 2K, we assume that's all we
1976          * should receive and we simply bounce the request back to the
1977          * gadget driver for further processing.
1978          */
1979         req->request.actual += req->request.length - count;
1980         if (s_pkt)
1981                 return 1;
1982         if ((event->status & DEPEVT_STATUS_LST) &&
1983                         (trb->ctrl & (DWC3_TRB_CTRL_LST |
1984                                 DWC3_TRB_CTRL_HWO)))
1985                 return 1;
1986         if ((event->status & DEPEVT_STATUS_IOC) &&
1987                         (trb->ctrl & DWC3_TRB_CTRL_IOC))
1988                 return 1;
1989         return 0;
1990 }
1991
1992 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1993                 const struct dwc3_event_depevt *event, int status)
1994 {
1995         struct dwc3_request     *req;
1996         struct dwc3_trb         *trb;
1997         unsigned int            slot;
1998         unsigned int            i;
1999         int                     ret;
2000
2001         do {
2002                 req = next_request(&dep->started_list);
2003                 if (WARN_ON_ONCE(!req))
2004                         return 1;
2005
2006                 i = 0;
2007                 do {
2008                         slot = req->first_trb_index + i;
2009                         if (slot == DWC3_TRB_NUM - 1)
2010                                 slot++;
2011                         slot %= DWC3_TRB_NUM;
2012                         trb = &dep->trb_pool[slot];
2013
2014                         ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
2015                                         event, status);
2016                         if (ret)
2017                                 break;
2018                 } while (++i < req->request.num_mapped_sgs);
2019
2020                 dwc3_gadget_giveback(dep, req, status);
2021
2022                 if (ret)
2023                         break;
2024         } while (1);
2025
2026         /*
2027          * Our endpoint might get disabled by another thread during
2028          * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2029          * early on so DWC3_EP_BUSY flag gets cleared
2030          */
2031         if (!dep->endpoint.desc)
2032                 return 1;
2033
2034         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2035                         list_empty(&dep->started_list)) {
2036                 if (list_empty(&dep->pending_list)) {
2037                         /*
2038                          * If there is no entry in request list then do
2039                          * not issue END TRANSFER now. Just set PENDING
2040                          * flag, so that END TRANSFER is issued when an
2041                          * entry is added into request list.
2042                          */
2043                         dep->flags = DWC3_EP_PENDING_REQUEST;
2044                 } else {
2045                         dwc3_stop_active_transfer(dwc, dep->number, true);
2046                         dep->flags = DWC3_EP_ENABLED;
2047                 }
2048                 return 1;
2049         }
2050
2051         if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
2052                 if ((event->status & DEPEVT_STATUS_IOC) &&
2053                                 (trb->ctrl & DWC3_TRB_CTRL_IOC))
2054                         return 0;
2055         return 1;
2056 }
2057
2058 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
2059                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
2060 {
2061         unsigned                status = 0;
2062         int                     clean_busy;
2063         u32                     is_xfer_complete;
2064
2065         is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE);
2066
2067         if (event->status & DEPEVT_STATUS_BUSERR)
2068                 status = -ECONNRESET;
2069
2070         clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
2071         if (clean_busy && (!dep->endpoint.desc || is_xfer_complete ||
2072                                 usb_endpoint_xfer_isoc(dep->endpoint.desc)))
2073                 dep->flags &= ~DWC3_EP_BUSY;
2074
2075         /*
2076          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2077          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2078          */
2079         if (dwc->revision < DWC3_REVISION_183A) {
2080                 u32             reg;
2081                 int             i;
2082
2083                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2084                         dep = dwc->eps[i];
2085
2086                         if (!(dep->flags & DWC3_EP_ENABLED))
2087                                 continue;
2088
2089                         if (!list_empty(&dep->started_list))
2090                                 return;
2091                 }
2092
2093                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2094                 reg |= dwc->u1u2;
2095                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2096
2097                 dwc->u1u2 = 0;
2098         }
2099
2100         /*
2101          * Our endpoint might get disabled by another thread during
2102          * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2103          * early on so DWC3_EP_BUSY flag gets cleared
2104          */
2105         if (!dep->endpoint.desc)
2106                 return;
2107
2108         if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2109                 int ret;
2110
2111                 ret = __dwc3_gadget_kick_transfer(dep, 0);
2112                 if (!ret || ret == -EBUSY)
2113                         return;
2114         }
2115 }
2116
2117 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2118                 const struct dwc3_event_depevt *event)
2119 {
2120         struct dwc3_ep          *dep;
2121         u8                      epnum = event->endpoint_number;
2122
2123         dep = dwc->eps[epnum];
2124
2125         if (!(dep->flags & DWC3_EP_ENABLED))
2126                 return;
2127
2128         if (epnum == 0 || epnum == 1) {
2129                 dwc3_ep0_interrupt(dwc, event);
2130                 return;
2131         }
2132
2133         switch (event->endpoint_event) {
2134         case DWC3_DEPEVT_XFERCOMPLETE:
2135                 dep->resource_index = 0;
2136
2137                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2138                         dwc3_trace(trace_dwc3_gadget,
2139                                         "%s is an Isochronous endpoint\n",
2140                                         dep->name);
2141                         return;
2142                 }
2143
2144                 dwc3_endpoint_transfer_complete(dwc, dep, event);
2145                 break;
2146         case DWC3_DEPEVT_XFERINPROGRESS:
2147                 dwc3_endpoint_transfer_complete(dwc, dep, event);
2148                 break;
2149         case DWC3_DEPEVT_XFERNOTREADY:
2150                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2151                         dwc3_gadget_start_isoc(dwc, dep, event);
2152                 } else {
2153                         int active;
2154                         int ret;
2155
2156                         active = event->status & DEPEVT_STATUS_TRANSFER_ACTIVE;
2157
2158                         dwc3_trace(trace_dwc3_gadget, "%s: reason %s",
2159                                         dep->name, active ? "Transfer Active"
2160                                         : "Transfer Not Active");
2161
2162                         ret = __dwc3_gadget_kick_transfer(dep, 0);
2163                         if (!ret || ret == -EBUSY)
2164                                 return;
2165
2166                         dwc3_trace(trace_dwc3_gadget,
2167                                         "%s: failed to kick transfers\n",
2168                                         dep->name);
2169                 }
2170
2171                 break;
2172         case DWC3_DEPEVT_STREAMEVT:
2173                 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
2174                         dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
2175                                         dep->name);
2176                         return;
2177                 }
2178
2179                 switch (event->status) {
2180                 case DEPEVT_STREAMEVT_FOUND:
2181                         dwc3_trace(trace_dwc3_gadget,
2182                                         "Stream %d found and started",
2183                                         event->parameters);
2184
2185                         break;
2186                 case DEPEVT_STREAMEVT_NOTFOUND:
2187                         /* FALLTHROUGH */
2188                 default:
2189                         dwc3_trace(trace_dwc3_gadget,
2190                                         "unable to find suitable stream\n");
2191                 }
2192                 break;
2193         case DWC3_DEPEVT_RXTXFIFOEVT:
2194                 dwc3_trace(trace_dwc3_gadget, "%s FIFO Overrun\n", dep->name);
2195                 break;
2196         case DWC3_DEPEVT_EPCMDCMPLT:
2197                 dwc3_trace(trace_dwc3_gadget, "Endpoint Command Complete");
2198                 break;
2199         }
2200 }
2201
2202 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2203 {
2204         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2205                 spin_unlock(&dwc->lock);
2206                 dwc->gadget_driver->disconnect(&dwc->gadget);
2207                 spin_lock(&dwc->lock);
2208         }
2209 }
2210
2211 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2212 {
2213         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2214                 spin_unlock(&dwc->lock);
2215                 dwc->gadget_driver->suspend(&dwc->gadget);
2216                 spin_lock(&dwc->lock);
2217         }
2218 }
2219
2220 static void dwc3_resume_gadget(struct dwc3 *dwc)
2221 {
2222         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2223                 spin_unlock(&dwc->lock);
2224                 dwc->gadget_driver->resume(&dwc->gadget);
2225                 spin_lock(&dwc->lock);
2226         }
2227 }
2228
2229 static void dwc3_reset_gadget(struct dwc3 *dwc)
2230 {
2231         if (!dwc->gadget_driver)
2232                 return;
2233
2234         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2235                 spin_unlock(&dwc->lock);
2236                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2237                 spin_lock(&dwc->lock);
2238         }
2239 }
2240
2241 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
2242 {
2243         struct dwc3_ep *dep;
2244         struct dwc3_gadget_ep_cmd_params params;
2245         u32 cmd;
2246         int ret;
2247
2248         dep = dwc->eps[epnum];
2249
2250         if (!dep->resource_index)
2251                 return;
2252
2253         /*
2254          * NOTICE: We are violating what the Databook says about the
2255          * EndTransfer command. Ideally we would _always_ wait for the
2256          * EndTransfer Command Completion IRQ, but that's causing too
2257          * much trouble synchronizing between us and gadget driver.
2258          *
2259          * We have discussed this with the IP Provider and it was
2260          * suggested to giveback all requests here, but give HW some
2261          * extra time to synchronize with the interconnect. We're using
2262          * an arbitrary 100us delay for that.
2263          *
2264          * Note also that a similar handling was tested by Synopsys
2265          * (thanks a lot Paul) and nothing bad has come out of it.
2266          * In short, what we're doing is:
2267          *
2268          * - Issue EndTransfer WITH CMDIOC bit set
2269          * - Wait 100us
2270          */
2271
2272         cmd = DWC3_DEPCMD_ENDTRANSFER;
2273         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2274         cmd |= DWC3_DEPCMD_CMDIOC;
2275         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2276         memset(&params, 0, sizeof(params));
2277         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
2278         WARN_ON_ONCE(ret);
2279         dep->resource_index = 0;
2280         dep->flags &= ~DWC3_EP_BUSY;
2281         udelay(100);
2282 }
2283
2284 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2285 {
2286         u32 epnum;
2287
2288         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2289                 struct dwc3_ep *dep;
2290
2291                 dep = dwc->eps[epnum];
2292                 if (!dep)
2293                         continue;
2294
2295                 if (!(dep->flags & DWC3_EP_ENABLED))
2296                         continue;
2297
2298                 dwc3_remove_requests(dwc, dep);
2299         }
2300 }
2301
2302 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2303 {
2304         u32 epnum;
2305
2306         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2307                 struct dwc3_ep *dep;
2308                 int ret;
2309
2310                 dep = dwc->eps[epnum];
2311                 if (!dep)
2312                         continue;
2313
2314                 if (!(dep->flags & DWC3_EP_STALL))
2315                         continue;
2316
2317                 dep->flags &= ~DWC3_EP_STALL;
2318
2319                 ret = dwc3_send_clear_stall_ep_cmd(dep);
2320                 WARN_ON_ONCE(ret);
2321         }
2322 }
2323
2324 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2325 {
2326         int                     reg;
2327
2328         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2329         reg &= ~DWC3_DCTL_INITU1ENA;
2330         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2331
2332         reg &= ~DWC3_DCTL_INITU2ENA;
2333         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2334
2335         dwc3_disconnect_gadget(dwc);
2336
2337         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2338         dwc->setup_packet_pending = false;
2339         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2340
2341         dwc->connected = false;
2342 }
2343
2344 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2345 {
2346         u32                     reg;
2347
2348         dwc->connected = true;
2349
2350         /*
2351          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2352          * would cause a missing Disconnect Event if there's a
2353          * pending Setup Packet in the FIFO.
2354          *
2355          * There's no suggested workaround on the official Bug
2356          * report, which states that "unless the driver/application
2357          * is doing any special handling of a disconnect event,
2358          * there is no functional issue".
2359          *
2360          * Unfortunately, it turns out that we _do_ some special
2361          * handling of a disconnect event, namely complete all
2362          * pending transfers, notify gadget driver of the
2363          * disconnection, and so on.
2364          *
2365          * Our suggested workaround is to follow the Disconnect
2366          * Event steps here, instead, based on a setup_packet_pending
2367          * flag. Such flag gets set whenever we have a SETUP_PENDING
2368          * status for EP0 TRBs and gets cleared on XferComplete for the
2369          * same endpoint.
2370          *
2371          * Refers to:
2372          *
2373          * STAR#9000466709: RTL: Device : Disconnect event not
2374          * generated if setup packet pending in FIFO
2375          */
2376         if (dwc->revision < DWC3_REVISION_188A) {
2377                 if (dwc->setup_packet_pending)
2378                         dwc3_gadget_disconnect_interrupt(dwc);
2379         }
2380
2381         dwc3_reset_gadget(dwc);
2382
2383         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2384         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2385         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2386         dwc->test_mode = false;
2387
2388         dwc3_stop_active_transfers(dwc);
2389         dwc3_clear_stall_all_ep(dwc);
2390
2391         /* Reset device address to zero */
2392         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2393         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2394         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2395 }
2396
2397 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
2398 {
2399         u32 reg;
2400         u32 usb30_clock = DWC3_GCTL_CLK_BUS;
2401
2402         /*
2403          * We change the clock only at SS but I dunno why I would want to do
2404          * this. Maybe it becomes part of the power saving plan.
2405          */
2406
2407         if (speed != DWC3_DSTS_SUPERSPEED)
2408                 return;
2409
2410         /*
2411          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2412          * each time on Connect Done.
2413          */
2414         if (!usb30_clock)
2415                 return;
2416
2417         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2418         reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
2419         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
2420 }
2421
2422 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2423 {
2424         struct dwc3_ep          *dep;
2425         int                     ret;
2426         u32                     reg;
2427         u8                      speed;
2428
2429         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2430         speed = reg & DWC3_DSTS_CONNECTSPD;
2431         dwc->speed = speed;
2432
2433         dwc3_update_ram_clk_sel(dwc, speed);
2434
2435         switch (speed) {
2436         case DWC3_DCFG_SUPERSPEED:
2437                 /*
2438                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2439                  * would cause a missing USB3 Reset event.
2440                  *
2441                  * In such situations, we should force a USB3 Reset
2442                  * event by calling our dwc3_gadget_reset_interrupt()
2443                  * routine.
2444                  *
2445                  * Refers to:
2446                  *
2447                  * STAR#9000483510: RTL: SS : USB3 reset event may
2448                  * not be generated always when the link enters poll
2449                  */
2450                 if (dwc->revision < DWC3_REVISION_190A)
2451                         dwc3_gadget_reset_interrupt(dwc);
2452
2453                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2454                 dwc->gadget.ep0->maxpacket = 512;
2455                 dwc->gadget.speed = USB_SPEED_SUPER;
2456                 break;
2457         case DWC3_DCFG_HIGHSPEED:
2458                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2459                 dwc->gadget.ep0->maxpacket = 64;
2460                 dwc->gadget.speed = USB_SPEED_HIGH;
2461                 break;
2462         case DWC3_DCFG_FULLSPEED2:
2463         case DWC3_DCFG_FULLSPEED1:
2464                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2465                 dwc->gadget.ep0->maxpacket = 64;
2466                 dwc->gadget.speed = USB_SPEED_FULL;
2467                 break;
2468         case DWC3_DCFG_LOWSPEED:
2469                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2470                 dwc->gadget.ep0->maxpacket = 8;
2471                 dwc->gadget.speed = USB_SPEED_LOW;
2472                 break;
2473         }
2474
2475         /* Enable USB2 LPM Capability */
2476
2477         if ((dwc->revision > DWC3_REVISION_194A)
2478                         && (speed != DWC3_DCFG_SUPERSPEED)) {
2479                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2480                 reg |= DWC3_DCFG_LPM_CAP;
2481                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2482
2483                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2484                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2485
2486                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2487
2488                 /*
2489                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2490                  * DCFG.LPMCap is set, core responses with an ACK and the
2491                  * BESL value in the LPM token is less than or equal to LPM
2492                  * NYET threshold.
2493                  */
2494                 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2495                                 && dwc->has_lpm_erratum,
2496                                 "LPM Erratum not available on dwc3 revisisions < 2.40a\n");
2497
2498                 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2499                         reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2500
2501                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2502         } else {
2503                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2504                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2505                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2506         }
2507
2508         dep = dwc->eps[0];
2509         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2510                         false);
2511         if (ret) {
2512                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2513                 return;
2514         }
2515
2516         dep = dwc->eps[1];
2517         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2518                         false);
2519         if (ret) {
2520                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2521                 return;
2522         }
2523
2524         /*
2525          * Configure PHY via GUSB3PIPECTLn if required.
2526          *
2527          * Update GTXFIFOSIZn
2528          *
2529          * In both cases reset values should be sufficient.
2530          */
2531 }
2532
2533 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2534 {
2535         /*
2536          * TODO take core out of low power mode when that's
2537          * implemented.
2538          */
2539
2540         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2541                 spin_unlock(&dwc->lock);
2542                 dwc->gadget_driver->resume(&dwc->gadget);
2543                 spin_lock(&dwc->lock);
2544         }
2545 }
2546
2547 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2548                 unsigned int evtinfo)
2549 {
2550         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2551         unsigned int            pwropt;
2552
2553         /*
2554          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2555          * Hibernation mode enabled which would show up when device detects
2556          * host-initiated U3 exit.
2557          *
2558          * In that case, device will generate a Link State Change Interrupt
2559          * from U3 to RESUME which is only necessary if Hibernation is
2560          * configured in.
2561          *
2562          * There are no functional changes due to such spurious event and we
2563          * just need to ignore it.
2564          *
2565          * Refers to:
2566          *
2567          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2568          * operational mode
2569          */
2570         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2571         if ((dwc->revision < DWC3_REVISION_250A) &&
2572                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2573                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2574                                 (next == DWC3_LINK_STATE_RESUME)) {
2575                         dwc3_trace(trace_dwc3_gadget,
2576                                         "ignoring transition U3 -> Resume");
2577                         return;
2578                 }
2579         }
2580
2581         /*
2582          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2583          * on the link partner, the USB session might do multiple entry/exit
2584          * of low power states before a transfer takes place.
2585          *
2586          * Due to this problem, we might experience lower throughput. The
2587          * suggested workaround is to disable DCTL[12:9] bits if we're
2588          * transitioning from U1/U2 to U0 and enable those bits again
2589          * after a transfer completes and there are no pending transfers
2590          * on any of the enabled endpoints.
2591          *
2592          * This is the first half of that workaround.
2593          *
2594          * Refers to:
2595          *
2596          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2597          * core send LGO_Ux entering U0
2598          */
2599         if (dwc->revision < DWC3_REVISION_183A) {
2600                 if (next == DWC3_LINK_STATE_U0) {
2601                         u32     u1u2;
2602                         u32     reg;
2603
2604                         switch (dwc->link_state) {
2605                         case DWC3_LINK_STATE_U1:
2606                         case DWC3_LINK_STATE_U2:
2607                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2608                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2609                                                 | DWC3_DCTL_ACCEPTU2ENA
2610                                                 | DWC3_DCTL_INITU1ENA
2611                                                 | DWC3_DCTL_ACCEPTU1ENA);
2612
2613                                 if (!dwc->u1u2)
2614                                         dwc->u1u2 = reg & u1u2;
2615
2616                                 reg &= ~u1u2;
2617
2618                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2619                                 break;
2620                         default:
2621                                 /* do nothing */
2622                                 break;
2623                         }
2624                 }
2625         }
2626
2627         switch (next) {
2628         case DWC3_LINK_STATE_U1:
2629                 if (dwc->speed == USB_SPEED_SUPER)
2630                         dwc3_suspend_gadget(dwc);
2631                 break;
2632         case DWC3_LINK_STATE_U2:
2633         case DWC3_LINK_STATE_U3:
2634                 dwc3_suspend_gadget(dwc);
2635                 break;
2636         case DWC3_LINK_STATE_RESUME:
2637                 dwc3_resume_gadget(dwc);
2638                 break;
2639         default:
2640                 /* do nothing */
2641                 break;
2642         }
2643
2644         dwc->link_state = next;
2645 }
2646
2647 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2648                 unsigned int evtinfo)
2649 {
2650         unsigned int is_ss = evtinfo & BIT(4);
2651
2652         /**
2653          * WORKAROUND: DWC3 revison 2.20a with hibernation support
2654          * have a known issue which can cause USB CV TD.9.23 to fail
2655          * randomly.
2656          *
2657          * Because of this issue, core could generate bogus hibernation
2658          * events which SW needs to ignore.
2659          *
2660          * Refers to:
2661          *
2662          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2663          * Device Fallback from SuperSpeed
2664          */
2665         if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2666                 return;
2667
2668         /* enter hibernation here */
2669 }
2670
2671 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2672                 const struct dwc3_event_devt *event)
2673 {
2674         switch (event->type) {
2675         case DWC3_DEVICE_EVENT_DISCONNECT:
2676                 dwc3_gadget_disconnect_interrupt(dwc);
2677                 break;
2678         case DWC3_DEVICE_EVENT_RESET:
2679                 dwc3_gadget_reset_interrupt(dwc);
2680                 break;
2681         case DWC3_DEVICE_EVENT_CONNECT_DONE:
2682                 dwc3_gadget_conndone_interrupt(dwc);
2683                 break;
2684         case DWC3_DEVICE_EVENT_WAKEUP:
2685                 dwc3_gadget_wakeup_interrupt(dwc);
2686                 break;
2687         case DWC3_DEVICE_EVENT_HIBER_REQ:
2688                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
2689                                         "unexpected hibernation event\n"))
2690                         break;
2691
2692                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2693                 break;
2694         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2695                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2696                 break;
2697         case DWC3_DEVICE_EVENT_EOPF:
2698                 dwc3_trace(trace_dwc3_gadget, "End of Periodic Frame");
2699                 break;
2700         case DWC3_DEVICE_EVENT_SOF:
2701                 dwc3_trace(trace_dwc3_gadget, "Start of Periodic Frame");
2702                 break;
2703         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2704                 dwc3_trace(trace_dwc3_gadget, "Erratic Error");
2705                 break;
2706         case DWC3_DEVICE_EVENT_CMD_CMPL:
2707                 dwc3_trace(trace_dwc3_gadget, "Command Complete");
2708                 break;
2709         case DWC3_DEVICE_EVENT_OVERFLOW:
2710                 dwc3_trace(trace_dwc3_gadget, "Overflow");
2711                 break;
2712         default:
2713                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2714         }
2715 }
2716
2717 static void dwc3_process_event_entry(struct dwc3 *dwc,
2718                 const union dwc3_event *event)
2719 {
2720         trace_dwc3_event(event->raw);
2721
2722         /* Endpoint IRQ, handle it and return early */
2723         if (event->type.is_devspec == 0) {
2724                 /* depevt */
2725                 return dwc3_endpoint_interrupt(dwc, &event->depevt);
2726         }
2727
2728         switch (event->type.type) {
2729         case DWC3_EVENT_TYPE_DEV:
2730                 dwc3_gadget_interrupt(dwc, &event->devt);
2731                 break;
2732         /* REVISIT what to do with Carkit and I2C events ? */
2733         default:
2734                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2735         }
2736 }
2737
2738 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
2739 {
2740         struct dwc3 *dwc = evt->dwc;
2741         irqreturn_t ret = IRQ_NONE;
2742         int left;
2743         u32 reg;
2744
2745         left = evt->count;
2746
2747         if (!(evt->flags & DWC3_EVENT_PENDING))
2748                 return IRQ_NONE;
2749
2750         while (left > 0) {
2751                 union dwc3_event event;
2752
2753                 event.raw = *(u32 *) (evt->buf + evt->lpos);
2754
2755                 dwc3_process_event_entry(dwc, &event);
2756
2757                 /*
2758                  * FIXME we wrap around correctly to the next entry as
2759                  * almost all entries are 4 bytes in size. There is one
2760                  * entry which has 12 bytes which is a regular entry
2761                  * followed by 8 bytes data. ATM I don't know how
2762                  * things are organized if we get next to the a
2763                  * boundary so I worry about that once we try to handle
2764                  * that.
2765                  */
2766                 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2767                 left -= 4;
2768
2769                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), 4);
2770         }
2771
2772         evt->count = 0;
2773         evt->flags &= ~DWC3_EVENT_PENDING;
2774         ret = IRQ_HANDLED;
2775
2776         /* Unmask interrupt */
2777         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
2778         reg &= ~DWC3_GEVNTSIZ_INTMASK;
2779         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
2780
2781         return ret;
2782 }
2783
2784 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
2785 {
2786         struct dwc3_event_buffer *evt = _evt;
2787         struct dwc3 *dwc = evt->dwc;
2788         unsigned long flags;
2789         irqreturn_t ret = IRQ_NONE;
2790
2791         spin_lock_irqsave(&dwc->lock, flags);
2792         ret = dwc3_process_event_buf(evt);
2793         spin_unlock_irqrestore(&dwc->lock, flags);
2794
2795         return ret;
2796 }
2797
2798 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
2799 {
2800         struct dwc3 *dwc = evt->dwc;
2801         u32 count;
2802         u32 reg;
2803
2804         if (pm_runtime_suspended(dwc->dev)) {
2805                 pm_runtime_get(dwc->dev);
2806                 disable_irq_nosync(dwc->irq_gadget);
2807                 dwc->pending_events = true;
2808                 return IRQ_HANDLED;
2809         }
2810
2811         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
2812         count &= DWC3_GEVNTCOUNT_MASK;
2813         if (!count)
2814                 return IRQ_NONE;
2815
2816         evt->count = count;
2817         evt->flags |= DWC3_EVENT_PENDING;
2818
2819         /* Mask interrupt */
2820         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
2821         reg |= DWC3_GEVNTSIZ_INTMASK;
2822         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
2823
2824         return IRQ_WAKE_THREAD;
2825 }
2826
2827 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
2828 {
2829         struct dwc3_event_buffer        *evt = _evt;
2830
2831         return dwc3_check_event_buf(evt);
2832 }
2833
2834 /**
2835  * dwc3_gadget_init - Initializes gadget related registers
2836  * @dwc: pointer to our controller context structure
2837  *
2838  * Returns 0 on success otherwise negative errno.
2839  */
2840 int dwc3_gadget_init(struct dwc3 *dwc)
2841 {
2842         int                                     ret;
2843
2844         dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2845                         &dwc->ctrl_req_addr, GFP_KERNEL);
2846         if (!dwc->ctrl_req) {
2847                 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2848                 ret = -ENOMEM;
2849                 goto err0;
2850         }
2851
2852         dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
2853                         &dwc->ep0_trb_addr, GFP_KERNEL);
2854         if (!dwc->ep0_trb) {
2855                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2856                 ret = -ENOMEM;
2857                 goto err1;
2858         }
2859
2860         dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL);
2861         if (!dwc->setup_buf) {
2862                 ret = -ENOMEM;
2863                 goto err2;
2864         }
2865
2866         dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
2867                         DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
2868                         GFP_KERNEL);
2869         if (!dwc->ep0_bounce) {
2870                 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2871                 ret = -ENOMEM;
2872                 goto err3;
2873         }
2874
2875         dwc->zlp_buf = kzalloc(DWC3_ZLP_BUF_SIZE, GFP_KERNEL);
2876         if (!dwc->zlp_buf) {
2877                 ret = -ENOMEM;
2878                 goto err4;
2879         }
2880
2881         dwc->gadget.ops                 = &dwc3_gadget_ops;
2882         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
2883         dwc->gadget.sg_supported        = true;
2884         dwc->gadget.name                = "dwc3-gadget";
2885         dwc->gadget.is_otg              = dwc->dr_mode == USB_DR_MODE_OTG;
2886
2887         /*
2888          * FIXME We might be setting max_speed to <SUPER, however versions
2889          * <2.20a of dwc3 have an issue with metastability (documented
2890          * elsewhere in this driver) which tells us we can't set max speed to
2891          * anything lower than SUPER.
2892          *
2893          * Because gadget.max_speed is only used by composite.c and function
2894          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
2895          * to happen so we avoid sending SuperSpeed Capability descriptor
2896          * together with our BOS descriptor as that could confuse host into
2897          * thinking we can handle super speed.
2898          *
2899          * Note that, in fact, we won't even support GetBOS requests when speed
2900          * is less than super speed because we don't have means, yet, to tell
2901          * composite.c that we are USB 2.0 + LPM ECN.
2902          */
2903         if (dwc->revision < DWC3_REVISION_220A)
2904                 dwc3_trace(trace_dwc3_gadget,
2905                                 "Changing max_speed on rev %08x\n",
2906                                 dwc->revision);
2907
2908         dwc->gadget.max_speed           = dwc->maximum_speed;
2909
2910         /*
2911          * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize
2912          * on ep out.
2913          */
2914         dwc->gadget.quirk_ep_out_aligned_size = true;
2915
2916         /*
2917          * REVISIT: Here we should clear all pending IRQs to be
2918          * sure we're starting from a well known location.
2919          */
2920
2921         ret = dwc3_gadget_init_endpoints(dwc);
2922         if (ret)
2923                 goto err5;
2924
2925         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2926         if (ret) {
2927                 dev_err(dwc->dev, "failed to register udc\n");
2928                 goto err5;
2929         }
2930
2931         return 0;
2932
2933 err5:
2934         kfree(dwc->zlp_buf);
2935
2936 err4:
2937         dwc3_gadget_free_endpoints(dwc);
2938         dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2939                         dwc->ep0_bounce, dwc->ep0_bounce_addr);
2940
2941 err3:
2942         kfree(dwc->setup_buf);
2943
2944 err2:
2945         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2946                         dwc->ep0_trb, dwc->ep0_trb_addr);
2947
2948 err1:
2949         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2950                         dwc->ctrl_req, dwc->ctrl_req_addr);
2951
2952 err0:
2953         return ret;
2954 }
2955
2956 /* -------------------------------------------------------------------------- */
2957
2958 void dwc3_gadget_exit(struct dwc3 *dwc)
2959 {
2960         usb_del_gadget_udc(&dwc->gadget);
2961
2962         dwc3_gadget_free_endpoints(dwc);
2963
2964         dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2965                         dwc->ep0_bounce, dwc->ep0_bounce_addr);
2966
2967         kfree(dwc->setup_buf);
2968         kfree(dwc->zlp_buf);
2969
2970         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2971                         dwc->ep0_trb, dwc->ep0_trb_addr);
2972
2973         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2974                         dwc->ctrl_req, dwc->ctrl_req_addr);
2975 }
2976
2977 int dwc3_gadget_suspend(struct dwc3 *dwc)
2978 {
2979         int ret;
2980
2981         if (!dwc->gadget_driver)
2982                 return 0;
2983
2984         ret = dwc3_gadget_run_stop(dwc, false, false);
2985         if (ret < 0)
2986                 return ret;
2987
2988         dwc3_disconnect_gadget(dwc);
2989         __dwc3_gadget_stop(dwc);
2990
2991         return 0;
2992 }
2993
2994 int dwc3_gadget_resume(struct dwc3 *dwc)
2995 {
2996         int                     ret;
2997
2998         if (!dwc->gadget_driver)
2999                 return 0;
3000
3001         ret = __dwc3_gadget_start(dwc);
3002         if (ret < 0)
3003                 goto err0;
3004
3005         ret = dwc3_gadget_run_stop(dwc, true, false);
3006         if (ret < 0)
3007                 goto err1;
3008
3009         return 0;
3010
3011 err1:
3012         __dwc3_gadget_stop(dwc);
3013
3014 err0:
3015         return ret;
3016 }
3017
3018 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3019 {
3020         if (dwc->pending_events) {
3021                 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3022                 dwc->pending_events = false;
3023                 enable_irq(dwc->irq_gadget);
3024         }
3025 }