]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/usb/dwc3/gadget.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
usb: dwc3: gadget: return 0 if we try to Wakeup in superspeed
[mirror_ubuntu-artful-kernel.git] / drivers / usb / dwc3 / gadget.c
CommitLineData
72246da4
FB		 1/**
2 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
3 *
4 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
72246da4
FB		 5 *
6 * Authors: Felipe Balbi <balbi@ti.com>,
7 * Sebastian Andrzej Siewior <bigeasy@linutronix.de>
8 *
5945f789
FB		 9 * This program is free software: you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 of
11 * the License as published by the Free Software Foundation.
72246da4 12 *
5945f789
FB		 13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
72246da4
FB		 17 */
18
19#include <linux/kernel.h>
20#include <linux/delay.h>
21#include <linux/slab.h>
22#include <linux/spinlock.h>
23#include <linux/platform_device.h>
24#include <linux/pm_runtime.h>
25#include <linux/interrupt.h>
26#include <linux/io.h>
27#include <linux/list.h>
28#include <linux/dma-mapping.h>
29
30#include <linux/usb/ch9.h>
31#include <linux/usb/gadget.h>
32
80977dc9 33#include "debug.h"
72246da4
FB		 34#include "core.h"
35#include "gadget.h"
36#include "io.h"
37
04a9bfcd
FB		 38/**
39 * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
40 * @dwc: pointer to our context structure
41 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
42 *
43 * Caller should take care of locking. This function will
44 * return 0 on success or -EINVAL if wrong Test Selector
45 * is passed
46 */
47int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
48{
49 u32 reg;
50
51 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
52 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
53
54 switch (mode) {
55 case TEST_J:
56 case TEST_K:
57 case TEST_SE0_NAK:
58 case TEST_PACKET:
59 case TEST_FORCE_EN:
60 reg |= mode << 1;
61 break;
62 default:
63 return -EINVAL;
64 }
65
66 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
67
68 return 0;
69}
70
911f1f88
PZ		 71/**
72 * dwc3_gadget_get_link_state - Gets current state of USB Link
73 * @dwc: pointer to our context structure
74 *
75 * Caller should take care of locking. This function will
76 * return the link state on success (>= 0) or -ETIMEDOUT.
77 */
78int dwc3_gadget_get_link_state(struct dwc3 *dwc)
79{
80 u32 reg;
81
82 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
83
84 return DWC3_DSTS_USBLNKST(reg);
85}
86
8598bde7
FB		 87/**
88 * dwc3_gadget_set_link_state - Sets USB Link to a particular State
89 * @dwc: pointer to our context structure
90 * @state: the state to put link into
91 *
92 * Caller should take care of locking. This function will
aee63e3c 93 * return 0 on success or -ETIMEDOUT.
8598bde7
FB		 94 */
95int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
96{
aee63e3c 97 int retries = 10000;
8598bde7
FB		 98 u32 reg;
99
802fde98
PZ		 100 /*
101 * Wait until device controller is ready. Only applies to 1.94a and
102 * later RTL.
103 */
104 if (dwc->revision >= DWC3_REVISION_194A) {
105 while (--retries) {
106 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
107 if (reg & DWC3_DSTS_DCNRD)
108 udelay(5);
109 else
110 break;
111 }
112
113 if (retries <= 0)
114 return -ETIMEDOUT;
115 }
116
8598bde7
FB		 117 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
118 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
119
120 /* set requested state */
121 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
122 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
123
802fde98
PZ		 124 /*
125 * The following code is racy when called from dwc3_gadget_wakeup,
126 * and is not needed, at least on newer versions
127 */
128 if (dwc->revision >= DWC3_REVISION_194A)
129 return 0;
130
8598bde7 131 /* wait for a change in DSTS */
aed430e5 132 retries = 10000;
8598bde7
FB		 133 while (--retries) {
134 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
135
8598bde7
FB		 136 if (DWC3_DSTS_USBLNKST(reg) == state)
137 return 0;
138
aee63e3c 139 udelay(5);
8598bde7
FB		 140 }
141
73815280
FB		 142 dwc3_trace(trace_dwc3_gadget,
143 "link state change request timed out");
8598bde7
FB		 144
145 return -ETIMEDOUT;
146}
147
ef966b9d 148static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
457e84b6 149{
ef966b9d 150 dep->trb_enqueue++;
4faf7550 151 dep->trb_enqueue %= DWC3_TRB_NUM;
ef966b9d 152}
457e84b6 153
ef966b9d
FB		 154static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
155{
156 dep->trb_dequeue++;
4faf7550 157 dep->trb_dequeue %= DWC3_TRB_NUM;
ef966b9d 158}
457e84b6 159
ef966b9d
FB		 160static int dwc3_ep_is_last_trb(unsigned int index)
161{
4faf7550 162 return index == DWC3_TRB_NUM - 1;
457e84b6
FB		 163}
164
72246da4
FB		 165void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
166 int status)
167{
168 struct dwc3 *dwc = dep->dwc;
e5ba5ec8 169 int i;
72246da4 170
aa3342c8 171 if (req->started) {
e5ba5ec8
PA		 172 i = 0;
173 do {
ef966b9d 174 dwc3_ep_inc_deq(dep);
e5ba5ec8
PA		 175 /*
176 * Skip LINK TRB. We can't use req->trb and check for
177 * DWC3_TRBCTL_LINK_TRB because it points the TRB we
178 * just completed (not the LINK TRB).
179 */
36b68aae 180 if (dwc3_ep_is_last_trb(dep->trb_dequeue))
ef966b9d 181 dwc3_ep_inc_deq(dep);
e5ba5ec8 182 } while(++i < req->request.num_mapped_sgs);
aa3342c8 183 req->started = false;
72246da4
FB		 184 }
185 list_del(&req->list);
eeb720fb 186 req->trb = NULL;
72246da4
FB		 187
188 if (req->request.status == -EINPROGRESS)
189 req->request.status = status;
190
0416e494
PA		 191 if (dwc->ep0_bounced && dep->number == 0)
192 dwc->ep0_bounced = false;
193 else
194 usb_gadget_unmap_request(&dwc->gadget, &req->request,
195 req->direction);
72246da4 196
2c4cbe6e 197 trace_dwc3_gadget_giveback(req);
72246da4
FB		 198
199 spin_unlock(&dwc->lock);
304f7e5e 200 usb_gadget_giveback_request(&dep->endpoint, &req->request);
72246da4
FB		 201 spin_lock(&dwc->lock);
202}
203
3ece0ec4 204int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
b09bb642
FB		 205{
206 u32 timeout = 500;
207 u32 reg;
208
2c4cbe6e 209 trace_dwc3_gadget_generic_cmd(cmd, param);
427c3df6 210
b09bb642
FB		 211 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
212 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
213
214 do {
215 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
216 if (!(reg & DWC3_DGCMD_CMDACT)) {
73815280
FB		 217 dwc3_trace(trace_dwc3_gadget,
218 "Command Complete --> %d",
b09bb642 219 DWC3_DGCMD_STATUS(reg));
891b1dc0
SSB		 220 if (DWC3_DGCMD_STATUS(reg))
221 return -EINVAL;
b09bb642
FB		 222 return 0;
223 }
224
225 /*
226 * We can't sleep here, because it's also called from
227 * interrupt context.
228 */
229 timeout--;
73815280
FB		 230 if (!timeout) {
231 dwc3_trace(trace_dwc3_gadget,
232 "Command Timed Out");
b09bb642 233 return -ETIMEDOUT;
73815280 234 }
b09bb642
FB		 235 udelay(1);
236 } while (1);
237}
238
c36d8e94
FB		 239static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
240
72246da4
FB		 241int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
242 unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
243{
244 struct dwc3_ep *dep = dwc->eps[ep];
61d58242 245 u32 timeout = 500;
72246da4
FB		 246 u32 reg;
247
2b0f11df 248 int susphy = false;
c0ca324d 249 int ret = -EINVAL;
72246da4 250
2c4cbe6e 251 trace_dwc3_gadget_ep_cmd(dep, cmd, params);
72246da4 252
2b0f11df
FB		 253 /*
254 * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
255 * we're issuing an endpoint command, we must check if
256 * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
257 *
258 * We will also set SUSPHY bit to what it was before returning as stated
259 * by the same section on Synopsys databook.
260 */
261 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
262 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
263 susphy = true;
264 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
265 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
266 }
267
c36d8e94
FB		 268 if (cmd == DWC3_DEPCMD_STARTTRANSFER) {
269 int needs_wakeup;
270
271 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
272 dwc->link_state == DWC3_LINK_STATE_U2 ||
273 dwc->link_state == DWC3_LINK_STATE_U3);
274
275 if (unlikely(needs_wakeup)) {
276 ret = __dwc3_gadget_wakeup(dwc);
277 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
278 ret);
279 }
280 }
281
dc1c70a7
FB		 282 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0);
283 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1);
284 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2);
72246da4
FB		 285
286 dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
287 do {
288 reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
289 if (!(reg & DWC3_DEPCMD_CMDACT)) {
7b9cc7a2
KL		 290 int cmd_status = DWC3_DEPCMD_STATUS(reg);
291
73815280
FB		 292 dwc3_trace(trace_dwc3_gadget,
293 "Command Complete --> %d",
7b9cc7a2
KL		 294 cmd_status);
295
296 switch (cmd_status) {
297 case 0:
298 ret = 0;
299 break;
300 case DEPEVT_TRANSFER_NO_RESOURCE:
301 dwc3_trace(trace_dwc3_gadget, "%s: no resource available");
302 ret = -EINVAL;
c0ca324d 303 break;
7b9cc7a2
KL		 304 case DEPEVT_TRANSFER_BUS_EXPIRY:
305 /*
306 * SW issues START TRANSFER command to
307 * isochronous ep with future frame interval. If
308 * future interval time has already passed when
309 * core receives the command, it will respond
310 * with an error status of 'Bus Expiry'.
311 *
312 * Instead of always returning -EINVAL, let's
313 * give a hint to the gadget driver that this is
314 * the case by returning -EAGAIN.
315 */
316 dwc3_trace(trace_dwc3_gadget, "%s: bus expiry");
317 ret = -EAGAIN;
318 break;
319 default:
320 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
321 }
322
c0ca324d 323 break;
72246da4
FB		 324 }
325
326 /*
72246da4
FB		 327 * We can't sleep here, because it is also called from
328 * interrupt context.
329 */
330 timeout--;
73815280
FB		 331 if (!timeout) {
332 dwc3_trace(trace_dwc3_gadget,
333 "Command Timed Out");
c0ca324d
FB		 334 ret = -ETIMEDOUT;
335 break;
73815280 336 }
72246da4 337 } while (1);
c0ca324d 338
2b0f11df
FB		 339 if (unlikely(susphy)) {
340 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
341 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
342 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
343 }
344
c0ca324d 345 return ret;
72246da4
FB		 346}
347
50c763f8
JY		 348static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
349{
350 struct dwc3 *dwc = dep->dwc;
351 struct dwc3_gadget_ep_cmd_params params;
352 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
353
354 /*
355 * As of core revision 2.60a the recommended programming model
356 * is to set the ClearPendIN bit when issuing a Clear Stall EP
357 * command for IN endpoints. This is to prevent an issue where
358 * some (non-compliant) hosts may not send ACK TPs for pending
359 * IN transfers due to a mishandled error condition. Synopsys
360 * STAR 9000614252.
361 */
362 if (dep->direction && (dwc->revision >= DWC3_REVISION_260A))
363 cmd |= DWC3_DEPCMD_CLEARPENDIN;
364
365 memset(&params, 0, sizeof(params));
366
367 return dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
368}
369
72246da4 370static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
f6bafc6a 371 struct dwc3_trb *trb)
72246da4 372{
c439ef87 373 u32 offset = (char *) trb - (char *) dep->trb_pool;
72246da4
FB		 374
375 return dep->trb_pool_dma + offset;
376}
377
378static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
379{
380 struct dwc3 *dwc = dep->dwc;
381
382 if (dep->trb_pool)
383 return 0;
384
72246da4
FB		 385 dep->trb_pool = dma_alloc_coherent(dwc->dev,
386 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
387 &dep->trb_pool_dma, GFP_KERNEL);
388 if (!dep->trb_pool) {
389 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
390 dep->name);
391 return -ENOMEM;
392 }
393
394 return 0;
395}
396
397static void dwc3_free_trb_pool(struct dwc3_ep *dep)
398{
399 struct dwc3 *dwc = dep->dwc;
400
401 dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
402 dep->trb_pool, dep->trb_pool_dma);
403
404 dep->trb_pool = NULL;
405 dep->trb_pool_dma = 0;
406}
407
c4509601
JY		 408static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
409
410/**
411 * dwc3_gadget_start_config - Configure EP resources
412 * @dwc: pointer to our controller context structure
413 * @dep: endpoint that is being enabled
414 *
415 * The assignment of transfer resources cannot perfectly follow the
416 * data book due to the fact that the controller driver does not have
417 * all knowledge of the configuration in advance. It is given this
418 * information piecemeal by the composite gadget framework after every
419 * SET_CONFIGURATION and SET_INTERFACE. Trying to follow the databook
420 * programming model in this scenario can cause errors. For two
421 * reasons:
422 *
423 * 1) The databook says to do DEPSTARTCFG for every SET_CONFIGURATION
424 * and SET_INTERFACE (8.1.5). This is incorrect in the scenario of
425 * multiple interfaces.
426 *
427 * 2) The databook does not mention doing more DEPXFERCFG for new
428 * endpoint on alt setting (8.1.6).
429 *
430 * The following simplified method is used instead:
431 *
432 * All hardware endpoints can be assigned a transfer resource and this
433 * setting will stay persistent until either a core reset or
434 * hibernation. So whenever we do a DEPSTARTCFG(0) we can go ahead and
435 * do DEPXFERCFG for every hardware endpoint as well. We are
436 * guaranteed that there are as many transfer resources as endpoints.
437 *
438 * This function is called for each endpoint when it is being enabled
439 * but is triggered only when called for EP0-out, which always happens
440 * first, and which should only happen in one of the above conditions.
441 */
72246da4
FB		 442static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
443{
444 struct dwc3_gadget_ep_cmd_params params;
445 u32 cmd;
c4509601
JY		 446 int i;
447 int ret;
448
449 if (dep->number)
450 return 0;
72246da4
FB		 451
452 memset(&params, 0x00, sizeof(params));
c4509601 453 cmd = DWC3_DEPCMD_DEPSTARTCFG;
72246da4 454
c4509601
JY		 455 ret = dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
456 if (ret)
457 return ret;
458
459 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
460 struct dwc3_ep *dep = dwc->eps[i];
72246da4 461
c4509601
JY		 462 if (!dep)
463 continue;
464
465 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
466 if (ret)
467 return ret;
72246da4
FB		 468 }
469
470 return 0;
471}
472
473static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
c90bfaec 474 const struct usb_endpoint_descriptor *desc,
4b345c9a 475 const struct usb_ss_ep_comp_descriptor *comp_desc,
265b70a7 476 bool ignore, bool restore)
72246da4
FB		 477{
478 struct dwc3_gadget_ep_cmd_params params;
479
480 memset(&params, 0x00, sizeof(params));
481
dc1c70a7 482 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
d2e9a13a
CP		 483 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
484
485 /* Burst size is only needed in SuperSpeed mode */
ee5cd41c 486 if (dwc->gadget.speed >= USB_SPEED_SUPER) {
676e3497
FB		 487 u32 burst = dep->endpoint.maxburst;
488 u32 nump;
489 u32 reg;
490
491 /* update NumP */
492 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
493 nump = DWC3_DCFG_NUMP(reg);
494 nump = max(nump, burst);
495 reg &= ~DWC3_DCFG_NUMP_MASK;
496 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
497 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
d2e9a13a 498
676e3497 499 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
d2e9a13a 500 }
72246da4 501
4b345c9a
FB		 502 if (ignore)
503 params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM;
504
265b70a7
PZ		 505 if (restore) {
506 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
507 params.param2 |= dep->saved_state;
508 }
509
dc1c70a7
FB		 510 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
511 | DWC3_DEPCFG_XFER_NOT_READY_EN;
72246da4 512
18b7ede5 513 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
dc1c70a7
FB		 514 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
515 | DWC3_DEPCFG_STREAM_EVENT_EN;
879631aa
FB		 516 dep->stream_capable = true;
517 }
518
0b93a4c8 519 if (!usb_endpoint_xfer_control(desc))
dc1c70a7 520 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
72246da4
FB		 521
522 /*
523 * We are doing 1:1 mapping for endpoints, meaning
524 * Physical Endpoints 2 maps to Logical Endpoint 2 and
525 * so on. We consider the direction bit as part of the physical
526 * endpoint number. So USB endpoint 0x81 is 0x03.
527 */
dc1c70a7 528 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
72246da4
FB		 529
530 /*
531 * We must use the lower 16 TX FIFOs even though
532 * HW might have more
533 */
534 if (dep->direction)
dc1c70a7 535 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
72246da4
FB		 536
537 if (desc->bInterval) {
dc1c70a7 538 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
72246da4
FB		 539 dep->interval = 1 << (desc->bInterval - 1);
540 }
541
542 return dwc3_send_gadget_ep_cmd(dwc, dep->number,
543 DWC3_DEPCMD_SETEPCONFIG, &params);
544}
545
546static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
547{
548 struct dwc3_gadget_ep_cmd_params params;
549
550 memset(&params, 0x00, sizeof(params));
551
dc1c70a7 552 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
72246da4
FB		 553
554 return dwc3_send_gadget_ep_cmd(dwc, dep->number,
555 DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
556}
557
558/**
559 * __dwc3_gadget_ep_enable - Initializes a HW endpoint
560 * @dep: endpoint to be initialized
561 * @desc: USB Endpoint Descriptor
562 *
563 * Caller should take care of locking
564 */
565static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
c90bfaec 566 const struct usb_endpoint_descriptor *desc,
4b345c9a 567 const struct usb_ss_ep_comp_descriptor *comp_desc,
265b70a7 568 bool ignore, bool restore)
72246da4
FB		 569{
570 struct dwc3 *dwc = dep->dwc;
571 u32 reg;
b09e99ee 572 int ret;
72246da4 573
73815280 574 dwc3_trace(trace_dwc3_gadget, "Enabling %s", dep->name);
ff62d6b6 575
72246da4
FB		 576 if (!(dep->flags & DWC3_EP_ENABLED)) {
577 ret = dwc3_gadget_start_config(dwc, dep);
578 if (ret)
579 return ret;
580 }
581
265b70a7
PZ		 582 ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore,
583 restore);
72246da4
FB		 584 if (ret)
585 return ret;
586
587 if (!(dep->flags & DWC3_EP_ENABLED)) {
f6bafc6a
FB		 588 struct dwc3_trb *trb_st_hw;
589 struct dwc3_trb *trb_link;
72246da4 590
16e78db7 591 dep->endpoint.desc = desc;
c90bfaec 592 dep->comp_desc = comp_desc;
72246da4
FB		 593 dep->type = usb_endpoint_type(desc);
594 dep->flags |= DWC3_EP_ENABLED;
595
596 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
597 reg |= DWC3_DALEPENA_EP(dep->number);
598 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
599
36b68aae 600 if (usb_endpoint_xfer_control(desc))
e901aa15 601 goto out;
72246da4 602
36b68aae 603 /* Link TRB. The HWO bit is never reset */
72246da4
FB		 604 trb_st_hw = &dep->trb_pool[0];
605
f6bafc6a 606 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
1200a82a 607 memset(trb_link, 0, sizeof(*trb_link));
72246da4 608
f6bafc6a
FB		 609 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
610 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
611 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
612 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
72246da4
FB		 613 }
614
e901aa15 615out:
aa739974
FB		 616 switch (usb_endpoint_type(desc)) {
617 case USB_ENDPOINT_XFER_CONTROL:
e901aa15 618 /* don't change name */
aa739974
FB		 619 break;
620 case USB_ENDPOINT_XFER_ISOC:
621 strlcat(dep->name, "-isoc", sizeof(dep->name));
622 break;
623 case USB_ENDPOINT_XFER_BULK:
624 strlcat(dep->name, "-bulk", sizeof(dep->name));
625 break;
626 case USB_ENDPOINT_XFER_INT:
627 strlcat(dep->name, "-int", sizeof(dep->name));
628 break;
629 default:
630 dev_err(dwc->dev, "invalid endpoint transfer type\n");
631 }
632
72246da4
FB		 633 return 0;
634}
635
b992e681 636static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
624407f9 637static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
72246da4
FB		 638{
639 struct dwc3_request *req;
640
aa3342c8 641 if (!list_empty(&dep->started_list)) {
b992e681 642 dwc3_stop_active_transfer(dwc, dep->number, true);
624407f9 643
57911504 644 /* - giveback all requests to gadget driver */
aa3342c8
FB		 645 while (!list_empty(&dep->started_list)) {
646 req = next_request(&dep->started_list);
1591633e
PA		 647
648 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
649 }
ea53b882
FB		 650 }
651
aa3342c8
FB		 652 while (!list_empty(&dep->pending_list)) {
653 req = next_request(&dep->pending_list);
72246da4 654
624407f9 655 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
72246da4 656 }
72246da4
FB		 657}
658
659/**
660 * __dwc3_gadget_ep_disable - Disables a HW endpoint
661 * @dep: the endpoint to disable
662 *
624407f9
SAS		 663 * This function also removes requests which are currently processed ny the
664 * hardware and those which are not yet scheduled.
665 * Caller should take care of locking.
72246da4 666 */
72246da4
FB		 667static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
668{
669 struct dwc3 *dwc = dep->dwc;
670 u32 reg;
671
7eaeac5c
FB		 672 dwc3_trace(trace_dwc3_gadget, "Disabling %s", dep->name);
673
624407f9 674 dwc3_remove_requests(dwc, dep);
72246da4 675
687ef981
FB		 676 /* make sure HW endpoint isn't stalled */
677 if (dep->flags & DWC3_EP_STALL)
7a608559 678 __dwc3_gadget_ep_set_halt(dep, 0, false);
687ef981 679
72246da4
FB		 680 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
681 reg &= ~DWC3_DALEPENA_EP(dep->number);
682 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
683
879631aa 684 dep->stream_capable = false;
f9c56cdd 685 dep->endpoint.desc = NULL;
c90bfaec 686 dep->comp_desc = NULL;
72246da4 687 dep->type = 0;
879631aa 688 dep->flags = 0;
72246da4 689
aa739974
FB		 690 snprintf(dep->name, sizeof(dep->name), "ep%d%s",
691 dep->number >> 1,
692 (dep->number & 1) ? "in" : "out");
693
72246da4
FB		 694 return 0;
695}
696
697/* -------------------------------------------------------------------------- */
698
699static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
700 const struct usb_endpoint_descriptor *desc)
701{
702 return -EINVAL;
703}
704
705static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
706{
707 return -EINVAL;
708}
709
710/* -------------------------------------------------------------------------- */
711
712static int dwc3_gadget_ep_enable(struct usb_ep *ep,
713 const struct usb_endpoint_descriptor *desc)
714{
715 struct dwc3_ep *dep;
716 struct dwc3 *dwc;
717 unsigned long flags;
718 int ret;
719
720 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
721 pr_debug("dwc3: invalid parameters\n");
722 return -EINVAL;
723 }
724
725 if (!desc->wMaxPacketSize) {
726 pr_debug("dwc3: missing wMaxPacketSize\n");
727 return -EINVAL;
728 }
729
730 dep = to_dwc3_ep(ep);
731 dwc = dep->dwc;
732
95ca961c
FB		 733 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
734 "%s is already enabled\n",
735 dep->name))
c6f83f38 736 return 0;
c6f83f38 737
72246da4 738 spin_lock_irqsave(&dwc->lock, flags);
265b70a7 739 ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false);
72246da4
FB		 740 spin_unlock_irqrestore(&dwc->lock, flags);
741
742 return ret;
743}
744
745static int dwc3_gadget_ep_disable(struct usb_ep *ep)
746{
747 struct dwc3_ep *dep;
748 struct dwc3 *dwc;
749 unsigned long flags;
750 int ret;
751
752 if (!ep) {
753 pr_debug("dwc3: invalid parameters\n");
754 return -EINVAL;
755 }
756
757 dep = to_dwc3_ep(ep);
758 dwc = dep->dwc;
759
95ca961c
FB		 760 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
761 "%s is already disabled\n",
762 dep->name))
72246da4 763 return 0;
72246da4 764
72246da4
FB		 765 spin_lock_irqsave(&dwc->lock, flags);
766 ret = __dwc3_gadget_ep_disable(dep);
767 spin_unlock_irqrestore(&dwc->lock, flags);
768
769 return ret;
770}
771
772static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
773 gfp_t gfp_flags)
774{
775 struct dwc3_request *req;
776 struct dwc3_ep *dep = to_dwc3_ep(ep);
72246da4
FB		 777
778 req = kzalloc(sizeof(*req), gfp_flags);
734d5a53 779 if (!req)
72246da4 780 return NULL;
72246da4
FB		 781
782 req->epnum = dep->number;
783 req->dep = dep;
72246da4 784
2c4cbe6e
FB		 785 trace_dwc3_alloc_request(req);
786
72246da4
FB		 787 return &req->request;
788}
789
790static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
791 struct usb_request *request)
792{
793 struct dwc3_request *req = to_dwc3_request(request);
794
2c4cbe6e 795 trace_dwc3_free_request(req);
72246da4
FB		 796 kfree(req);
797}
798
c71fc37c
FB		 799/**
800 * dwc3_prepare_one_trb - setup one TRB from one request
801 * @dep: endpoint for which this request is prepared
802 * @req: dwc3_request pointer
803 */
68e823e2 804static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
eeb720fb 805 struct dwc3_request *req, dma_addr_t dma,
e5ba5ec8 806 unsigned length, unsigned last, unsigned chain, unsigned node)
c71fc37c 807{
f6bafc6a 808 struct dwc3_trb *trb;
c71fc37c 809
73815280 810 dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s%s",
eeb720fb
FB		 811 dep->name, req, (unsigned long long) dma,
812 length, last ? " last" : "",
813 chain ? " chain" : "");
814
915e202a 815
4faf7550 816 trb = &dep->trb_pool[dep->trb_enqueue];
c71fc37c 817
eeb720fb 818 if (!req->trb) {
aa3342c8 819 dwc3_gadget_move_started_request(req);
f6bafc6a
FB		 820 req->trb = trb;
821 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
4faf7550 822 req->first_trb_index = dep->trb_enqueue;
eeb720fb 823 }
c71fc37c 824
ef966b9d 825 dwc3_ep_inc_enq(dep);
36b68aae
FB		 826 /* Skip the LINK-TRB */
827 if (dwc3_ep_is_last_trb(dep->trb_enqueue))
ef966b9d 828 dwc3_ep_inc_enq(dep);
e5ba5ec8 829
f6bafc6a
FB		 830 trb->size = DWC3_TRB_SIZE_LENGTH(length);
831 trb->bpl = lower_32_bits(dma);
832 trb->bph = upper_32_bits(dma);
c71fc37c 833
16e78db7 834 switch (usb_endpoint_type(dep->endpoint.desc)) {
c71fc37c 835 case USB_ENDPOINT_XFER_CONTROL:
f6bafc6a 836 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
c71fc37c
FB		 837 break;
838
839 case USB_ENDPOINT_XFER_ISOC:
e5ba5ec8
PA		 840 if (!node)
841 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
842 else
843 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
ca4d44ea
FB		 844
845 /* always enable Interrupt on Missed ISOC */
846 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
c71fc37c
FB		 847 break;
848
849 case USB_ENDPOINT_XFER_BULK:
850 case USB_ENDPOINT_XFER_INT:
f6bafc6a 851 trb->ctrl = DWC3_TRBCTL_NORMAL;
c71fc37c
FB		 852 break;
853 default:
854 /*
855 * This is only possible with faulty memory because we
856 * checked it already :)
857 */
858 BUG();
859 }
860
ca4d44ea
FB		 861 /* always enable Continue on Short Packet */
862 trb->ctrl |= DWC3_TRB_CTRL_CSP;
f3af3651 863
f3af3651 864 if (!req->request.no_interrupt && !chain)
ca4d44ea 865 trb->ctrl |= DWC3_TRB_CTRL_IOC | DWC3_TRB_CTRL_ISP_IMI;
f3af3651 866
ca4d44ea 867 if (last)
e5ba5ec8 868 trb->ctrl |= DWC3_TRB_CTRL_LST;
c71fc37c 869
e5ba5ec8
PA		 870 if (chain)
871 trb->ctrl |= DWC3_TRB_CTRL_CHN;
872
16e78db7 873 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
f6bafc6a 874 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
c71fc37c 875
f6bafc6a 876 trb->ctrl |= DWC3_TRB_CTRL_HWO;
2c4cbe6e
FB		 877
878 trace_dwc3_prepare_trb(dep, trb);
c71fc37c
FB		 879}
880
c4233573
FB		 881static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
882{
883 struct dwc3_trb *tmp;
884
885 /*
886 * If enqueue & dequeue are equal than it is either full or empty.
887 *
888 * One way to know for sure is if the TRB right before us has HWO bit
889 * set or not. If it has, then we're definitely full and can't fit any
890 * more transfers in our ring.
891 */
892 if (dep->trb_enqueue == dep->trb_dequeue) {
893 /* If we're full, enqueue/dequeue are > 0 */
894 if (dep->trb_enqueue) {
895 tmp = &dep->trb_pool[dep->trb_enqueue - 1];
896 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
897 return 0;
898 }
899
900 return DWC3_TRB_NUM - 1;
901 }
902
903 return dep->trb_dequeue - dep->trb_enqueue;
904}
905
72246da4
FB		 906/*
907 * dwc3_prepare_trbs - setup TRBs from requests
908 * @dep: endpoint for which requests are being prepared
72246da4 909 *
1d046793
PZ		 910 * The function goes through the requests list and sets up TRBs for the
911 * transfers. The function returns once there are no more TRBs available or
912 * it runs out of requests.
72246da4 913 */
c4233573 914static void dwc3_prepare_trbs(struct dwc3_ep *dep)
72246da4 915{
68e823e2 916 struct dwc3_request *req, *n;
72246da4 917 u32 trbs_left;
c71fc37c 918 unsigned int last_one = 0;
72246da4
FB		 919
920 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
921
c4233573 922 trbs_left = dwc3_calc_trbs_left(dep);
72246da4 923
aa3342c8 924 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
eeb720fb
FB		 925 unsigned length;
926 dma_addr_t dma;
e5ba5ec8 927 last_one = false;
72246da4 928
eeb720fb
FB		 929 if (req->request.num_mapped_sgs > 0) {
930 struct usb_request *request = &req->request;
931 struct scatterlist *sg = request->sg;
932 struct scatterlist *s;
933 int i;
72246da4 934
eeb720fb
FB		 935 for_each_sg(sg, s, request->num_mapped_sgs, i) {
936 unsigned chain = true;
72246da4 937
eeb720fb
FB		 938 length = sg_dma_len(s);
939 dma = sg_dma_address(s);
72246da4 940
6aff4832
FB		 941 if (sg_is_last(s)) {
942 if (list_is_last(&req->list, &dep->pending_list))
e5ba5ec8 943 last_one = true;
6aff4832 944
eeb720fb
FB		 945 chain = false;
946 }
72246da4 947
eeb720fb
FB		 948 trbs_left--;
949 if (!trbs_left)
950 last_one = true;
72246da4 951
eeb720fb
FB		 952 if (last_one)
953 chain = false;
72246da4 954
eeb720fb 955 dwc3_prepare_one_trb(dep, req, dma, length,
e5ba5ec8 956 last_one, chain, i);
72246da4 957
eeb720fb
FB		 958 if (last_one)
959 break;
960 }
39e60635
AV		 961
962 if (last_one)
963 break;
72246da4 964 } else {
eeb720fb
FB		 965 dma = req->request.dma;
966 length = req->request.length;
967 trbs_left--;
72246da4 968
eeb720fb 969 if (!trbs_left)
6aff4832 970 last_one = true;
879631aa 971
eeb720fb 972 /* Is this the last request? */
aa3342c8 973 if (list_is_last(&req->list, &dep->pending_list))
6aff4832 974 last_one = true;
72246da4 975
eeb720fb 976 dwc3_prepare_one_trb(dep, req, dma, length,
e5ba5ec8 977 last_one, false, 0);
72246da4 978
eeb720fb
FB		 979 if (last_one)
980 break;
72246da4 981 }
72246da4 982 }
72246da4
FB		 983}
984
4fae2e3e 985static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param)
72246da4
FB		 986{
987 struct dwc3_gadget_ep_cmd_params params;
988 struct dwc3_request *req;
989 struct dwc3 *dwc = dep->dwc;
4fae2e3e 990 int starting;
72246da4
FB		 991 int ret;
992 u32 cmd;
993
4fae2e3e 994 starting = !(dep->flags & DWC3_EP_BUSY);
72246da4 995
4fae2e3e
FB		 996 dwc3_prepare_trbs(dep);
997 req = next_request(&dep->started_list);
72246da4
FB		 998 if (!req) {
999 dep->flags |= DWC3_EP_PENDING_REQUEST;
1000 return 0;
1001 }
1002
1003 memset(&params, 0, sizeof(params));
72246da4 1004
4fae2e3e 1005 if (starting) {
1877d6c9
PA		 1006 params.param0 = upper_32_bits(req->trb_dma);
1007 params.param1 = lower_32_bits(req->trb_dma);
72246da4 1008 cmd = DWC3_DEPCMD_STARTTRANSFER;
1877d6c9 1009 } else {
72246da4 1010 cmd = DWC3_DEPCMD_UPDATETRANSFER;
1877d6c9 1011 }
72246da4
FB		 1012
1013 cmd |= DWC3_DEPCMD_PARAM(cmd_param);
1014 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
1015 if (ret < 0) {
72246da4
FB		 1016 /*
1017 * FIXME we need to iterate over the list of requests
1018 * here and stop, unmap, free and del each of the linked
1d046793 1019 * requests instead of what we do now.
72246da4 1020 */
0fc9a1be
FB		 1021 usb_gadget_unmap_request(&dwc->gadget, &req->request,
1022 req->direction);
72246da4
FB		 1023 list_del(&req->list);
1024 return ret;
1025 }
1026
1027 dep->flags |= DWC3_EP_BUSY;
25b8ff68 1028
4fae2e3e 1029 if (starting) {
b4996a86 1030 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dwc,
f898ae09 1031 dep->number);
b4996a86 1032 WARN_ON_ONCE(!dep->resource_index);
f898ae09 1033 }
25b8ff68 1034
72246da4
FB		 1035 return 0;
1036}
1037
d6d6ec7b
PA		 1038static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1039 struct dwc3_ep *dep, u32 cur_uf)
1040{
1041 u32 uf;
1042
aa3342c8 1043 if (list_empty(&dep->pending_list)) {
73815280
FB		 1044 dwc3_trace(trace_dwc3_gadget,
1045 "ISOC ep %s run out for requests",
1046 dep->name);
f4a53c55 1047 dep->flags |= DWC3_EP_PENDING_REQUEST;
d6d6ec7b
PA		 1048 return;
1049 }
1050
1051 /* 4 micro frames in the future */
1052 uf = cur_uf + dep->interval * 4;
1053
4fae2e3e 1054 __dwc3_gadget_kick_transfer(dep, uf);
d6d6ec7b
PA		 1055}
1056
1057static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1058 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1059{
1060 u32 cur_uf, mask;
1061
1062 mask = ~(dep->interval - 1);
1063 cur_uf = event->parameters & mask;
1064
1065 __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1066}
1067
72246da4
FB		 1068static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1069{
0fc9a1be
FB		 1070 struct dwc3 *dwc = dep->dwc;
1071 int ret;
1072
bb423984 1073 if (!dep->endpoint.desc) {
ec5e795c
FB		 1074 dwc3_trace(trace_dwc3_gadget,
1075 "trying to queue request %p to disabled %s\n",
bb423984
FB		 1076 &req->request, dep->endpoint.name);
1077 return -ESHUTDOWN;
1078 }
1079
1080 if (WARN(req->dep != dep, "request %p belongs to '%s'\n",
1081 &req->request, req->dep->name)) {
ec5e795c
FB		 1082 dwc3_trace(trace_dwc3_gadget, "request %p belongs to '%s'\n",
1083 &req->request, req->dep->name);
bb423984
FB		 1084 return -EINVAL;
1085 }
1086
72246da4
FB		 1087 req->request.actual = 0;
1088 req->request.status = -EINPROGRESS;
1089 req->direction = dep->direction;
1090 req->epnum = dep->number;
1091
fe84f522
FB		 1092 trace_dwc3_ep_queue(req);
1093
72246da4
FB		 1094 /*
1095 * We only add to our list of requests now and
1096 * start consuming the list once we get XferNotReady
1097 * IRQ.
1098 *
1099 * That way, we avoid doing anything that we don't need
1100 * to do now and defer it until the point we receive a
1101 * particular token from the Host side.
1102 *
1103 * This will also avoid Host cancelling URBs due to too
1d046793 1104 * many NAKs.
72246da4 1105 */
0fc9a1be
FB		 1106 ret = usb_gadget_map_request(&dwc->gadget, &req->request,
1107 dep->direction);
1108 if (ret)
1109 return ret;
1110
aa3342c8 1111 list_add_tail(&req->list, &dep->pending_list);
72246da4 1112
1d6a3918
FB		 1113 /*
1114 * If there are no pending requests and the endpoint isn't already
1115 * busy, we will just start the request straight away.
1116 *
1117 * This will save one IRQ (XFER_NOT_READY) and possibly make it a
1118 * little bit faster.
1119 */
1120 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
62e345ae 1121 !usb_endpoint_xfer_int(dep->endpoint.desc) &&
1d6a3918 1122 !(dep->flags & DWC3_EP_BUSY)) {
4fae2e3e 1123 ret = __dwc3_gadget_kick_transfer(dep, 0);
a8f32817 1124 goto out;
1d6a3918
FB		 1125 }
1126
72246da4 1127 /*
b511e5e7 1128 * There are a few special cases:
72246da4 1129 *
f898ae09
PZ		 1130 * 1. XferNotReady with empty list of requests. We need to kick the
1131 * transfer here in that situation, otherwise we will be NAKing
1132 * forever. If we get XferNotReady before gadget driver has a
1133 * chance to queue a request, we will ACK the IRQ but won't be
1134 * able to receive the data until the next request is queued.
1135 * The following code is handling exactly that.
72246da4 1136 *
72246da4
FB		 1137 */
1138 if (dep->flags & DWC3_EP_PENDING_REQUEST) {
f4a53c55
PA		 1139 /*
1140 * If xfernotready is already elapsed and it is a case
1141 * of isoc transfer, then issue END TRANSFER, so that
1142 * you can receive xfernotready again and can have
1143 * notion of current microframe.
1144 */
1145 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
aa3342c8 1146 if (list_empty(&dep->started_list)) {
b992e681 1147 dwc3_stop_active_transfer(dwc, dep->number, true);
cdc359dd
PA		 1148 dep->flags = DWC3_EP_ENABLED;
1149 }
f4a53c55
PA		 1150 return 0;
1151 }
1152
4fae2e3e 1153 ret = __dwc3_gadget_kick_transfer(dep, 0);
89185916
FB		 1154 if (!ret)
1155 dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1156
a8f32817 1157 goto out;
b511e5e7 1158 }
72246da4 1159
b511e5e7
FB		 1160 /*
1161 * 2. XferInProgress on Isoc EP with an active transfer. We need to
1162 * kick the transfer here after queuing a request, otherwise the
1163 * core may not see the modified TRB(s).
1164 */
1165 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
79c9046e
PA		 1166 (dep->flags & DWC3_EP_BUSY) &&
1167 !(dep->flags & DWC3_EP_MISSED_ISOC)) {
b4996a86 1168 WARN_ON_ONCE(!dep->resource_index);
4fae2e3e 1169 ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index);
a8f32817 1170 goto out;
a0925324 1171 }
72246da4 1172
b997ada5
FB		 1173 /*
1174 * 4. Stream Capable Bulk Endpoints. We need to start the transfer
1175 * right away, otherwise host will not know we have streams to be
1176 * handled.
1177 */
a8f32817 1178 if (dep->stream_capable)
4fae2e3e 1179 ret = __dwc3_gadget_kick_transfer(dep, 0);
b997ada5 1180
a8f32817
FB		 1181out:
1182 if (ret && ret != -EBUSY)
ec5e795c
FB		 1183 dwc3_trace(trace_dwc3_gadget,
1184 "%s: failed to kick transfers\n",
a8f32817
FB		 1185 dep->name);
1186 if (ret == -EBUSY)
1187 ret = 0;
1188
1189 return ret;
72246da4
FB		 1190}
1191
04c03d10
FB		 1192static void __dwc3_gadget_ep_zlp_complete(struct usb_ep *ep,
1193 struct usb_request *request)
1194{
1195 dwc3_gadget_ep_free_request(ep, request);
1196}
1197
1198static int __dwc3_gadget_ep_queue_zlp(struct dwc3 *dwc, struct dwc3_ep *dep)
1199{
1200 struct dwc3_request *req;
1201 struct usb_request *request;
1202 struct usb_ep *ep = &dep->endpoint;
1203
1204 dwc3_trace(trace_dwc3_gadget, "queueing ZLP\n");
1205 request = dwc3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
1206 if (!request)
1207 return -ENOMEM;
1208
1209 request->length = 0;
1210 request->buf = dwc->zlp_buf;
1211 request->complete = __dwc3_gadget_ep_zlp_complete;
1212
1213 req = to_dwc3_request(request);
1214
1215 return __dwc3_gadget_ep_queue(dep, req);
1216}
1217
72246da4
FB		 1218static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1219 gfp_t gfp_flags)
1220{
1221 struct dwc3_request *req = to_dwc3_request(request);
1222 struct dwc3_ep *dep = to_dwc3_ep(ep);
1223 struct dwc3 *dwc = dep->dwc;
1224
1225 unsigned long flags;
1226
1227 int ret;
1228
fdee4eba 1229 spin_lock_irqsave(&dwc->lock, flags);
72246da4 1230 ret = __dwc3_gadget_ep_queue(dep, req);
04c03d10
FB		 1231
1232 /*
1233 * Okay, here's the thing, if gadget driver has requested for a ZLP by
1234 * setting request->zero, instead of doing magic, we will just queue an
1235 * extra usb_request ourselves so that it gets handled the same way as
1236 * any other request.
1237 */
d9261898
JY		 1238 if (ret == 0 && request->zero && request->length &&
1239 (request->length % ep->maxpacket == 0))
04c03d10
FB		 1240 ret = __dwc3_gadget_ep_queue_zlp(dwc, dep);
1241
72246da4
FB		 1242 spin_unlock_irqrestore(&dwc->lock, flags);
1243
1244 return ret;
1245}
1246
1247static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1248 struct usb_request *request)
1249{
1250 struct dwc3_request *req = to_dwc3_request(request);
1251 struct dwc3_request *r = NULL;
1252
1253 struct dwc3_ep *dep = to_dwc3_ep(ep);
1254 struct dwc3 *dwc = dep->dwc;
1255
1256 unsigned long flags;
1257 int ret = 0;
1258
2c4cbe6e
FB		 1259 trace_dwc3_ep_dequeue(req);
1260
72246da4
FB		 1261 spin_lock_irqsave(&dwc->lock, flags);
1262
aa3342c8 1263 list_for_each_entry(r, &dep->pending_list, list) {
72246da4
FB		 1264 if (r == req)
1265 break;
1266 }
1267
1268 if (r != req) {
aa3342c8 1269 list_for_each_entry(r, &dep->started_list, list) {
72246da4
FB		 1270 if (r == req)
1271 break;
1272 }
1273 if (r == req) {
1274 /* wait until it is processed */
b992e681 1275 dwc3_stop_active_transfer(dwc, dep->number, true);
e8d4e8be 1276 goto out1;
72246da4
FB		 1277 }
1278 dev_err(dwc->dev, "request %p was not queued to %s\n",
1279 request, ep->name);
1280 ret = -EINVAL;
1281 goto out0;
1282 }
1283
e8d4e8be 1284out1:
72246da4
FB		 1285 /* giveback the request */
1286 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1287
1288out0:
1289 spin_unlock_irqrestore(&dwc->lock, flags);
1290
1291 return ret;
1292}
1293
7a608559 1294int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
72246da4
FB		 1295{
1296 struct dwc3_gadget_ep_cmd_params params;
1297 struct dwc3 *dwc = dep->dwc;
1298 int ret;
1299
5ad02fb8
FB		 1300 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1301 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1302 return -EINVAL;
1303 }
1304
72246da4
FB		 1305 memset(&params, 0x00, sizeof(params));
1306
1307 if (value) {
7a608559 1308 if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) ||
aa3342c8
FB		 1309 (!list_empty(&dep->started_list) ||
1310 !list_empty(&dep->pending_list)))) {
ec5e795c 1311 dwc3_trace(trace_dwc3_gadget,
052ba52e 1312 "%s: pending request, cannot halt",
7a608559
FB		 1313 dep->name);
1314 return -EAGAIN;
1315 }
1316
72246da4
FB		 1317 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1318 DWC3_DEPCMD_SETSTALL, &params);
1319 if (ret)
3f89204b 1320 dev_err(dwc->dev, "failed to set STALL on %s\n",
72246da4
FB		 1321 dep->name);
1322 else
1323 dep->flags |= DWC3_EP_STALL;
1324 } else {
50c763f8 1325 ret = dwc3_send_clear_stall_ep_cmd(dep);
72246da4 1326 if (ret)
3f89204b 1327 dev_err(dwc->dev, "failed to clear STALL on %s\n",
72246da4
FB		 1328 dep->name);
1329 else
a535d81c 1330 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
72246da4 1331 }
5275455a 1332
72246da4
FB		 1333 return ret;
1334}
1335
1336static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1337{
1338 struct dwc3_ep *dep = to_dwc3_ep(ep);
1339 struct dwc3 *dwc = dep->dwc;
1340
1341 unsigned long flags;
1342
1343 int ret;
1344
1345 spin_lock_irqsave(&dwc->lock, flags);
7a608559 1346 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
72246da4
FB		 1347 spin_unlock_irqrestore(&dwc->lock, flags);
1348
1349 return ret;
1350}
1351
1352static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1353{
1354 struct dwc3_ep *dep = to_dwc3_ep(ep);
249a4569
PZ		 1355 struct dwc3 *dwc = dep->dwc;
1356 unsigned long flags;
95aa4e8d 1357 int ret;
72246da4 1358
249a4569 1359 spin_lock_irqsave(&dwc->lock, flags);
72246da4
FB		 1360 dep->flags |= DWC3_EP_WEDGE;
1361
08f0d966 1362 if (dep->number == 0 || dep->number == 1)
95aa4e8d 1363 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
08f0d966 1364 else
7a608559 1365 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
95aa4e8d
FB		 1366 spin_unlock_irqrestore(&dwc->lock, flags);
1367
1368 return ret;
72246da4
FB		 1369}
1370
1371/* -------------------------------------------------------------------------- */
1372
1373static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1374 .bLength = USB_DT_ENDPOINT_SIZE,
1375 .bDescriptorType = USB_DT_ENDPOINT,
1376 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1377};
1378
1379static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1380 .enable = dwc3_gadget_ep0_enable,
1381 .disable = dwc3_gadget_ep0_disable,
1382 .alloc_request = dwc3_gadget_ep_alloc_request,
1383 .free_request = dwc3_gadget_ep_free_request,
1384 .queue = dwc3_gadget_ep0_queue,
1385 .dequeue = dwc3_gadget_ep_dequeue,
08f0d966 1386 .set_halt = dwc3_gadget_ep0_set_halt,
72246da4
FB		 1387 .set_wedge = dwc3_gadget_ep_set_wedge,
1388};
1389
1390static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1391 .enable = dwc3_gadget_ep_enable,
1392 .disable = dwc3_gadget_ep_disable,
1393 .alloc_request = dwc3_gadget_ep_alloc_request,
1394 .free_request = dwc3_gadget_ep_free_request,
1395 .queue = dwc3_gadget_ep_queue,
1396 .dequeue = dwc3_gadget_ep_dequeue,
1397 .set_halt = dwc3_gadget_ep_set_halt,
1398 .set_wedge = dwc3_gadget_ep_set_wedge,
1399};
1400
1401/* -------------------------------------------------------------------------- */
1402
1403static int dwc3_gadget_get_frame(struct usb_gadget *g)
1404{
1405 struct dwc3 *dwc = gadget_to_dwc(g);
1406 u32 reg;
1407
1408 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1409 return DWC3_DSTS_SOFFN(reg);
1410}
1411
218ef7b6 1412static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
72246da4 1413{
72246da4 1414 unsigned long timeout;
72246da4 1415
218ef7b6 1416 int ret;
72246da4
FB		 1417 u32 reg;
1418
72246da4
FB		 1419 u8 link_state;
1420 u8 speed;
1421
72246da4
FB		 1422 /*
1423 * According to the Databook Remote wakeup request should
1424 * be issued only when the device is in early suspend state.
1425 *
1426 * We can check that via USB Link State bits in DSTS register.
1427 */
1428 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1429
1430 speed = reg & DWC3_DSTS_CONNECTSPD;
ee5cd41c
JY		 1431 if ((speed == DWC3_DSTS_SUPERSPEED) ||
1432 (speed == DWC3_DSTS_SUPERSPEED_PLUS)) {
ec5e795c 1433 dwc3_trace(trace_dwc3_gadget, "no wakeup on SuperSpeed\n");
6b742899 1434 return 0;
72246da4
FB		 1435 }
1436
1437 link_state = DWC3_DSTS_USBLNKST(reg);
1438
1439 switch (link_state) {
1440 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
1441 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
1442 break;
1443 default:
ec5e795c
FB		 1444 dwc3_trace(trace_dwc3_gadget,
1445 "can't wakeup from '%s'\n",
1446 dwc3_gadget_link_string(link_state));
218ef7b6 1447 return -EINVAL;
72246da4
FB		 1448 }
1449
8598bde7
FB		 1450 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1451 if (ret < 0) {
1452 dev_err(dwc->dev, "failed to put link in Recovery\n");
218ef7b6 1453 return ret;
8598bde7 1454 }
72246da4 1455
802fde98
PZ		 1456 /* Recent versions do this automatically */
1457 if (dwc->revision < DWC3_REVISION_194A) {
1458 /* write zeroes to Link Change Request */
fcc023c7 1459 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
802fde98
PZ		 1460 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1461 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1462 }
72246da4 1463
1d046793 1464 /* poll until Link State changes to ON */
72246da4
FB		 1465 timeout = jiffies + msecs_to_jiffies(100);
1466
1d046793 1467 while (!time_after(jiffies, timeout)) {
72246da4
FB		 1468 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1469
1470 /* in HS, means ON */
1471 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1472 break;
1473 }
1474
1475 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1476 dev_err(dwc->dev, "failed to send remote wakeup\n");
218ef7b6 1477 return -EINVAL;
72246da4
FB		 1478 }
1479
218ef7b6
FB		 1480 return 0;
1481}
1482
1483static int dwc3_gadget_wakeup(struct usb_gadget *g)
1484{
1485 struct dwc3 *dwc = gadget_to_dwc(g);
1486 unsigned long flags;
1487 int ret;
1488
1489 spin_lock_irqsave(&dwc->lock, flags);
1490 ret = __dwc3_gadget_wakeup(dwc);
72246da4
FB		 1491 spin_unlock_irqrestore(&dwc->lock, flags);
1492
1493 return ret;
1494}
1495
1496static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1497 int is_selfpowered)
1498{
1499 struct dwc3 *dwc = gadget_to_dwc(g);
249a4569 1500 unsigned long flags;
72246da4 1501
249a4569 1502 spin_lock_irqsave(&dwc->lock, flags);
bcdea503 1503 g->is_selfpowered = !!is_selfpowered;
249a4569 1504 spin_unlock_irqrestore(&dwc->lock, flags);
72246da4
FB		 1505
1506 return 0;
1507}
1508
7b2a0368 1509static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
72246da4
FB		 1510{
1511 u32 reg;
61d58242 1512 u32 timeout = 500;
72246da4
FB		 1513
1514 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
8db7ed15 1515 if (is_on) {
802fde98
PZ		 1516 if (dwc->revision <= DWC3_REVISION_187A) {
1517 reg &= ~DWC3_DCTL_TRGTULST_MASK;
1518 reg |= DWC3_DCTL_TRGTULST_RX_DET;
1519 }
1520
1521 if (dwc->revision >= DWC3_REVISION_194A)
1522 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1523 reg |= DWC3_DCTL_RUN_STOP;
7b2a0368
FB		 1524
1525 if (dwc->has_hibernation)
1526 reg |= DWC3_DCTL_KEEP_CONNECT;
1527
9fcb3bd8 1528 dwc->pullups_connected = true;
8db7ed15 1529 } else {
72246da4 1530 reg &= ~DWC3_DCTL_RUN_STOP;
7b2a0368
FB		 1531
1532 if (dwc->has_hibernation && !suspend)
1533 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1534
9fcb3bd8 1535 dwc->pullups_connected = false;
8db7ed15 1536 }
72246da4
FB		 1537
1538 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1539
1540 do {
1541 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1542 if (is_on) {
1543 if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1544 break;
1545 } else {
1546 if (reg & DWC3_DSTS_DEVCTRLHLT)
1547 break;
1548 }
72246da4
FB		 1549 timeout--;
1550 if (!timeout)
6f17f74b 1551 return -ETIMEDOUT;
61d58242 1552 udelay(1);
72246da4
FB		 1553 } while (1);
1554
73815280 1555 dwc3_trace(trace_dwc3_gadget, "gadget %s data soft-%s",
72246da4
FB		 1556 dwc->gadget_driver
1557 ? dwc->gadget_driver->function : "no-function",
1558 is_on ? "connect" : "disconnect");
6f17f74b
PA		 1559
1560 return 0;
72246da4
FB		 1561}
1562
1563static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1564{
1565 struct dwc3 *dwc = gadget_to_dwc(g);
1566 unsigned long flags;
6f17f74b 1567 int ret;
72246da4
FB		 1568
1569 is_on = !!is_on;
1570
1571 spin_lock_irqsave(&dwc->lock, flags);
7b2a0368 1572 ret = dwc3_gadget_run_stop(dwc, is_on, false);
72246da4
FB		 1573 spin_unlock_irqrestore(&dwc->lock, flags);
1574
6f17f74b 1575 return ret;
72246da4
FB		 1576}
1577
8698e2ac
FB		 1578static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1579{
1580 u32 reg;
1581
1582 /* Enable all but Start and End of Frame IRQs */
1583 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1584 DWC3_DEVTEN_EVNTOVERFLOWEN |
1585 DWC3_DEVTEN_CMDCMPLTEN |
1586 DWC3_DEVTEN_ERRTICERREN |
1587 DWC3_DEVTEN_WKUPEVTEN |
1588 DWC3_DEVTEN_ULSTCNGEN |
1589 DWC3_DEVTEN_CONNECTDONEEN |
1590 DWC3_DEVTEN_USBRSTEN |
1591 DWC3_DEVTEN_DISCONNEVTEN);
1592
1593 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1594}
1595
1596static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1597{
1598 /* mask all interrupts */
1599 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1600}
1601
1602static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
b15a762f 1603static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
8698e2ac 1604
d7be2952 1605static int __dwc3_gadget_start(struct dwc3 *dwc)
72246da4 1606{
72246da4 1607 struct dwc3_ep *dep;
72246da4
FB		 1608 int ret = 0;
1609 u32 reg;
1610
72246da4
FB		 1611 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1612 reg &= ~(DWC3_DCFG_SPEED_MASK);
07e7f47b
FB		 1613
1614 /**
1615 * WORKAROUND: DWC3 revision < 2.20a have an issue
1616 * which would cause metastability state on Run/Stop
1617 * bit if we try to force the IP to USB2-only mode.
1618 *
1619 * Because of that, we cannot configure the IP to any
1620 * speed other than the SuperSpeed
1621 *
1622 * Refers to:
1623 *
1624 * STAR#9000525659: Clock Domain Crossing on DCTL in
1625 * USB 2.0 Mode
1626 */
f7e846f0 1627 if (dwc->revision < DWC3_REVISION_220A) {
07e7f47b 1628 reg |= DWC3_DCFG_SUPERSPEED;
f7e846f0
FB		 1629 } else {
1630 switch (dwc->maximum_speed) {
1631 case USB_SPEED_LOW:
1632 reg |= DWC3_DSTS_LOWSPEED;
1633 break;
1634 case USB_SPEED_FULL:
1635 reg |= DWC3_DSTS_FULLSPEED1;
1636 break;
1637 case USB_SPEED_HIGH:
1638 reg |= DWC3_DSTS_HIGHSPEED;
1639 break;
7580862b
JY		 1640 case USB_SPEED_SUPER_PLUS:
1641 reg |= DWC3_DSTS_SUPERSPEED_PLUS;
1642 break;
f7e846f0 1643 default:
77966eb8
JY		 1644 dev_err(dwc->dev, "invalid dwc->maximum_speed (%d)\n",
1645 dwc->maximum_speed);
1646 /* fall through */
1647 case USB_SPEED_SUPER:
1648 reg |= DWC3_DCFG_SUPERSPEED;
1649 break;
f7e846f0
FB		 1650 }
1651 }
72246da4
FB		 1652 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1653
2a58f9c1
FB		 1654 /*
1655 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1656 * field instead of letting dwc3 itself calculate that automatically.
1657 *
1658 * This way, we maximize the chances that we'll be able to get several
1659 * bursts of data without going through any sort of endpoint throttling.
1660 */
1661 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1662 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1663 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1664
72246da4
FB		 1665 /* Start with SuperSpeed Default */
1666 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1667
1668 dep = dwc->eps[0];
265b70a7
PZ		 1669 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1670 false);
72246da4
FB		 1671 if (ret) {
1672 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
d7be2952 1673 goto err0;
72246da4
FB		 1674 }
1675
1676 dep = dwc->eps[1];
265b70a7
PZ		 1677 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1678 false);
72246da4
FB		 1679 if (ret) {
1680 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
d7be2952 1681 goto err1;
72246da4
FB		 1682 }
1683
1684 /* begin to receive SETUP packets */
c7fcdeb2 1685 dwc->ep0state = EP0_SETUP_PHASE;
72246da4
FB		 1686 dwc3_ep0_out_start(dwc);
1687
8698e2ac
FB		 1688 dwc3_gadget_enable_irq(dwc);
1689
72246da4
FB		 1690 return 0;
1691
b0d7ffd4 1692err1:
d7be2952 1693 __dwc3_gadget_ep_disable(dwc->eps[0]);
b0d7ffd4
FB		 1694
1695err0:
72246da4
FB		 1696 return ret;
1697}
1698
d7be2952
FB		 1699static int dwc3_gadget_start(struct usb_gadget *g,
1700 struct usb_gadget_driver *driver)
72246da4
FB		 1701{
1702 struct dwc3 *dwc = gadget_to_dwc(g);
1703 unsigned long flags;
d7be2952 1704 int ret = 0;
8698e2ac 1705 int irq;
72246da4 1706
d7be2952
FB		 1707 irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1708 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1709 IRQF_SHARED, "dwc3", dwc->ev_buf);
1710 if (ret) {
1711 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1712 irq, ret);
1713 goto err0;
1714 }
1715
72246da4 1716 spin_lock_irqsave(&dwc->lock, flags);
d7be2952
FB		 1717 if (dwc->gadget_driver) {
1718 dev_err(dwc->dev, "%s is already bound to %s\n",
1719 dwc->gadget.name,
1720 dwc->gadget_driver->driver.name);
1721 ret = -EBUSY;
1722 goto err1;
1723 }
1724
1725 dwc->gadget_driver = driver;
1726
1727 __dwc3_gadget_start(dwc);
1728 spin_unlock_irqrestore(&dwc->lock, flags);
1729
1730 return 0;
1731
1732err1:
1733 spin_unlock_irqrestore(&dwc->lock, flags);
1734 free_irq(irq, dwc);
1735
1736err0:
1737 return ret;
1738}
72246da4 1739
d7be2952
FB		 1740static void __dwc3_gadget_stop(struct dwc3 *dwc)
1741{
8698e2ac 1742 dwc3_gadget_disable_irq(dwc);
72246da4
FB		 1743 __dwc3_gadget_ep_disable(dwc->eps[0]);
1744 __dwc3_gadget_ep_disable(dwc->eps[1]);
d7be2952 1745}
72246da4 1746
d7be2952
FB		 1747static int dwc3_gadget_stop(struct usb_gadget *g)
1748{
1749 struct dwc3 *dwc = gadget_to_dwc(g);
1750 unsigned long flags;
1751 int irq;
72246da4 1752
d7be2952
FB		 1753 spin_lock_irqsave(&dwc->lock, flags);
1754 __dwc3_gadget_stop(dwc);
1755 dwc->gadget_driver = NULL;
72246da4
FB		 1756 spin_unlock_irqrestore(&dwc->lock, flags);
1757
b0d7ffd4 1758 irq = platform_get_irq(to_platform_device(dwc->dev), 0);
dea520a4 1759 free_irq(irq, dwc->ev_buf);
b0d7ffd4 1760
72246da4
FB		 1761 return 0;
1762}
802fde98 1763
72246da4
FB		 1764static const struct usb_gadget_ops dwc3_gadget_ops = {
1765 .get_frame = dwc3_gadget_get_frame,
1766 .wakeup = dwc3_gadget_wakeup,
1767 .set_selfpowered = dwc3_gadget_set_selfpowered,
1768 .pullup = dwc3_gadget_pullup,
1769 .udc_start = dwc3_gadget_start,
1770 .udc_stop = dwc3_gadget_stop,
1771};
1772
1773/* -------------------------------------------------------------------------- */
1774
6a1e3ef4
FB		 1775static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
1776 u8 num, u32 direction)
72246da4
FB		 1777{
1778 struct dwc3_ep *dep;
6a1e3ef4 1779 u8 i;
72246da4 1780
6a1e3ef4
FB		 1781 for (i = 0; i < num; i++) {
1782 u8 epnum = (i << 1) | (!!direction);
72246da4 1783
72246da4 1784 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
734d5a53 1785 if (!dep)
72246da4 1786 return -ENOMEM;
72246da4
FB		 1787
1788 dep->dwc = dwc;
1789 dep->number = epnum;
9aa62ae4 1790 dep->direction = !!direction;
72246da4
FB		 1791 dwc->eps[epnum] = dep;
1792
1793 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1794 (epnum & 1) ? "in" : "out");
6a1e3ef4 1795
72246da4 1796 dep->endpoint.name = dep->name;
72246da4 1797
73815280 1798 dwc3_trace(trace_dwc3_gadget, "initializing %s", dep->name);
653df35e 1799
72246da4 1800 if (epnum == 0 || epnum == 1) {
e117e742 1801 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
6048e4c6 1802 dep->endpoint.maxburst = 1;
72246da4
FB		 1803 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1804 if (!epnum)
1805 dwc->gadget.ep0 = &dep->endpoint;
1806 } else {
1807 int ret;
1808
e117e742 1809 usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
12d36c16 1810 dep->endpoint.max_streams = 15;
72246da4
FB		 1811 dep->endpoint.ops = &dwc3_gadget_ep_ops;
1812 list_add_tail(&dep->endpoint.ep_list,
1813 &dwc->gadget.ep_list);
1814
1815 ret = dwc3_alloc_trb_pool(dep);
25b8ff68 1816 if (ret)
72246da4 1817 return ret;
72246da4 1818 }
25b8ff68 1819
a474d3b7
RB		 1820 if (epnum == 0 || epnum == 1) {
1821 dep->endpoint.caps.type_control = true;
1822 } else {
1823 dep->endpoint.caps.type_iso = true;
1824 dep->endpoint.caps.type_bulk = true;
1825 dep->endpoint.caps.type_int = true;
1826 }
1827
1828 dep->endpoint.caps.dir_in = !!direction;
1829 dep->endpoint.caps.dir_out = !direction;
1830
aa3342c8
FB		 1831 INIT_LIST_HEAD(&dep->pending_list);
1832 INIT_LIST_HEAD(&dep->started_list);
72246da4
FB		 1833 }
1834
1835 return 0;
1836}
1837
6a1e3ef4
FB		 1838static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1839{
1840 int ret;
1841
1842 INIT_LIST_HEAD(&dwc->gadget.ep_list);
1843
1844 ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
1845 if (ret < 0) {
73815280
FB		 1846 dwc3_trace(trace_dwc3_gadget,
1847 "failed to allocate OUT endpoints");
6a1e3ef4
FB		 1848 return ret;
1849 }
1850
1851 ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
1852 if (ret < 0) {
73815280
FB		 1853 dwc3_trace(trace_dwc3_gadget,
1854 "failed to allocate IN endpoints");
6a1e3ef4
FB		 1855 return ret;
1856 }
1857
1858 return 0;
1859}
1860
72246da4
FB		 1861static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1862{
1863 struct dwc3_ep *dep;
1864 u8 epnum;
1865
1866 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1867 dep = dwc->eps[epnum];
6a1e3ef4
FB		 1868 if (!dep)
1869 continue;
5bf8fae3
GC		 1870 /*
1871 * Physical endpoints 0 and 1 are special; they form the
1872 * bi-directional USB endpoint 0.
1873 *
1874 * For those two physical endpoints, we don't allocate a TRB
1875 * pool nor do we add them the endpoints list. Due to that, we
1876 * shouldn't do these two operations otherwise we would end up
1877 * with all sorts of bugs when removing dwc3.ko.
1878 */
1879 if (epnum != 0 && epnum != 1) {
1880 dwc3_free_trb_pool(dep);
72246da4 1881 list_del(&dep->endpoint.ep_list);
5bf8fae3 1882 }
72246da4
FB		 1883
1884 kfree(dep);
1885 }
1886}
1887
72246da4 1888/* -------------------------------------------------------------------------- */
e5caff68 1889
e5ba5ec8
PA		 1890static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
1891 struct dwc3_request *req, struct dwc3_trb *trb,
72246da4
FB		 1892 const struct dwc3_event_depevt *event, int status)
1893{
72246da4
FB		 1894 unsigned int count;
1895 unsigned int s_pkt = 0;
d6d6ec7b 1896 unsigned int trb_status;
72246da4 1897
2c4cbe6e
FB		 1898 trace_dwc3_complete_trb(dep, trb);
1899
e5ba5ec8
PA		 1900 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
1901 /*
1902 * We continue despite the error. There is not much we
1903 * can do. If we don't clean it up we loop forever. If
1904 * we skip the TRB then it gets overwritten after a
1905 * while since we use them in a ring buffer. A BUG()
1906 * would help. Lets hope that if this occurs, someone
1907 * fixes the root cause instead of looking away :)
1908 */
1909 dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1910 dep->name, trb);
1911 count = trb->size & DWC3_TRB_SIZE_MASK;
1912
1913 if (dep->direction) {
1914 if (count) {
1915 trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
1916 if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
ec5e795c
FB		 1917 dwc3_trace(trace_dwc3_gadget,
1918 "%s: incomplete IN transfer\n",
e5ba5ec8
PA		 1919 dep->name);
1920 /*
1921 * If missed isoc occurred and there is
1922 * no request queued then issue END
1923 * TRANSFER, so that core generates
1924 * next xfernotready and we will issue
1925 * a fresh START TRANSFER.
1926 * If there are still queued request
1927 * then wait, do not issue either END
1928 * or UPDATE TRANSFER, just attach next
aa3342c8 1929 * request in pending_list during
e5ba5ec8
PA		 1930 * giveback.If any future queued request
1931 * is successfully transferred then we
1932 * will issue UPDATE TRANSFER for all
aa3342c8 1933 * request in the pending_list.
e5ba5ec8
PA		 1934 */
1935 dep->flags |= DWC3_EP_MISSED_ISOC;
1936 } else {
1937 dev_err(dwc->dev, "incomplete IN transfer %s\n",
1938 dep->name);
1939 status = -ECONNRESET;
1940 }
1941 } else {
1942 dep->flags &= ~DWC3_EP_MISSED_ISOC;
1943 }
1944 } else {
1945 if (count && (event->status & DEPEVT_STATUS_SHORT))
1946 s_pkt = 1;
1947 }
1948
1949 /*
1950 * We assume here we will always receive the entire data block
1951 * which we should receive. Meaning, if we program RX to
1952 * receive 4K but we receive only 2K, we assume that's all we
1953 * should receive and we simply bounce the request back to the
1954 * gadget driver for further processing.
1955 */
1956 req->request.actual += req->request.length - count;
1957 if (s_pkt)
1958 return 1;
1959 if ((event->status & DEPEVT_STATUS_LST) &&
1960 (trb->ctrl & (DWC3_TRB_CTRL_LST |
1961 DWC3_TRB_CTRL_HWO)))
1962 return 1;
1963 if ((event->status & DEPEVT_STATUS_IOC) &&
1964 (trb->ctrl & DWC3_TRB_CTRL_IOC))
1965 return 1;
1966 return 0;
1967}
1968
1969static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1970 const struct dwc3_event_depevt *event, int status)
1971{
1972 struct dwc3_request *req;
1973 struct dwc3_trb *trb;
1974 unsigned int slot;
1975 unsigned int i;
1976 int ret;
1977
72246da4 1978 do {
aa3342c8 1979 req = next_request(&dep->started_list);
ac7bdcc1 1980 if (WARN_ON_ONCE(!req))
d115d705 1981 return 1;
ac7bdcc1 1982
d115d705
VS		 1983 i = 0;
1984 do {
53fd8818 1985 slot = req->first_trb_index + i;
36b68aae 1986 if (slot == DWC3_TRB_NUM - 1)
d115d705
VS		 1987 slot++;
1988 slot %= DWC3_TRB_NUM;
1989 trb = &dep->trb_pool[slot];
1990
1991 ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
1992 event, status);
1993 if (ret)
1994 break;
1995 } while (++i < req->request.num_mapped_sgs);
1996
1997 dwc3_gadget_giveback(dep, req, status);
e5ba5ec8
PA		 1998
1999 if (ret)
72246da4 2000 break;
d115d705 2001 } while (1);
72246da4 2002
cdc359dd 2003 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
aa3342c8
FB		 2004 list_empty(&dep->started_list)) {
2005 if (list_empty(&dep->pending_list)) {
cdc359dd
PA		 2006 /*
2007 * If there is no entry in request list then do
2008 * not issue END TRANSFER now. Just set PENDING
2009 * flag, so that END TRANSFER is issued when an
2010 * entry is added into request list.
2011 */
2012 dep->flags = DWC3_EP_PENDING_REQUEST;
2013 } else {
b992e681 2014 dwc3_stop_active_transfer(dwc, dep->number, true);
cdc359dd
PA		 2015 dep->flags = DWC3_EP_ENABLED;
2016 }
7efea86c
PA		 2017 return 1;
2018 }
2019
72246da4
FB		 2020 return 1;
2021}
2022
2023static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
029d97ff 2024 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
72246da4
FB		 2025{
2026 unsigned status = 0;
2027 int clean_busy;
e18b7975
FB		 2028 u32 is_xfer_complete;
2029
2030 is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE);
72246da4
FB		 2031
2032 if (event->status & DEPEVT_STATUS_BUSERR)
2033 status = -ECONNRESET;
2034
1d046793 2035 clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
e18b7975
FB		 2036 if (clean_busy && (is_xfer_complete ||
2037 usb_endpoint_xfer_isoc(dep->endpoint.desc)))
72246da4 2038 dep->flags &= ~DWC3_EP_BUSY;
fae2b904
FB		 2039
2040 /*
2041 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2042 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2043 */
2044 if (dwc->revision < DWC3_REVISION_183A) {
2045 u32 reg;
2046 int i;
2047
2048 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
348e026f 2049 dep = dwc->eps[i];
fae2b904
FB		 2050
2051 if (!(dep->flags & DWC3_EP_ENABLED))
2052 continue;
2053
aa3342c8 2054 if (!list_empty(&dep->started_list))
fae2b904
FB		 2055 return;
2056 }
2057
2058 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2059 reg |= dwc->u1u2;
2060 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2061
2062 dwc->u1u2 = 0;
2063 }
8a1a9c9e 2064
e6e709b7 2065 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
8a1a9c9e
FB		 2066 int ret;
2067
4fae2e3e 2068 ret = __dwc3_gadget_kick_transfer(dep, 0);
8a1a9c9e
FB		 2069 if (!ret || ret == -EBUSY)
2070 return;
2071 }
72246da4
FB		 2072}
2073
72246da4
FB		 2074static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2075 const struct dwc3_event_depevt *event)
2076{
2077 struct dwc3_ep *dep;
2078 u8 epnum = event->endpoint_number;
2079
2080 dep = dwc->eps[epnum];
2081
3336abb5
FB		 2082 if (!(dep->flags & DWC3_EP_ENABLED))
2083 return;
2084
72246da4
FB		 2085 if (epnum == 0 || epnum == 1) {
2086 dwc3_ep0_interrupt(dwc, event);
2087 return;
2088 }
2089
2090 switch (event->endpoint_event) {
2091 case DWC3_DEPEVT_XFERCOMPLETE:
b4996a86 2092 dep->resource_index = 0;
c2df85ca 2093
16e78db7 2094 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
ec5e795c
FB		 2095 dwc3_trace(trace_dwc3_gadget,
2096 "%s is an Isochronous endpoint\n",
72246da4
FB		 2097 dep->name);
2098 return;
2099 }
2100
029d97ff 2101 dwc3_endpoint_transfer_complete(dwc, dep, event);
72246da4
FB		 2102 break;
2103 case DWC3_DEPEVT_XFERINPROGRESS:
029d97ff 2104 dwc3_endpoint_transfer_complete(dwc, dep, event);
72246da4
FB		 2105 break;
2106 case DWC3_DEPEVT_XFERNOTREADY:
16e78db7 2107 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
72246da4
FB		 2108 dwc3_gadget_start_isoc(dwc, dep, event);
2109 } else {
6bb4fe12 2110 int active;
72246da4
FB		 2111 int ret;
2112
6bb4fe12
FB		 2113 active = event->status & DEPEVT_STATUS_TRANSFER_ACTIVE;
2114
73815280 2115 dwc3_trace(trace_dwc3_gadget, "%s: reason %s",
6bb4fe12 2116 dep->name, active ? "Transfer Active"
72246da4
FB		 2117 : "Transfer Not Active");
2118
4fae2e3e 2119 ret = __dwc3_gadget_kick_transfer(dep, 0);
72246da4
FB		 2120 if (!ret || ret == -EBUSY)
2121 return;
2122
ec5e795c
FB		 2123 dwc3_trace(trace_dwc3_gadget,
2124 "%s: failed to kick transfers\n",
72246da4
FB		 2125 dep->name);
2126 }
2127
879631aa
FB		 2128 break;
2129 case DWC3_DEPEVT_STREAMEVT:
16e78db7 2130 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
879631aa
FB		 2131 dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
2132 dep->name);
2133 return;
2134 }
2135
2136 switch (event->status) {
2137 case DEPEVT_STREAMEVT_FOUND:
73815280
FB		 2138 dwc3_trace(trace_dwc3_gadget,
2139 "Stream %d found and started",
879631aa
FB		 2140 event->parameters);
2141
2142 break;
2143 case DEPEVT_STREAMEVT_NOTFOUND:
2144 /* FALLTHROUGH */
2145 default:
ec5e795c
FB		 2146 dwc3_trace(trace_dwc3_gadget,
2147 "unable to find suitable stream\n");
879631aa 2148 }
72246da4
FB		 2149 break;
2150 case DWC3_DEPEVT_RXTXFIFOEVT:
ec5e795c 2151 dwc3_trace(trace_dwc3_gadget, "%s FIFO Overrun\n", dep->name);
72246da4 2152 break;
72246da4 2153 case DWC3_DEPEVT_EPCMDCMPLT:
73815280 2154 dwc3_trace(trace_dwc3_gadget, "Endpoint Command Complete");
72246da4
FB		 2155 break;
2156 }
2157}
2158
2159static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2160{
2161 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2162 spin_unlock(&dwc->lock);
2163 dwc->gadget_driver->disconnect(&dwc->gadget);
2164 spin_lock(&dwc->lock);
2165 }
2166}
2167
bc5ba2e0
FB		 2168static void dwc3_suspend_gadget(struct dwc3 *dwc)
2169{
73a30bfc 2170 if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
bc5ba2e0
FB		 2171 spin_unlock(&dwc->lock);
2172 dwc->gadget_driver->suspend(&dwc->gadget);
2173 spin_lock(&dwc->lock);
2174 }
2175}
2176
2177static void dwc3_resume_gadget(struct dwc3 *dwc)
2178{
73a30bfc 2179 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
bc5ba2e0
FB		 2180 spin_unlock(&dwc->lock);
2181 dwc->gadget_driver->resume(&dwc->gadget);
5c7b3b02 2182 spin_lock(&dwc->lock);
8e74475b
FB		 2183 }
2184}
2185
2186static void dwc3_reset_gadget(struct dwc3 *dwc)
2187{
2188 if (!dwc->gadget_driver)
2189 return;
2190
2191 if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2192 spin_unlock(&dwc->lock);
2193 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
bc5ba2e0
FB		 2194 spin_lock(&dwc->lock);
2195 }
2196}
2197
b992e681 2198static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
72246da4
FB		 2199{
2200 struct dwc3_ep *dep;
2201 struct dwc3_gadget_ep_cmd_params params;
2202 u32 cmd;
2203 int ret;
2204
2205 dep = dwc->eps[epnum];
2206
b4996a86 2207 if (!dep->resource_index)
3daf74d7
PA		 2208 return;
2209
57911504
PA		 2210 /*
2211 * NOTICE: We are violating what the Databook says about the
2212 * EndTransfer command. Ideally we would _always_ wait for the
2213 * EndTransfer Command Completion IRQ, but that's causing too
2214 * much trouble synchronizing between us and gadget driver.
2215 *
2216 * We have discussed this with the IP Provider and it was
2217 * suggested to giveback all requests here, but give HW some
2218 * extra time to synchronize with the interconnect. We're using
dc93b41a 2219 * an arbitrary 100us delay for that.
57911504
PA		 2220 *
2221 * Note also that a similar handling was tested by Synopsys
2222 * (thanks a lot Paul) and nothing bad has come out of it.
2223 * In short, what we're doing is:
2224 *
2225 * - Issue EndTransfer WITH CMDIOC bit set
2226 * - Wait 100us
2227 */
2228
3daf74d7 2229 cmd = DWC3_DEPCMD_ENDTRANSFER;
b992e681
PZ		 2230 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2231 cmd |= DWC3_DEPCMD_CMDIOC;
b4996a86 2232 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
3daf74d7
PA		 2233 memset(&params, 0, sizeof(params));
2234 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
2235 WARN_ON_ONCE(ret);
b4996a86 2236 dep->resource_index = 0;
041d81f4 2237 dep->flags &= ~DWC3_EP_BUSY;
57911504 2238 udelay(100);
72246da4
FB		 2239}
2240
2241static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2242{
2243 u32 epnum;
2244
2245 for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2246 struct dwc3_ep *dep;
2247
2248 dep = dwc->eps[epnum];
6a1e3ef4
FB		 2249 if (!dep)
2250 continue;
2251
72246da4
FB		 2252 if (!(dep->flags & DWC3_EP_ENABLED))
2253 continue;
2254
624407f9 2255 dwc3_remove_requests(dwc, dep);
72246da4
FB		 2256 }
2257}
2258
2259static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2260{
2261 u32 epnum;
2262
2263 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2264 struct dwc3_ep *dep;
72246da4
FB		 2265 int ret;
2266
2267 dep = dwc->eps[epnum];
6a1e3ef4
FB		 2268 if (!dep)
2269 continue;
72246da4
FB		 2270
2271 if (!(dep->flags & DWC3_EP_STALL))
2272 continue;
2273
2274 dep->flags &= ~DWC3_EP_STALL;
2275
50c763f8 2276 ret = dwc3_send_clear_stall_ep_cmd(dep);
72246da4
FB		 2277 WARN_ON_ONCE(ret);
2278 }
2279}
2280
2281static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2282{
c4430a26
FB		 2283 int reg;
2284
72246da4
FB		 2285 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2286 reg &= ~DWC3_DCTL_INITU1ENA;
2287 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2288
2289 reg &= ~DWC3_DCTL_INITU2ENA;
2290 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
72246da4 2291
72246da4
FB		 2292 dwc3_disconnect_gadget(dwc);
2293
2294 dwc->gadget.speed = USB_SPEED_UNKNOWN;
df62df56 2295 dwc->setup_packet_pending = false;
06a374ed 2296 usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
72246da4
FB		 2297}
2298
72246da4
FB		 2299static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2300{
2301 u32 reg;
2302
df62df56
FB		 2303 /*
2304 * WORKAROUND: DWC3 revisions <1.88a have an issue which
2305 * would cause a missing Disconnect Event if there's a
2306 * pending Setup Packet in the FIFO.
2307 *
2308 * There's no suggested workaround on the official Bug
2309 * report, which states that "unless the driver/application
2310 * is doing any special handling of a disconnect event,
2311 * there is no functional issue".
2312 *
2313 * Unfortunately, it turns out that we _do_ some special
2314 * handling of a disconnect event, namely complete all
2315 * pending transfers, notify gadget driver of the
2316 * disconnection, and so on.
2317 *
2318 * Our suggested workaround is to follow the Disconnect
2319 * Event steps here, instead, based on a setup_packet_pending
b5d335e5
FB		 2320 * flag. Such flag gets set whenever we have a SETUP_PENDING
2321 * status for EP0 TRBs and gets cleared on XferComplete for the
df62df56
FB		 2322 * same endpoint.
2323 *
2324 * Refers to:
2325 *
2326 * STAR#9000466709: RTL: Device : Disconnect event not
2327 * generated if setup packet pending in FIFO
2328 */
2329 if (dwc->revision < DWC3_REVISION_188A) {
2330 if (dwc->setup_packet_pending)
2331 dwc3_gadget_disconnect_interrupt(dwc);
2332 }
2333
8e74475b 2334 dwc3_reset_gadget(dwc);
72246da4
FB		 2335
2336 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2337 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2338 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3b637367 2339 dwc->test_mode = false;
72246da4
FB		 2340
2341 dwc3_stop_active_transfers(dwc);
2342 dwc3_clear_stall_all_ep(dwc);
2343
2344 /* Reset device address to zero */
2345 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2346 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2347 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
72246da4
FB		 2348}
2349
2350static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
2351{
2352 u32 reg;
2353 u32 usb30_clock = DWC3_GCTL_CLK_BUS;
2354
2355 /*
2356 * We change the clock only at SS but I dunno why I would want to do
2357 * this. Maybe it becomes part of the power saving plan.
2358 */
2359
ee5cd41c
JY		 2360 if ((speed != DWC3_DSTS_SUPERSPEED) &&
2361 (speed != DWC3_DSTS_SUPERSPEED_PLUS))
72246da4
FB		 2362 return;
2363
2364 /*
2365 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2366 * each time on Connect Done.
2367 */
2368 if (!usb30_clock)
2369 return;
2370
2371 reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2372 reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
2373 dwc3_writel(dwc->regs, DWC3_GCTL, reg);
2374}
2375
72246da4
FB		 2376static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2377{
72246da4
FB		 2378 struct dwc3_ep *dep;
2379 int ret;
2380 u32 reg;
2381 u8 speed;
2382
72246da4
FB		 2383 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2384 speed = reg & DWC3_DSTS_CONNECTSPD;
2385 dwc->speed = speed;
2386
2387 dwc3_update_ram_clk_sel(dwc, speed);
2388
2389 switch (speed) {
7580862b
JY		 2390 case DWC3_DCFG_SUPERSPEED_PLUS:
2391 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2392 dwc->gadget.ep0->maxpacket = 512;
2393 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2394 break;
72246da4 2395 case DWC3_DCFG_SUPERSPEED:
05870c5b
FB		 2396 /*
2397 * WORKAROUND: DWC3 revisions <1.90a have an issue which
2398 * would cause a missing USB3 Reset event.
2399 *
2400 * In such situations, we should force a USB3 Reset
2401 * event by calling our dwc3_gadget_reset_interrupt()
2402 * routine.
2403 *
2404 * Refers to:
2405 *
2406 * STAR#9000483510: RTL: SS : USB3 reset event may
2407 * not be generated always when the link enters poll
2408 */
2409 if (dwc->revision < DWC3_REVISION_190A)
2410 dwc3_gadget_reset_interrupt(dwc);
2411
72246da4
FB		 2412 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2413 dwc->gadget.ep0->maxpacket = 512;
2414 dwc->gadget.speed = USB_SPEED_SUPER;
2415 break;
2416 case DWC3_DCFG_HIGHSPEED:
2417 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2418 dwc->gadget.ep0->maxpacket = 64;
2419 dwc->gadget.speed = USB_SPEED_HIGH;
2420 break;
2421 case DWC3_DCFG_FULLSPEED2:
2422 case DWC3_DCFG_FULLSPEED1:
2423 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2424 dwc->gadget.ep0->maxpacket = 64;
2425 dwc->gadget.speed = USB_SPEED_FULL;
2426 break;
2427 case DWC3_DCFG_LOWSPEED:
2428 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2429 dwc->gadget.ep0->maxpacket = 8;
2430 dwc->gadget.speed = USB_SPEED_LOW;
2431 break;
2432 }
2433
2b758350
PA		 2434 /* Enable USB2 LPM Capability */
2435
ee5cd41c
JY		 2436 if ((dwc->revision > DWC3_REVISION_194A) &&
2437 (speed != DWC3_DCFG_SUPERSPEED) &&
2438 (speed != DWC3_DCFG_SUPERSPEED_PLUS)) {
2b758350
PA		 2439 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2440 reg |= DWC3_DCFG_LPM_CAP;
2441 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2442
2443 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2444 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2445
460d098c 2446 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2b758350 2447
80caf7d2
HR		 2448 /*
2449 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2450 * DCFG.LPMCap is set, core responses with an ACK and the
2451 * BESL value in the LPM token is less than or equal to LPM
2452 * NYET threshold.
2453 */
2454 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2455 && dwc->has_lpm_erratum,
2456 "LPM Erratum not available on dwc3 revisisions < 2.40a\n");
2457
2458 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2459 reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2460
356363bf
FB		 2461 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2462 } else {
2463 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2464 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2b758350
PA		 2465 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2466 }
2467
72246da4 2468 dep = dwc->eps[0];
265b70a7
PZ		 2469 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2470 false);
72246da4
FB		 2471 if (ret) {
2472 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2473 return;
2474 }
2475
2476 dep = dwc->eps[1];
265b70a7
PZ		 2477 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2478 false);
72246da4
FB		 2479 if (ret) {
2480 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2481 return;
2482 }
2483
2484 /*
2485 * Configure PHY via GUSB3PIPECTLn if required.
2486 *
2487 * Update GTXFIFOSIZn
2488 *
2489 * In both cases reset values should be sufficient.
2490 */
2491}
2492
2493static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2494{
72246da4
FB		 2495 /*
2496 * TODO take core out of low power mode when that's
2497 * implemented.
2498 */
2499
ad14d4e0
JL		 2500 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2501 spin_unlock(&dwc->lock);
2502 dwc->gadget_driver->resume(&dwc->gadget);
2503 spin_lock(&dwc->lock);
2504 }
72246da4
FB		 2505}
2506
2507static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2508 unsigned int evtinfo)
2509{
fae2b904 2510 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
0b0cc1cd
FB		 2511 unsigned int pwropt;
2512
2513 /*
2514 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2515 * Hibernation mode enabled which would show up when device detects
2516 * host-initiated U3 exit.
2517 *
2518 * In that case, device will generate a Link State Change Interrupt
2519 * from U3 to RESUME which is only necessary if Hibernation is
2520 * configured in.
2521 *
2522 * There are no functional changes due to such spurious event and we
2523 * just need to ignore it.
2524 *
2525 * Refers to:
2526 *
2527 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2528 * operational mode
2529 */
2530 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2531 if ((dwc->revision < DWC3_REVISION_250A) &&
2532 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2533 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2534 (next == DWC3_LINK_STATE_RESUME)) {
73815280
FB		 2535 dwc3_trace(trace_dwc3_gadget,
2536 "ignoring transition U3 -> Resume");
0b0cc1cd
FB		 2537 return;
2538 }
2539 }
fae2b904
FB		 2540
2541 /*
2542 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2543 * on the link partner, the USB session might do multiple entry/exit
2544 * of low power states before a transfer takes place.
2545 *
2546 * Due to this problem, we might experience lower throughput. The
2547 * suggested workaround is to disable DCTL[12:9] bits if we're
2548 * transitioning from U1/U2 to U0 and enable those bits again
2549 * after a transfer completes and there are no pending transfers
2550 * on any of the enabled endpoints.
2551 *
2552 * This is the first half of that workaround.
2553 *
2554 * Refers to:
2555 *
2556 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2557 * core send LGO_Ux entering U0
2558 */
2559 if (dwc->revision < DWC3_REVISION_183A) {
2560 if (next == DWC3_LINK_STATE_U0) {
2561 u32 u1u2;
2562 u32 reg;
2563
2564 switch (dwc->link_state) {
2565 case DWC3_LINK_STATE_U1:
2566 case DWC3_LINK_STATE_U2:
2567 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2568 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2569 | DWC3_DCTL_ACCEPTU2ENA
2570 | DWC3_DCTL_INITU1ENA
2571 | DWC3_DCTL_ACCEPTU1ENA);
2572
2573 if (!dwc->u1u2)
2574 dwc->u1u2 = reg & u1u2;
2575
2576 reg &= ~u1u2;
2577
2578 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2579 break;
2580 default:
2581 /* do nothing */
2582 break;
2583 }
2584 }
2585 }
2586
bc5ba2e0
FB		 2587 switch (next) {
2588 case DWC3_LINK_STATE_U1:
2589 if (dwc->speed == USB_SPEED_SUPER)
2590 dwc3_suspend_gadget(dwc);
2591 break;
2592 case DWC3_LINK_STATE_U2:
2593 case DWC3_LINK_STATE_U3:
2594 dwc3_suspend_gadget(dwc);
2595 break;
2596 case DWC3_LINK_STATE_RESUME:
2597 dwc3_resume_gadget(dwc);
2598 break;
2599 default:
2600 /* do nothing */
2601 break;
2602 }
2603
e57ebc1d 2604 dwc->link_state = next;
72246da4
FB		 2605}
2606
e1dadd3b
FB		 2607static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2608 unsigned int evtinfo)
2609{
2610 unsigned int is_ss = evtinfo & BIT(4);
2611
2612 /**
2613 * WORKAROUND: DWC3 revison 2.20a with hibernation support
2614 * have a known issue which can cause USB CV TD.9.23 to fail
2615 * randomly.
2616 *
2617 * Because of this issue, core could generate bogus hibernation
2618 * events which SW needs to ignore.
2619 *
2620 * Refers to:
2621 *
2622 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2623 * Device Fallback from SuperSpeed
2624 */
2625 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2626 return;
2627
2628 /* enter hibernation here */
2629}
2630
72246da4
FB		 2631static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2632 const struct dwc3_event_devt *event)
2633{
2634 switch (event->type) {
2635 case DWC3_DEVICE_EVENT_DISCONNECT:
2636 dwc3_gadget_disconnect_interrupt(dwc);
2637 break;
2638 case DWC3_DEVICE_EVENT_RESET:
2639 dwc3_gadget_reset_interrupt(dwc);
2640 break;
2641 case DWC3_DEVICE_EVENT_CONNECT_DONE:
2642 dwc3_gadget_conndone_interrupt(dwc);
2643 break;
2644 case DWC3_DEVICE_EVENT_WAKEUP:
2645 dwc3_gadget_wakeup_interrupt(dwc);
2646 break;
e1dadd3b
FB		 2647 case DWC3_DEVICE_EVENT_HIBER_REQ:
2648 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
2649 "unexpected hibernation event\n"))
2650 break;
2651
2652 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2653 break;
72246da4
FB		 2654 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2655 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2656 break;
2657 case DWC3_DEVICE_EVENT_EOPF:
73815280 2658 dwc3_trace(trace_dwc3_gadget, "End of Periodic Frame");
72246da4
FB		 2659 break;
2660 case DWC3_DEVICE_EVENT_SOF:
73815280 2661 dwc3_trace(trace_dwc3_gadget, "Start of Periodic Frame");
72246da4
FB		 2662 break;
2663 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
73815280 2664 dwc3_trace(trace_dwc3_gadget, "Erratic Error");
72246da4
FB		 2665 break;
2666 case DWC3_DEVICE_EVENT_CMD_CMPL:
73815280 2667 dwc3_trace(trace_dwc3_gadget, "Command Complete");
72246da4
FB		 2668 break;
2669 case DWC3_DEVICE_EVENT_OVERFLOW:
73815280 2670 dwc3_trace(trace_dwc3_gadget, "Overflow");
72246da4
FB		 2671 break;
2672 default:
e9f2aa87 2673 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
72246da4
FB		 2674 }
2675}
2676
2677static void dwc3_process_event_entry(struct dwc3 *dwc,
2678 const union dwc3_event *event)
2679{
2c4cbe6e
FB		 2680 trace_dwc3_event(event->raw);
2681
72246da4
FB		 2682 /* Endpoint IRQ, handle it and return early */
2683 if (event->type.is_devspec == 0) {
2684 /* depevt */
2685 return dwc3_endpoint_interrupt(dwc, &event->depevt);
2686 }
2687
2688 switch (event->type.type) {
2689 case DWC3_EVENT_TYPE_DEV:
2690 dwc3_gadget_interrupt(dwc, &event->devt);
2691 break;
2692 /* REVISIT what to do with Carkit and I2C events ? */
2693 default:
2694 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2695 }
2696}
2697
dea520a4 2698static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
b15a762f 2699{
dea520a4 2700 struct dwc3 *dwc = evt->dwc;
b15a762f 2701 irqreturn_t ret = IRQ_NONE;
f42f2447 2702 int left;
e8adfc30 2703 u32 reg;
b15a762f 2704
f42f2447 2705 left = evt->count;
b15a762f 2706
f42f2447
FB		 2707 if (!(evt->flags & DWC3_EVENT_PENDING))
2708 return IRQ_NONE;
b15a762f 2709
f42f2447
FB		 2710 while (left > 0) {
2711 union dwc3_event event;
b15a762f 2712
f42f2447 2713 event.raw = *(u32 *) (evt->buf + evt->lpos);
b15a762f 2714
f42f2447 2715 dwc3_process_event_entry(dwc, &event);
b15a762f 2716
f42f2447
FB		 2717 /*
2718 * FIXME we wrap around correctly to the next entry as
2719 * almost all entries are 4 bytes in size. There is one
2720 * entry which has 12 bytes which is a regular entry
2721 * followed by 8 bytes data. ATM I don't know how
2722 * things are organized if we get next to the a
2723 * boundary so I worry about that once we try to handle
2724 * that.
2725 */
2726 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2727 left -= 4;
b15a762f 2728
660e9bde 2729 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), 4);
f42f2447 2730 }
b15a762f 2731
f42f2447
FB		 2732 evt->count = 0;
2733 evt->flags &= ~DWC3_EVENT_PENDING;
2734 ret = IRQ_HANDLED;
b15a762f 2735
f42f2447 2736 /* Unmask interrupt */
660e9bde 2737 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
f42f2447 2738 reg &= ~DWC3_GEVNTSIZ_INTMASK;
660e9bde 2739 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
b15a762f 2740
f42f2447
FB		 2741 return ret;
2742}
e8adfc30 2743
dea520a4 2744static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
f42f2447 2745{
dea520a4
FB		 2746 struct dwc3_event_buffer *evt = _evt;
2747 struct dwc3 *dwc = evt->dwc;
e5f68b4a 2748 unsigned long flags;
f42f2447 2749 irqreturn_t ret = IRQ_NONE;
f42f2447 2750
e5f68b4a 2751 spin_lock_irqsave(&dwc->lock, flags);
dea520a4 2752 ret = dwc3_process_event_buf(evt);
e5f68b4a 2753 spin_unlock_irqrestore(&dwc->lock, flags);
b15a762f
FB		 2754
2755 return ret;
2756}
2757
dea520a4 2758static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
72246da4 2759{
dea520a4 2760 struct dwc3 *dwc = evt->dwc;
72246da4 2761 u32 count;
e8adfc30 2762 u32 reg;
72246da4 2763
660e9bde 2764 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
72246da4
FB		 2765 count &= DWC3_GEVNTCOUNT_MASK;
2766 if (!count)
2767 return IRQ_NONE;
2768
b15a762f
FB		 2769 evt->count = count;
2770 evt->flags |= DWC3_EVENT_PENDING;
72246da4 2771
e8adfc30 2772 /* Mask interrupt */
660e9bde 2773 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
e8adfc30 2774 reg |= DWC3_GEVNTSIZ_INTMASK;
660e9bde 2775 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
e8adfc30 2776
b15a762f 2777 return IRQ_WAKE_THREAD;
72246da4
FB		 2778}
2779
dea520a4 2780static irqreturn_t dwc3_interrupt(int irq, void *_evt)
72246da4 2781{
dea520a4 2782 struct dwc3_event_buffer *evt = _evt;
72246da4 2783
dea520a4 2784 return dwc3_check_event_buf(evt);
72246da4
FB		 2785}
2786
2787/**
2788 * dwc3_gadget_init - Initializes gadget related registers
1d046793 2789 * @dwc: pointer to our controller context structure
72246da4
FB		 2790 *
2791 * Returns 0 on success otherwise negative errno.
2792 */
41ac7b3a 2793int dwc3_gadget_init(struct dwc3 *dwc)
72246da4 2794{
72246da4 2795 int ret;
72246da4
FB		 2796
2797 dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2798 &dwc->ctrl_req_addr, GFP_KERNEL);
2799 if (!dwc->ctrl_req) {
2800 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2801 ret = -ENOMEM;
2802 goto err0;
2803 }
2804
2abd9d5f 2805 dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
72246da4
FB		 2806 &dwc->ep0_trb_addr, GFP_KERNEL);
2807 if (!dwc->ep0_trb) {
2808 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2809 ret = -ENOMEM;
2810 goto err1;
2811 }
2812
3ef35faf 2813 dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL);
72246da4 2814 if (!dwc->setup_buf) {
72246da4
FB		 2815 ret = -ENOMEM;
2816 goto err2;
2817 }
2818
5812b1c2 2819 dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
3ef35faf
FB		 2820 DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
2821 GFP_KERNEL);
5812b1c2
FB		 2822 if (!dwc->ep0_bounce) {
2823 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2824 ret = -ENOMEM;
2825 goto err3;
2826 }
2827
04c03d10
FB		 2828 dwc->zlp_buf = kzalloc(DWC3_ZLP_BUF_SIZE, GFP_KERNEL);
2829 if (!dwc->zlp_buf) {
2830 ret = -ENOMEM;
2831 goto err4;
2832 }
2833
72246da4 2834 dwc->gadget.ops = &dwc3_gadget_ops;
72246da4 2835 dwc->gadget.speed = USB_SPEED_UNKNOWN;
eeb720fb 2836 dwc->gadget.sg_supported = true;
72246da4 2837 dwc->gadget.name = "dwc3-gadget";
6a4290cc 2838 dwc->gadget.is_otg = dwc->dr_mode == USB_DR_MODE_OTG;
72246da4 2839
b9e51b2b
BM		 2840 /*
2841 * FIXME We might be setting max_speed to <SUPER, however versions
2842 * <2.20a of dwc3 have an issue with metastability (documented
2843 * elsewhere in this driver) which tells us we can't set max speed to
2844 * anything lower than SUPER.
2845 *
2846 * Because gadget.max_speed is only used by composite.c and function
2847 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
2848 * to happen so we avoid sending SuperSpeed Capability descriptor
2849 * together with our BOS descriptor as that could confuse host into
2850 * thinking we can handle super speed.
2851 *
2852 * Note that, in fact, we won't even support GetBOS requests when speed
2853 * is less than super speed because we don't have means, yet, to tell
2854 * composite.c that we are USB 2.0 + LPM ECN.
2855 */
2856 if (dwc->revision < DWC3_REVISION_220A)
2857 dwc3_trace(trace_dwc3_gadget,
2858 "Changing max_speed on rev %08x\n",
2859 dwc->revision);
2860
2861 dwc->gadget.max_speed = dwc->maximum_speed;
2862
a4b9d94b
DC		 2863 /*
2864 * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize
2865 * on ep out.
2866 */
2867 dwc->gadget.quirk_ep_out_aligned_size = true;
2868
72246da4
FB		 2869 /*
2870 * REVISIT: Here we should clear all pending IRQs to be
2871 * sure we're starting from a well known location.
2872 */
2873
2874 ret = dwc3_gadget_init_endpoints(dwc);
2875 if (ret)
04c03d10 2876 goto err5;
72246da4 2877
72246da4
FB		 2878 ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2879 if (ret) {
2880 dev_err(dwc->dev, "failed to register udc\n");
04c03d10 2881 goto err5;
72246da4
FB		 2882 }
2883
2884 return 0;
2885
04c03d10
FB		 2886err5:
2887 kfree(dwc->zlp_buf);
2888
5812b1c2 2889err4:
e1f80467 2890 dwc3_gadget_free_endpoints(dwc);
3ef35faf
FB		 2891 dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2892 dwc->ep0_bounce, dwc->ep0_bounce_addr);
5812b1c2 2893
72246da4 2894err3:
0fc9a1be 2895 kfree(dwc->setup_buf);
72246da4
FB		 2896
2897err2:
2898 dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2899 dwc->ep0_trb, dwc->ep0_trb_addr);
2900
2901err1:
2902 dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2903 dwc->ctrl_req, dwc->ctrl_req_addr);
2904
2905err0:
2906 return ret;
2907}
2908
7415f17c
FB		 2909/* -------------------------------------------------------------------------- */
2910
72246da4
FB		 2911void dwc3_gadget_exit(struct dwc3 *dwc)
2912{
72246da4 2913 usb_del_gadget_udc(&dwc->gadget);
72246da4 2914
72246da4
FB		 2915 dwc3_gadget_free_endpoints(dwc);
2916
3ef35faf
FB		 2917 dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2918 dwc->ep0_bounce, dwc->ep0_bounce_addr);
5812b1c2 2919
0fc9a1be 2920 kfree(dwc->setup_buf);
04c03d10 2921 kfree(dwc->zlp_buf);
72246da4
FB		 2922
2923 dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2924 dwc->ep0_trb, dwc->ep0_trb_addr);
2925
2926 dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2927 dwc->ctrl_req, dwc->ctrl_req_addr);
72246da4 2928}
7415f17c 2929
0b0231aa 2930int dwc3_gadget_suspend(struct dwc3 *dwc)
7415f17c 2931{
9f8a67b6
FB		 2932 int ret;
2933
9772b47a
RQ		 2934 if (!dwc->gadget_driver)
2935 return 0;
2936
9f8a67b6
FB		 2937 ret = dwc3_gadget_run_stop(dwc, false, false);
2938 if (ret < 0)
2939 return ret;
7415f17c 2940
9f8a67b6
FB		 2941 dwc3_disconnect_gadget(dwc);
2942 __dwc3_gadget_stop(dwc);
7415f17c
FB		 2943
2944 return 0;
2945}
2946
2947int dwc3_gadget_resume(struct dwc3 *dwc)
2948{
7415f17c
FB		 2949 int ret;
2950
9772b47a
RQ		 2951 if (!dwc->gadget_driver)
2952 return 0;
2953
9f8a67b6
FB		 2954 ret = __dwc3_gadget_start(dwc);
2955 if (ret < 0)
7415f17c
FB		 2956 goto err0;
2957
9f8a67b6
FB		 2958 ret = dwc3_gadget_run_stop(dwc, true, false);
2959 if (ret < 0)
7415f17c
FB		 2960 goto err1;
2961
7415f17c
FB		 2962 return 0;
2963
2964err1:
9f8a67b6 2965 __dwc3_gadget_stop(dwc);
7415f17c
FB		 2966
2967err0:
2968 return ret;
2969}
Ubuntu Artful kernel mirror