]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/usb/dwc3/gadget.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
usb: dwc3: gadget: simplify unaligned and zlp handling
[mirror_ubuntu-hirsute-kernel.git] / drivers / usb / dwc3 / gadget.c
CommitLineData
5fd54ace 1// SPDX-License-Identifier: GPL-2.0
bfad65ee 2/*
72246da4
FB		 3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4 *
5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
72246da4
FB		 6 *
7 * Authors: Felipe Balbi <balbi@ti.com>,
8 * Sebastian Andrzej Siewior <bigeasy@linutronix.de>
72246da4
FB		 9 */
10
11#include <linux/kernel.h>
12#include <linux/delay.h>
13#include <linux/slab.h>
14#include <linux/spinlock.h>
15#include <linux/platform_device.h>
16#include <linux/pm_runtime.h>
17#include <linux/interrupt.h>
18#include <linux/io.h>
19#include <linux/list.h>
20#include <linux/dma-mapping.h>
21
22#include <linux/usb/ch9.h>
23#include <linux/usb/gadget.h>
24
80977dc9 25#include "debug.h"
72246da4
FB		 26#include "core.h"
27#include "gadget.h"
28#include "io.h"
29
04a9bfcd 30/**
bfad65ee 31 * dwc3_gadget_set_test_mode - enables usb2 test modes
04a9bfcd
FB		 32 * @dwc: pointer to our context structure
33 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
34 *
bfad65ee
FB		 35 * Caller should take care of locking. This function will return 0 on
36 * success or -EINVAL if wrong Test Selector is passed.
04a9bfcd
FB		 37 */
38int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
39{
40 u32 reg;
41
42 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
43 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
44
45 switch (mode) {
46 case TEST_J:
47 case TEST_K:
48 case TEST_SE0_NAK:
49 case TEST_PACKET:
50 case TEST_FORCE_EN:
51 reg |= mode << 1;
52 break;
53 default:
54 return -EINVAL;
55 }
56
57 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
58
59 return 0;
60}
61
911f1f88 62/**
bfad65ee 63 * dwc3_gadget_get_link_state - gets current state of usb link
911f1f88
PZ		 64 * @dwc: pointer to our context structure
65 *
66 * Caller should take care of locking. This function will
67 * return the link state on success (>= 0) or -ETIMEDOUT.
68 */
69int dwc3_gadget_get_link_state(struct dwc3 *dwc)
70{
71 u32 reg;
72
73 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
74
75 return DWC3_DSTS_USBLNKST(reg);
76}
77
8598bde7 78/**
bfad65ee 79 * dwc3_gadget_set_link_state - sets usb link to a particular state
8598bde7
FB		 80 * @dwc: pointer to our context structure
81 * @state: the state to put link into
82 *
83 * Caller should take care of locking. This function will
aee63e3c 84 * return 0 on success or -ETIMEDOUT.
8598bde7
FB		 85 */
86int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
87{
aee63e3c 88 int retries = 10000;
8598bde7
FB		 89 u32 reg;
90
802fde98
PZ		 91 /*
92 * Wait until device controller is ready. Only applies to 1.94a and
93 * later RTL.
94 */
95 if (dwc->revision >= DWC3_REVISION_194A) {
96 while (--retries) {
97 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
98 if (reg & DWC3_DSTS_DCNRD)
99 udelay(5);
100 else
101 break;
102 }
103
104 if (retries <= 0)
105 return -ETIMEDOUT;
106 }
107
8598bde7
FB		 108 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
109 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
110
111 /* set requested state */
112 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
113 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
114
802fde98
PZ		 115 /*
116 * The following code is racy when called from dwc3_gadget_wakeup,
117 * and is not needed, at least on newer versions
118 */
119 if (dwc->revision >= DWC3_REVISION_194A)
120 return 0;
121
8598bde7 122 /* wait for a change in DSTS */
aed430e5 123 retries = 10000;
8598bde7
FB		 124 while (--retries) {
125 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
126
8598bde7
FB		 127 if (DWC3_DSTS_USBLNKST(reg) == state)
128 return 0;
129
aee63e3c 130 udelay(5);
8598bde7
FB		 131 }
132
8598bde7
FB		 133 return -ETIMEDOUT;
134}
135
dca0119c 136/**
bfad65ee
FB		 137 * dwc3_ep_inc_trb - increment a trb index.
138 * @index: Pointer to the TRB index to increment.
dca0119c
JY		 139 *
140 * The index should never point to the link TRB. After incrementing,
141 * if it is point to the link TRB, wrap around to the beginning. The
142 * link TRB is always at the last TRB entry.
143 */
144static void dwc3_ep_inc_trb(u8 *index)
457e84b6 145{
dca0119c
JY		 146 (*index)++;
147 if (*index == (DWC3_TRB_NUM - 1))
148 *index = 0;
ef966b9d 149}
457e84b6 150
bfad65ee
FB		 151/**
152 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
153 * @dep: The endpoint whose enqueue pointer we're incrementing
154 */
dca0119c 155static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
ef966b9d 156{
dca0119c 157 dwc3_ep_inc_trb(&dep->trb_enqueue);
ef966b9d 158}
457e84b6 159
bfad65ee
FB		 160/**
161 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
162 * @dep: The endpoint whose enqueue pointer we're incrementing
163 */
dca0119c 164static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
ef966b9d 165{
dca0119c 166 dwc3_ep_inc_trb(&dep->trb_dequeue);
457e84b6
FB		 167}
168
c91815b5
FB		 169void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
170 struct dwc3_request *req, int status)
72246da4
FB		 171{
172 struct dwc3 *dwc = dep->dwc;
173
737f1ae2 174 req->started = false;
72246da4 175 list_del(&req->list);
e62c5bc5 176 req->remaining = 0;
72246da4
FB		 177
178 if (req->request.status == -EINPROGRESS)
179 req->request.status = status;
180
4a71fcb8
JP		 181 if (req->trb)
182 usb_gadget_unmap_request_by_dev(dwc->sysdev,
c91815b5 183 &req->request, req->direction);
4a71fcb8
JP		 184
185 req->trb = NULL;
2c4cbe6e 186 trace_dwc3_gadget_giveback(req);
72246da4 187
c91815b5
FB		 188 if (dep->number > 1)
189 pm_runtime_put(dwc->dev);
190}
191
192/**
193 * dwc3_gadget_giveback - call struct usb_request's ->complete callback
194 * @dep: The endpoint to whom the request belongs to
195 * @req: The request we're giving back
196 * @status: completion code for the request
197 *
198 * Must be called with controller's lock held and interrupts disabled. This
199 * function will unmap @req and call its ->complete() callback to notify upper
200 * layers that it has completed.
201 */
202void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
203 int status)
204{
205 struct dwc3 *dwc = dep->dwc;
206
207 dwc3_gadget_del_and_unmap_request(dep, req, status);
208
72246da4 209 spin_unlock(&dwc->lock);
304f7e5e 210 usb_gadget_giveback_request(&dep->endpoint, &req->request);
72246da4
FB		 211 spin_lock(&dwc->lock);
212}
213
bfad65ee
FB		 214/**
215 * dwc3_send_gadget_generic_command - issue a generic command for the controller
216 * @dwc: pointer to the controller context
217 * @cmd: the command to be issued
218 * @param: command parameter
219 *
220 * Caller should take care of locking. Issue @cmd with a given @param to @dwc
221 * and wait for its completion.
222 */
3ece0ec4 223int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
b09bb642
FB		 224{
225 u32 timeout = 500;
71f7e702 226 int status = 0;
0fe886cd 227 int ret = 0;
b09bb642
FB		 228 u32 reg;
229
230 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
231 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
232
233 do {
234 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
235 if (!(reg & DWC3_DGCMD_CMDACT)) {
71f7e702
FB		 236 status = DWC3_DGCMD_STATUS(reg);
237 if (status)
0fe886cd
FB		 238 ret = -EINVAL;
239 break;
b09bb642 240 }
e3aee486 241 } while (--timeout);
0fe886cd
FB		 242
243 if (!timeout) {
0fe886cd 244 ret = -ETIMEDOUT;
71f7e702 245 status = -ETIMEDOUT;
0fe886cd
FB		 246 }
247
71f7e702
FB		 248 trace_dwc3_gadget_generic_cmd(cmd, param, status);
249
0fe886cd 250 return ret;
b09bb642
FB		 251}
252
c36d8e94
FB		 253static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
254
bfad65ee
FB		 255/**
256 * dwc3_send_gadget_ep_cmd - issue an endpoint command
257 * @dep: the endpoint to which the command is going to be issued
258 * @cmd: the command to be issued
259 * @params: parameters to the command
260 *
261 * Caller should handle locking. This function will issue @cmd with given
262 * @params to @dep and wait for its completion.
263 */
2cd4718d
FB		 264int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
265 struct dwc3_gadget_ep_cmd_params *params)
72246da4 266{
8897a761 267 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
2cd4718d 268 struct dwc3 *dwc = dep->dwc;
8722e095 269 u32 timeout = 1000;
72246da4
FB		 270 u32 reg;
271
0933df15 272 int cmd_status = 0;
2b0f11df 273 int susphy = false;
c0ca324d 274 int ret = -EINVAL;
72246da4 275
2b0f11df
FB		 276 /*
277 * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
278 * we're issuing an endpoint command, we must check if
279 * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
280 *
281 * We will also set SUSPHY bit to what it was before returning as stated
282 * by the same section on Synopsys databook.
283 */
ab2a92e7
FB		 284 if (dwc->gadget.speed <= USB_SPEED_HIGH) {
285 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
286 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
287 susphy = true;
288 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
289 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
290 }
2b0f11df
FB		 291 }
292
5999914f 293 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
c36d8e94
FB		 294 int needs_wakeup;
295
296 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
297 dwc->link_state == DWC3_LINK_STATE_U2 ||
298 dwc->link_state == DWC3_LINK_STATE_U3);
299
300 if (unlikely(needs_wakeup)) {
301 ret = __dwc3_gadget_wakeup(dwc);
302 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
303 ret);
304 }
305 }
306
2eb88016
FB		 307 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
308 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
309 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
72246da4 310
8897a761
FB		 311 /*
312 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
313 * not relying on XferNotReady, we can make use of a special "No
314 * Response Update Transfer" command where we should clear both CmdAct
315 * and CmdIOC bits.
316 *
317 * With this, we don't need to wait for command completion and can
318 * straight away issue further commands to the endpoint.
319 *
320 * NOTICE: We're making an assumption that control endpoints will never
321 * make use of Update Transfer command. This is a safe assumption
322 * because we can never have more than one request at a time with
323 * Control Endpoints. If anybody changes that assumption, this chunk
324 * needs to be updated accordingly.
325 */
326 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
327 !usb_endpoint_xfer_isoc(desc))
328 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
329 else
330 cmd |= DWC3_DEPCMD_CMDACT;
331
332 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
72246da4 333 do {
2eb88016 334 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
72246da4 335 if (!(reg & DWC3_DEPCMD_CMDACT)) {
0933df15 336 cmd_status = DWC3_DEPCMD_STATUS(reg);
7b9cc7a2 337
7b9cc7a2
KL		 338 switch (cmd_status) {
339 case 0:
340 ret = 0;
341 break;
342 case DEPEVT_TRANSFER_NO_RESOURCE:
7b9cc7a2 343 ret = -EINVAL;
c0ca324d 344 break;
7b9cc7a2
KL		 345 case DEPEVT_TRANSFER_BUS_EXPIRY:
346 /*
347 * SW issues START TRANSFER command to
348 * isochronous ep with future frame interval. If
349 * future interval time has already passed when
350 * core receives the command, it will respond
351 * with an error status of 'Bus Expiry'.
352 *
353 * Instead of always returning -EINVAL, let's
354 * give a hint to the gadget driver that this is
355 * the case by returning -EAGAIN.
356 */
7b9cc7a2
KL		 357 ret = -EAGAIN;
358 break;
359 default:
360 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
361 }
362
c0ca324d 363 break;
72246da4 364 }
f6bb225b 365 } while (--timeout);
72246da4 366
f6bb225b 367 if (timeout == 0) {
f6bb225b 368 ret = -ETIMEDOUT;
0933df15 369 cmd_status = -ETIMEDOUT;
f6bb225b 370 }
c0ca324d 371
0933df15
FB		 372 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
373
6cb2e4e3
FB		 374 if (ret == 0) {
375 switch (DWC3_DEPCMD_CMD(cmd)) {
376 case DWC3_DEPCMD_STARTTRANSFER:
377 dep->flags |= DWC3_EP_TRANSFER_STARTED;
378 break;
379 case DWC3_DEPCMD_ENDTRANSFER:
380 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
381 break;
382 default:
383 /* nothing */
384 break;
385 }
386 }
387
2b0f11df
FB		 388 if (unlikely(susphy)) {
389 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
390 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
391 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
392 }
393
c0ca324d 394 return ret;
72246da4
FB		 395}
396
50c763f8
JY		 397static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
398{
399 struct dwc3 *dwc = dep->dwc;
400 struct dwc3_gadget_ep_cmd_params params;
401 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
402
403 /*
404 * As of core revision 2.60a the recommended programming model
405 * is to set the ClearPendIN bit when issuing a Clear Stall EP
406 * command for IN endpoints. This is to prevent an issue where
407 * some (non-compliant) hosts may not send ACK TPs for pending
408 * IN transfers due to a mishandled error condition. Synopsys
409 * STAR 9000614252.
410 */
5e6c88d2
LB		 411 if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
412 (dwc->gadget.speed >= USB_SPEED_SUPER))
50c763f8
JY		 413 cmd |= DWC3_DEPCMD_CLEARPENDIN;
414
415 memset(&params, 0, sizeof(params));
416
2cd4718d 417 return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
50c763f8
JY		 418}
419
72246da4 420static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
f6bafc6a 421 struct dwc3_trb *trb)
72246da4 422{
c439ef87 423 u32 offset = (char *) trb - (char *) dep->trb_pool;
72246da4
FB		 424
425 return dep->trb_pool_dma + offset;
426}
427
428static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
429{
430 struct dwc3 *dwc = dep->dwc;
431
432 if (dep->trb_pool)
433 return 0;
434
d64ff406 435 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
72246da4
FB		 436 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
437 &dep->trb_pool_dma, GFP_KERNEL);
438 if (!dep->trb_pool) {
439 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
440 dep->name);
441 return -ENOMEM;
442 }
443
444 return 0;
445}
446
447static void dwc3_free_trb_pool(struct dwc3_ep *dep)
448{
449 struct dwc3 *dwc = dep->dwc;
450
d64ff406 451 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
72246da4
FB		 452 dep->trb_pool, dep->trb_pool_dma);
453
454 dep->trb_pool = NULL;
455 dep->trb_pool_dma = 0;
456}
457
c4509601
JY		 458static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
459
460/**
bfad65ee 461 * dwc3_gadget_start_config - configure ep resources
c4509601
JY		 462 * @dwc: pointer to our controller context structure
463 * @dep: endpoint that is being enabled
464 *
bfad65ee
FB		 465 * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
466 * completion, it will set Transfer Resource for all available endpoints.
c4509601 467 *
bfad65ee
FB		 468 * The assignment of transfer resources cannot perfectly follow the data book
469 * due to the fact that the controller driver does not have all knowledge of the
470 * configuration in advance. It is given this information piecemeal by the
471 * composite gadget framework after every SET_CONFIGURATION and
472 * SET_INTERFACE. Trying to follow the databook programming model in this
473 * scenario can cause errors. For two reasons:
c4509601 474 *
bfad65ee
FB		 475 * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
476 * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
477 * incorrect in the scenario of multiple interfaces.
478 *
479 * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
c4509601
JY		 480 * endpoint on alt setting (8.1.6).
481 *
482 * The following simplified method is used instead:
483 *
bfad65ee
FB		 484 * All hardware endpoints can be assigned a transfer resource and this setting
485 * will stay persistent until either a core reset or hibernation. So whenever we
486 * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
487 * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
c4509601
JY		 488 * guaranteed that there are as many transfer resources as endpoints.
489 *
bfad65ee
FB		 490 * This function is called for each endpoint when it is being enabled but is
491 * triggered only when called for EP0-out, which always happens first, and which
492 * should only happen in one of the above conditions.
c4509601 493 */
72246da4
FB		 494static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
495{
496 struct dwc3_gadget_ep_cmd_params params;
497 u32 cmd;
c4509601
JY		 498 int i;
499 int ret;
500
501 if (dep->number)
502 return 0;
72246da4
FB		 503
504 memset(&params, 0x00, sizeof(params));
c4509601 505 cmd = DWC3_DEPCMD_DEPSTARTCFG;
72246da4 506
2cd4718d 507 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
c4509601
JY		 508 if (ret)
509 return ret;
510
511 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
512 struct dwc3_ep *dep = dwc->eps[i];
72246da4 513
c4509601
JY		 514 if (!dep)
515 continue;
516
517 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
518 if (ret)
519 return ret;
72246da4
FB		 520 }
521
522 return 0;
523}
524
525static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
21e64bf2 526 bool modify, bool restore)
72246da4 527{
39ebb05c
JY		 528 const struct usb_ss_ep_comp_descriptor *comp_desc;
529 const struct usb_endpoint_descriptor *desc;
72246da4
FB		 530 struct dwc3_gadget_ep_cmd_params params;
531
21e64bf2
FB		 532 if (dev_WARN_ONCE(dwc->dev, modify && restore,
533 "Can't modify and restore\n"))
534 return -EINVAL;
535
39ebb05c
JY		 536 comp_desc = dep->endpoint.comp_desc;
537 desc = dep->endpoint.desc;
538
72246da4
FB		 539 memset(&params, 0x00, sizeof(params));
540
dc1c70a7 541 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
d2e9a13a
CP		 542 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
543
544 /* Burst size is only needed in SuperSpeed mode */
ee5cd41c 545 if (dwc->gadget.speed >= USB_SPEED_SUPER) {
676e3497 546 u32 burst = dep->endpoint.maxburst;
676e3497 547 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
d2e9a13a 548 }
72246da4 549
21e64bf2
FB		 550 if (modify) {
551 params.param0 |= DWC3_DEPCFG_ACTION_MODIFY;
552 } else if (restore) {
265b70a7
PZ		 553 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
554 params.param2 |= dep->saved_state;
21e64bf2
FB		 555 } else {
556 params.param0 |= DWC3_DEPCFG_ACTION_INIT;
265b70a7
PZ		 557 }
558
4bc48c97
FB		 559 if (usb_endpoint_xfer_control(desc))
560 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
13fa2e69
FB		 561
562 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
563 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
72246da4 564
18b7ede5 565 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
dc1c70a7
FB		 566 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
567 | DWC3_DEPCFG_STREAM_EVENT_EN;
879631aa
FB		 568 dep->stream_capable = true;
569 }
570
0b93a4c8 571 if (!usb_endpoint_xfer_control(desc))
dc1c70a7 572 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
72246da4
FB		 573
574 /*
575 * We are doing 1:1 mapping for endpoints, meaning
576 * Physical Endpoints 2 maps to Logical Endpoint 2 and
577 * so on. We consider the direction bit as part of the physical
578 * endpoint number. So USB endpoint 0x81 is 0x03.
579 */
dc1c70a7 580 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
72246da4
FB		 581
582 /*
583 * We must use the lower 16 TX FIFOs even though
584 * HW might have more
585 */
586 if (dep->direction)
dc1c70a7 587 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
72246da4
FB		 588
589 if (desc->bInterval) {
dc1c70a7 590 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
72246da4
FB		 591 dep->interval = 1 << (desc->bInterval - 1);
592 }
593
2cd4718d 594 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
72246da4
FB		 595}
596
597static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
598{
599 struct dwc3_gadget_ep_cmd_params params;
600
601 memset(&params, 0x00, sizeof(params));
602
dc1c70a7 603 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
72246da4 604
2cd4718d
FB		 605 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
606 &params);
72246da4
FB		 607}
608
609/**
bfad65ee 610 * __dwc3_gadget_ep_enable - initializes a hw endpoint
72246da4 611 * @dep: endpoint to be initialized
bfad65ee
FB		 612 * @modify: if true, modify existing endpoint configuration
613 * @restore: if true, restore endpoint configuration from scratch buffer
72246da4 614 *
bfad65ee
FB		 615 * Caller should take care of locking. Execute all necessary commands to
616 * initialize a HW endpoint so it can be used by a gadget driver.
72246da4
FB		 617 */
618static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
21e64bf2 619 bool modify, bool restore)
72246da4 620{
39ebb05c 621 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
72246da4 622 struct dwc3 *dwc = dep->dwc;
39ebb05c 623
72246da4 624 u32 reg;
b09e99ee 625 int ret;
72246da4
FB		 626
627 if (!(dep->flags & DWC3_EP_ENABLED)) {
628 ret = dwc3_gadget_start_config(dwc, dep);
629 if (ret)
630 return ret;
631 }
632
39ebb05c 633 ret = dwc3_gadget_set_ep_config(dwc, dep, modify, restore);
72246da4
FB		 634 if (ret)
635 return ret;
636
637 if (!(dep->flags & DWC3_EP_ENABLED)) {
f6bafc6a
FB		 638 struct dwc3_trb *trb_st_hw;
639 struct dwc3_trb *trb_link;
72246da4 640
72246da4
FB		 641 dep->type = usb_endpoint_type(desc);
642 dep->flags |= DWC3_EP_ENABLED;
76a638f8 643 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
72246da4
FB		 644
645 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
646 reg |= DWC3_DALEPENA_EP(dep->number);
647 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
648
76a638f8
BW		 649 init_waitqueue_head(&dep->wait_end_transfer);
650
36b68aae 651 if (usb_endpoint_xfer_control(desc))
2870e501 652 goto out;
72246da4 653
0d25744a
JY		 654 /* Initialize the TRB ring */
655 dep->trb_dequeue = 0;
656 dep->trb_enqueue = 0;
657 memset(dep->trb_pool, 0,
658 sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
659
36b68aae 660 /* Link TRB. The HWO bit is never reset */
72246da4
FB		 661 trb_st_hw = &dep->trb_pool[0];
662
f6bafc6a 663 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
f6bafc6a
FB		 664 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
665 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
666 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
667 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
72246da4
FB		 668 }
669
a97ea994
FB		 670 /*
671 * Issue StartTransfer here with no-op TRB so we can always rely on No
672 * Response Update Transfer command.
673 */
52fcc0be
FB		 674 if (usb_endpoint_xfer_bulk(desc) ||
675 usb_endpoint_xfer_int(desc)) {
a97ea994
FB		 676 struct dwc3_gadget_ep_cmd_params params;
677 struct dwc3_trb *trb;
678 dma_addr_t trb_dma;
679 u32 cmd;
680
681 memset(&params, 0, sizeof(params));
682 trb = &dep->trb_pool[0];
683 trb_dma = dwc3_trb_dma_offset(dep, trb);
684
685 params.param0 = upper_32_bits(trb_dma);
686 params.param1 = lower_32_bits(trb_dma);
687
688 cmd = DWC3_DEPCMD_STARTTRANSFER;
689
690 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
691 if (ret < 0)
692 return ret;
693
a97ea994
FB		 694 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
695 WARN_ON_ONCE(!dep->resource_index);
696 }
697
2870e501
FB		 698out:
699 trace_dwc3_gadget_ep_enable(dep);
700
72246da4
FB		 701 return 0;
702}
703
8f608e8a 704static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force);
624407f9 705static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
72246da4
FB		 706{
707 struct dwc3_request *req;
708
8f608e8a 709 dwc3_stop_active_transfer(dep, true);
624407f9 710
0e146028
FB		 711 /* - giveback all requests to gadget driver */
712 while (!list_empty(&dep->started_list)) {
713 req = next_request(&dep->started_list);
1591633e 714
0e146028 715 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
ea53b882
FB		 716 }
717
aa3342c8
FB		 718 while (!list_empty(&dep->pending_list)) {
719 req = next_request(&dep->pending_list);
72246da4 720
624407f9 721 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
72246da4 722 }
72246da4
FB		 723}
724
725/**
bfad65ee 726 * __dwc3_gadget_ep_disable - disables a hw endpoint
72246da4
FB		 727 * @dep: the endpoint to disable
728 *
bfad65ee
FB		 729 * This function undoes what __dwc3_gadget_ep_enable did and also removes
730 * requests which are currently being processed by the hardware and those which
731 * are not yet scheduled.
732 *
624407f9 733 * Caller should take care of locking.
72246da4 734 */
72246da4
FB		 735static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
736{
737 struct dwc3 *dwc = dep->dwc;
738 u32 reg;
739
2870e501 740 trace_dwc3_gadget_ep_disable(dep);
7eaeac5c 741
624407f9 742 dwc3_remove_requests(dwc, dep);
72246da4 743
687ef981
FB		 744 /* make sure HW endpoint isn't stalled */
745 if (dep->flags & DWC3_EP_STALL)
7a608559 746 __dwc3_gadget_ep_set_halt(dep, 0, false);
687ef981 747
72246da4
FB		 748 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
749 reg &= ~DWC3_DALEPENA_EP(dep->number);
750 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
751
879631aa 752 dep->stream_capable = false;
72246da4 753 dep->type = 0;
76a638f8 754 dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
72246da4 755
39ebb05c
JY		 756 /* Clear out the ep descriptors for non-ep0 */
757 if (dep->number > 1) {
758 dep->endpoint.comp_desc = NULL;
759 dep->endpoint.desc = NULL;
760 }
761
72246da4
FB		 762 return 0;
763}
764
765/* -------------------------------------------------------------------------- */
766
767static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
768 const struct usb_endpoint_descriptor *desc)
769{
770 return -EINVAL;
771}
772
773static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
774{
775 return -EINVAL;
776}
777
778/* -------------------------------------------------------------------------- */
779
780static int dwc3_gadget_ep_enable(struct usb_ep *ep,
781 const struct usb_endpoint_descriptor *desc)
782{
783 struct dwc3_ep *dep;
784 struct dwc3 *dwc;
785 unsigned long flags;
786 int ret;
787
788 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
789 pr_debug("dwc3: invalid parameters\n");
790 return -EINVAL;
791 }
792
793 if (!desc->wMaxPacketSize) {
794 pr_debug("dwc3: missing wMaxPacketSize\n");
795 return -EINVAL;
796 }
797
798 dep = to_dwc3_ep(ep);
799 dwc = dep->dwc;
800
95ca961c
FB		 801 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
802 "%s is already enabled\n",
803 dep->name))
c6f83f38 804 return 0;
c6f83f38 805
72246da4 806 spin_lock_irqsave(&dwc->lock, flags);
39ebb05c 807 ret = __dwc3_gadget_ep_enable(dep, false, false);
72246da4
FB		 808 spin_unlock_irqrestore(&dwc->lock, flags);
809
810 return ret;
811}
812
813static int dwc3_gadget_ep_disable(struct usb_ep *ep)
814{
815 struct dwc3_ep *dep;
816 struct dwc3 *dwc;
817 unsigned long flags;
818 int ret;
819
820 if (!ep) {
821 pr_debug("dwc3: invalid parameters\n");
822 return -EINVAL;
823 }
824
825 dep = to_dwc3_ep(ep);
826 dwc = dep->dwc;
827
95ca961c
FB		 828 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
829 "%s is already disabled\n",
830 dep->name))
72246da4 831 return 0;
72246da4 832
72246da4
FB		 833 spin_lock_irqsave(&dwc->lock, flags);
834 ret = __dwc3_gadget_ep_disable(dep);
835 spin_unlock_irqrestore(&dwc->lock, flags);
836
837 return ret;
838}
839
840static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
0bd0f6d2 841 gfp_t gfp_flags)
72246da4
FB		 842{
843 struct dwc3_request *req;
844 struct dwc3_ep *dep = to_dwc3_ep(ep);
72246da4
FB		 845
846 req = kzalloc(sizeof(*req), gfp_flags);
734d5a53 847 if (!req)
72246da4 848 return NULL;
72246da4
FB		 849
850 req->epnum = dep->number;
851 req->dep = dep;
72246da4 852
2c4cbe6e
FB		 853 trace_dwc3_alloc_request(req);
854
72246da4
FB		 855 return &req->request;
856}
857
858static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
859 struct usb_request *request)
860{
861 struct dwc3_request *req = to_dwc3_request(request);
862
2c4cbe6e 863 trace_dwc3_free_request(req);
72246da4
FB		 864 kfree(req);
865}
866
2c78c029
FB		 867static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep);
868
e49d3cf4
FB		 869static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
870 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
871 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
c71fc37c 872{
6b9018d4
FB		 873 struct dwc3 *dwc = dep->dwc;
874 struct usb_gadget *gadget = &dwc->gadget;
875 enum usb_device_speed speed = gadget->speed;
c71fc37c 876
ef966b9d 877 dwc3_ep_inc_enq(dep);
e5ba5ec8 878
f6bafc6a
FB		 879 trb->size = DWC3_TRB_SIZE_LENGTH(length);
880 trb->bpl = lower_32_bits(dma);
881 trb->bph = upper_32_bits(dma);
c71fc37c 882
16e78db7 883 switch (usb_endpoint_type(dep->endpoint.desc)) {
c71fc37c 884 case USB_ENDPOINT_XFER_CONTROL:
f6bafc6a 885 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
c71fc37c
FB		 886 break;
887
888 case USB_ENDPOINT_XFER_ISOC:
6b9018d4 889 if (!node) {
e5ba5ec8 890 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
6b9018d4 891
40d829fb
MG		 892 /*
893 * USB Specification 2.0 Section 5.9.2 states that: "If
894 * there is only a single transaction in the microframe,
895 * only a DATA0 data packet PID is used. If there are
896 * two transactions per microframe, DATA1 is used for
897 * the first transaction data packet and DATA0 is used
898 * for the second transaction data packet. If there are
899 * three transactions per microframe, DATA2 is used for
900 * the first transaction data packet, DATA1 is used for
901 * the second, and DATA0 is used for the third."
902 *
903 * IOW, we should satisfy the following cases:
904 *
905 * 1) length <= maxpacket
906 * - DATA0
907 *
908 * 2) maxpacket < length <= (2 * maxpacket)
909 * - DATA1, DATA0
910 *
911 * 3) (2 * maxpacket) < length <= (3 * maxpacket)
912 * - DATA2, DATA1, DATA0
913 */
6b9018d4
FB		 914 if (speed == USB_SPEED_HIGH) {
915 struct usb_ep *ep = &dep->endpoint;
ec5bb87e 916 unsigned int mult = 2;
40d829fb
MG		 917 unsigned int maxp = usb_endpoint_maxp(ep->desc);
918
919 if (length <= (2 * maxp))
920 mult--;
921
922 if (length <= maxp)
923 mult--;
924
925 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
6b9018d4
FB		 926 }
927 } else {
e5ba5ec8 928 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
6b9018d4 929 }
ca4d44ea
FB		 930
931 /* always enable Interrupt on Missed ISOC */
932 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
c71fc37c
FB		 933 break;
934
935 case USB_ENDPOINT_XFER_BULK:
936 case USB_ENDPOINT_XFER_INT:
f6bafc6a 937 trb->ctrl = DWC3_TRBCTL_NORMAL;
c71fc37c
FB		 938 break;
939 default:
940 /*
941 * This is only possible with faulty memory because we
942 * checked it already :)
943 */
0a695d4c
FB		 944 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
945 usb_endpoint_type(dep->endpoint.desc));
c71fc37c
FB		 946 }
947
ca4d44ea 948 /* always enable Continue on Short Packet */
c9508c8c 949 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
58f29034 950 trb->ctrl |= DWC3_TRB_CTRL_CSP;
f3af3651 951
e49d3cf4 952 if (short_not_ok)
c9508c8c
FB		 953 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
954 }
955
e49d3cf4 956 if ((!no_interrupt && !chain) ||
2c78c029 957 (dwc3_calc_trbs_left(dep) == 0))
c9508c8c 958 trb->ctrl |= DWC3_TRB_CTRL_IOC;
f3af3651 959
e5ba5ec8
PA		 960 if (chain)
961 trb->ctrl |= DWC3_TRB_CTRL_CHN;
962
16e78db7 963 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
e49d3cf4 964 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
c71fc37c 965
f6bafc6a 966 trb->ctrl |= DWC3_TRB_CTRL_HWO;
2c4cbe6e
FB		 967
968 trace_dwc3_prepare_trb(dep, trb);
c71fc37c
FB		 969}
970
e49d3cf4
FB		 971/**
972 * dwc3_prepare_one_trb - setup one TRB from one request
973 * @dep: endpoint for which this request is prepared
974 * @req: dwc3_request pointer
975 * @chain: should this TRB be chained to the next?
976 * @node: only for isochronous endpoints. First TRB needs different type.
977 */
978static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
979 struct dwc3_request *req, unsigned chain, unsigned node)
980{
981 struct dwc3_trb *trb;
a31e63b6
AKV		 982 unsigned int length;
983 dma_addr_t dma;
e49d3cf4
FB		 984 unsigned stream_id = req->request.stream_id;
985 unsigned short_not_ok = req->request.short_not_ok;
986 unsigned no_interrupt = req->request.no_interrupt;
a31e63b6
AKV		 987
988 if (req->request.num_sgs > 0) {
989 length = sg_dma_len(req->start_sg);
990 dma = sg_dma_address(req->start_sg);
991 } else {
992 length = req->request.length;
993 dma = req->request.dma;
994 }
e49d3cf4
FB		 995
996 trb = &dep->trb_pool[dep->trb_enqueue];
997
998 if (!req->trb) {
999 dwc3_gadget_move_started_request(req);
1000 req->trb = trb;
1001 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
e49d3cf4
FB		 1002 }
1003
1004 __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
1005 stream_id, short_not_ok, no_interrupt);
1006}
1007
361572b5 1008/**
bfad65ee 1009 * dwc3_ep_prev_trb - returns the previous TRB in the ring
361572b5
JY		 1010 * @dep: The endpoint with the TRB ring
1011 * @index: The index of the current TRB in the ring
1012 *
1013 * Returns the TRB prior to the one pointed to by the index. If the
1014 * index is 0, we will wrap backwards, skip the link TRB, and return
1015 * the one just before that.
1016 */
1017static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1018{
45438a0c 1019 u8 tmp = index;
361572b5 1020
45438a0c
FB		 1021 if (!tmp)
1022 tmp = DWC3_TRB_NUM - 1;
361572b5 1023
45438a0c 1024 return &dep->trb_pool[tmp - 1];
361572b5
JY		 1025}
1026
c4233573
FB		 1027static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1028{
1029 struct dwc3_trb *tmp;
32db3d94 1030 u8 trbs_left;
c4233573
FB		 1031
1032 /*
1033 * If enqueue & dequeue are equal than it is either full or empty.
1034 *
1035 * One way to know for sure is if the TRB right before us has HWO bit
1036 * set or not. If it has, then we're definitely full and can't fit any
1037 * more transfers in our ring.
1038 */
1039 if (dep->trb_enqueue == dep->trb_dequeue) {
361572b5 1040 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
202adafe 1041 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
361572b5 1042 return 0;
c4233573
FB		 1043
1044 return DWC3_TRB_NUM - 1;
1045 }
1046
9d7aba77 1047 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
3de2685f 1048 trbs_left &= (DWC3_TRB_NUM - 1);
32db3d94 1049
9d7aba77
JY		 1050 if (dep->trb_dequeue < dep->trb_enqueue)
1051 trbs_left--;
1052
32db3d94 1053 return trbs_left;
c4233573
FB		 1054}
1055
5ee85d89 1056static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
7ae7df49 1057 struct dwc3_request *req)
5ee85d89 1058{
a31e63b6 1059 struct scatterlist *sg = req->start_sg;
5ee85d89 1060 struct scatterlist *s;
5ee85d89
FB		 1061 int i;
1062
c96e6725
AKV		 1063 unsigned int remaining = req->request.num_mapped_sgs
1064 - req->num_queued_sgs;
1065
1066 for_each_sg(sg, s, remaining, i) {
c6267a51
FB		 1067 unsigned int length = req->request.length;
1068 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1069 unsigned int rem = length % maxp;
5ee85d89
FB		 1070 unsigned chain = true;
1071
4bc48c97 1072 if (sg_is_last(s))
5ee85d89
FB		 1073 chain = false;
1074
c6267a51
FB		 1075 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1076 struct dwc3 *dwc = dep->dwc;
1077 struct dwc3_trb *trb;
1078
1079 req->unaligned = true;
1080
1081 /* prepare normal TRB */
1082 dwc3_prepare_one_trb(dep, req, true, i);
1083
1084 /* Now prepare one extra TRB to align transfer size */
1085 trb = &dep->trb_pool[dep->trb_enqueue];
1086 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1087 maxp - rem, false, 0,
1088 req->request.stream_id,
1089 req->request.short_not_ok,
1090 req->request.no_interrupt);
1091 } else {
1092 dwc3_prepare_one_trb(dep, req, chain, i);
1093 }
5ee85d89 1094
a31e63b6
AKV		 1095 /*
1096 * There can be a situation where all sgs in sglist are not
1097 * queued because of insufficient trb number. To handle this
1098 * case, update start_sg to next sg to be queued, so that
1099 * we have free trbs we can continue queuing from where we
1100 * previously stopped
1101 */
1102 if (chain)
1103 req->start_sg = sg_next(s);
1104
c96e6725
AKV		 1105 req->num_queued_sgs++;
1106
7ae7df49 1107 if (!dwc3_calc_trbs_left(dep))
5ee85d89
FB		 1108 break;
1109 }
1110}
1111
1112static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
7ae7df49 1113 struct dwc3_request *req)
5ee85d89 1114{
c6267a51
FB		 1115 unsigned int length = req->request.length;
1116 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1117 unsigned int rem = length % maxp;
1118
1119 if (rem && usb_endpoint_dir_out(dep->endpoint.desc)) {
1120 struct dwc3 *dwc = dep->dwc;
1121 struct dwc3_trb *trb;
1122
1123 req->unaligned = true;
1124
1125 /* prepare normal TRB */
1126 dwc3_prepare_one_trb(dep, req, true, 0);
1127
1128 /* Now prepare one extra TRB to align transfer size */
1129 trb = &dep->trb_pool[dep->trb_enqueue];
1130 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1131 false, 0, req->request.stream_id,
1132 req->request.short_not_ok,
1133 req->request.no_interrupt);
d6e5a549
FB		 1134 } else if (req->request.zero && req->request.length &&
1135 (IS_ALIGNED(req->request.length,dep->endpoint.maxpacket))) {
1136 struct dwc3 *dwc = dep->dwc;
1137 struct dwc3_trb *trb;
1138
1139 req->zero = true;
1140
1141 /* prepare normal TRB */
1142 dwc3_prepare_one_trb(dep, req, true, 0);
1143
1144 /* Now prepare one extra TRB to handle ZLP */
1145 trb = &dep->trb_pool[dep->trb_enqueue];
1146 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1147 false, 0, req->request.stream_id,
1148 req->request.short_not_ok,
1149 req->request.no_interrupt);
c6267a51
FB		 1150 } else {
1151 dwc3_prepare_one_trb(dep, req, false, 0);
1152 }
5ee85d89
FB		 1153}
1154
72246da4
FB		 1155/*
1156 * dwc3_prepare_trbs - setup TRBs from requests
1157 * @dep: endpoint for which requests are being prepared
72246da4 1158 *
1d046793
PZ		 1159 * The function goes through the requests list and sets up TRBs for the
1160 * transfers. The function returns once there are no more TRBs available or
1161 * it runs out of requests.
72246da4 1162 */
c4233573 1163static void dwc3_prepare_trbs(struct dwc3_ep *dep)
72246da4 1164{
68e823e2 1165 struct dwc3_request *req, *n;
72246da4
FB		 1166
1167 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1168
d86c5a67
FB		 1169 /*
1170 * We can get in a situation where there's a request in the started list
1171 * but there weren't enough TRBs to fully kick it in the first time
1172 * around, so it has been waiting for more TRBs to be freed up.
1173 *
1174 * In that case, we should check if we have a request with pending_sgs
1175 * in the started list and prepare TRBs for that request first,
1176 * otherwise we will prepare TRBs completely out of order and that will
1177 * break things.
1178 */
1179 list_for_each_entry(req, &dep->started_list, list) {
1180 if (req->num_pending_sgs > 0)
1181 dwc3_prepare_one_trb_sg(dep, req);
1182
1183 if (!dwc3_calc_trbs_left(dep))
1184 return;
1185 }
1186
aa3342c8 1187 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
cdb55b39
FB		 1188 struct dwc3 *dwc = dep->dwc;
1189 int ret;
1190
1191 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1192 dep->direction);
1193 if (ret)
1194 return;
1195
1196 req->sg = req->request.sg;
a31e63b6 1197 req->start_sg = req->sg;
c96e6725 1198 req->num_queued_sgs = 0;
cdb55b39
FB		 1199 req->num_pending_sgs = req->request.num_mapped_sgs;
1200
1f512119 1201 if (req->num_pending_sgs > 0)
7ae7df49 1202 dwc3_prepare_one_trb_sg(dep, req);
5ee85d89 1203 else
7ae7df49 1204 dwc3_prepare_one_trb_linear(dep, req);
72246da4 1205
7ae7df49 1206 if (!dwc3_calc_trbs_left(dep))
5ee85d89 1207 return;
72246da4 1208 }
72246da4
FB		 1209}
1210
7fdca766 1211static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
72246da4
FB		 1212{
1213 struct dwc3_gadget_ep_cmd_params params;
1214 struct dwc3_request *req;
4fae2e3e 1215 int starting;
72246da4
FB		 1216 int ret;
1217 u32 cmd;
1218
ccb94ebf
FB		 1219 if (!dwc3_calc_trbs_left(dep))
1220 return 0;
1221
1912cbc6 1222 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
72246da4 1223
4fae2e3e
FB		 1224 dwc3_prepare_trbs(dep);
1225 req = next_request(&dep->started_list);
72246da4
FB		 1226 if (!req) {
1227 dep->flags |= DWC3_EP_PENDING_REQUEST;
1228 return 0;
1229 }
1230
1231 memset(&params, 0, sizeof(params));
72246da4 1232
4fae2e3e 1233 if (starting) {
1877d6c9
PA		 1234 params.param0 = upper_32_bits(req->trb_dma);
1235 params.param1 = lower_32_bits(req->trb_dma);
7fdca766
FB		 1236 cmd = DWC3_DEPCMD_STARTTRANSFER;
1237
1238 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1239 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1877d6c9 1240 } else {
b6b1c6db
FB		 1241 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1242 DWC3_DEPCMD_PARAM(dep->resource_index);
1877d6c9 1243 }
72246da4 1244
2cd4718d 1245 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
72246da4 1246 if (ret < 0) {
72246da4
FB		 1247 /*
1248 * FIXME we need to iterate over the list of requests
1249 * here and stop, unmap, free and del each of the linked
1d046793 1250 * requests instead of what we do now.
72246da4 1251 */
ce3fc8b3
JD		 1252 if (req->trb)
1253 memset(req->trb, 0, sizeof(struct dwc3_trb));
c91815b5 1254 dwc3_gadget_del_and_unmap_request(dep, req, ret);
72246da4
FB		 1255 return ret;
1256 }
1257
4fae2e3e 1258 if (starting) {
2eb88016 1259 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
b4996a86 1260 WARN_ON_ONCE(!dep->resource_index);
f898ae09 1261 }
25b8ff68 1262
72246da4
FB		 1263 return 0;
1264}
1265
6cb2e4e3
FB		 1266static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1267{
1268 u32 reg;
1269
1270 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1271 return DWC3_DSTS_SOFFN(reg);
1272}
1273
5828cada 1274static void __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
d6d6ec7b 1275{
aa3342c8 1276 if (list_empty(&dep->pending_list)) {
8f608e8a 1277 dev_info(dep->dwc->dev, "%s: ran out of requests\n",
73815280 1278 dep->name);
f4a53c55 1279 dep->flags |= DWC3_EP_PENDING_REQUEST;
d6d6ec7b
PA		 1280 return;
1281 }
1282
af771d73
JY		 1283 /*
1284 * Schedule the first trb for one interval in the future or at
1285 * least 4 microframes.
1286 */
5828cada 1287 dep->frame_number += max_t(u32, 4, dep->interval);
7fdca766 1288 __dwc3_gadget_kick_transfer(dep);
d6d6ec7b
PA		 1289}
1290
72246da4
FB		 1291static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1292{
0fc9a1be 1293 struct dwc3 *dwc = dep->dwc;
0fc9a1be 1294
bb423984 1295 if (!dep->endpoint.desc) {
5eb30ced
FB		 1296 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1297 dep->name);
bb423984
FB		 1298 return -ESHUTDOWN;
1299 }
1300
04fb365c
FB		 1301 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1302 &req->request, req->dep->name))
bb423984 1303 return -EINVAL;
bb423984 1304
fc8bb91b
FB		 1305 pm_runtime_get(dwc->dev);
1306
72246da4
FB		 1307 req->request.actual = 0;
1308 req->request.status = -EINPROGRESS;
1309 req->direction = dep->direction;
1310 req->epnum = dep->number;
1311
fe84f522
FB		 1312 trace_dwc3_ep_queue(req);
1313
aa3342c8 1314 list_add_tail(&req->list, &dep->pending_list);
72246da4 1315
d889c23c
FB		 1316 /*
1317 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1318 * wait for a XferNotReady event so we will know what's the current
1319 * (micro-)frame number.
1320 *
1321 * Without this trick, we are very, very likely gonna get Bus Expiry
1322 * errors which will force us issue EndTransfer command.
1323 */
1324 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
fe990cea
FB		 1325 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1326 !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1327 return 0;
1328
6cb2e4e3 1329 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
fe990cea 1330 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
5828cada 1331 __dwc3_gadget_start_isoc(dep);
fe990cea 1332 return 0;
6cb2e4e3 1333 }
08a36b54 1334 }
64e01080 1335 }
b997ada5 1336
7fdca766 1337 return __dwc3_gadget_kick_transfer(dep);
72246da4
FB		 1338}
1339
1340static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1341 gfp_t gfp_flags)
1342{
1343 struct dwc3_request *req = to_dwc3_request(request);
1344 struct dwc3_ep *dep = to_dwc3_ep(ep);
1345 struct dwc3 *dwc = dep->dwc;
1346
1347 unsigned long flags;
1348
1349 int ret;
1350
fdee4eba 1351 spin_lock_irqsave(&dwc->lock, flags);
72246da4
FB		 1352 ret = __dwc3_gadget_ep_queue(dep, req);
1353 spin_unlock_irqrestore(&dwc->lock, flags);
1354
1355 return ret;
1356}
1357
1358static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1359 struct usb_request *request)
1360{
1361 struct dwc3_request *req = to_dwc3_request(request);
1362 struct dwc3_request *r = NULL;
1363
1364 struct dwc3_ep *dep = to_dwc3_ep(ep);
1365 struct dwc3 *dwc = dep->dwc;
1366
1367 unsigned long flags;
1368 int ret = 0;
1369
2c4cbe6e
FB		 1370 trace_dwc3_ep_dequeue(req);
1371
72246da4
FB		 1372 spin_lock_irqsave(&dwc->lock, flags);
1373
aa3342c8 1374 list_for_each_entry(r, &dep->pending_list, list) {
72246da4
FB		 1375 if (r == req)
1376 break;
1377 }
1378
1379 if (r != req) {
aa3342c8 1380 list_for_each_entry(r, &dep->started_list, list) {
72246da4
FB		 1381 if (r == req)
1382 break;
1383 }
1384 if (r == req) {
1385 /* wait until it is processed */
8f608e8a 1386 dwc3_stop_active_transfer(dep, true);
cf3113d8
FB		 1387
1388 /*
1389 * If request was already started, this means we had to
1390 * stop the transfer. With that we also need to ignore
1391 * all TRBs used by the request, however TRBs can only
1392 * be modified after completion of END_TRANSFER
1393 * command. So what we do here is that we wait for
1394 * END_TRANSFER completion and only after that, we jump
1395 * over TRBs by clearing HWO and incrementing dequeue
1396 * pointer.
1397 *
1398 * Note that we have 2 possible types of transfers here:
1399 *
1400 * i) Linear buffer request
1401 * ii) SG-list based request
1402 *
1403 * SG-list based requests will have r->num_pending_sgs
1404 * set to a valid number (> 0). Linear requests,
1405 * normally use a single TRB.
1406 *
1407 * For each of these two cases, if r->unaligned flag is
1408 * set, one extra TRB has been used to align transfer
1409 * size to wMaxPacketSize.
1410 *
1411 * All of these cases need to be taken into
1412 * consideration so we don't mess up our TRB ring
1413 * pointers.
1414 */
1415 wait_event_lock_irq(dep->wait_end_transfer,
1416 !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1417 dwc->lock);
1418
1419 if (!r->trb)
1420 goto out1;
1421
1422 if (r->num_pending_sgs) {
1423 struct dwc3_trb *trb;
1424 int i = 0;
1425
1426 for (i = 0; i < r->num_pending_sgs; i++) {
1427 trb = r->trb + i;
1428 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1429 dwc3_ep_inc_deq(dep);
1430 }
1431
d6e5a549 1432 if (r->unaligned || r->zero) {
cf3113d8
FB		 1433 trb = r->trb + r->num_pending_sgs + 1;
1434 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1435 dwc3_ep_inc_deq(dep);
1436 }
1437 } else {
1438 struct dwc3_trb *trb = r->trb;
1439
1440 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1441 dwc3_ep_inc_deq(dep);
1442
d6e5a549 1443 if (r->unaligned || r->zero) {
cf3113d8
FB		 1444 trb = r->trb + 1;
1445 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1446 dwc3_ep_inc_deq(dep);
1447 }
1448 }
e8d4e8be 1449 goto out1;
72246da4 1450 }
04fb365c 1451 dev_err(dwc->dev, "request %pK was not queued to %s\n",
72246da4
FB		 1452 request, ep->name);
1453 ret = -EINVAL;
1454 goto out0;
1455 }
1456
e8d4e8be 1457out1:
72246da4 1458 /* giveback the request */
0bd0f6d2 1459
72246da4
FB		 1460 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1461
1462out0:
1463 spin_unlock_irqrestore(&dwc->lock, flags);
1464
1465 return ret;
1466}
1467
7a608559 1468int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
72246da4
FB		 1469{
1470 struct dwc3_gadget_ep_cmd_params params;
1471 struct dwc3 *dwc = dep->dwc;
1472 int ret;
1473
5ad02fb8
FB		 1474 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1475 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1476 return -EINVAL;
1477 }
1478
72246da4
FB		 1479 memset(&params, 0x00, sizeof(params));
1480
1481 if (value) {
69450c4d
FB		 1482 struct dwc3_trb *trb;
1483
1484 unsigned transfer_in_flight;
1485 unsigned started;
1486
ffb80fc6
FB		 1487 if (dep->flags & DWC3_EP_STALL)
1488 return 0;
1489
69450c4d
FB		 1490 if (dep->number > 1)
1491 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1492 else
1493 trb = &dwc->ep0_trb[dep->trb_enqueue];
1494
1495 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1496 started = !list_empty(&dep->started_list);
1497
1498 if (!protocol && ((dep->direction && transfer_in_flight) ||
1499 (!dep->direction && started))) {
7a608559
FB		 1500 return -EAGAIN;
1501 }
1502
2cd4718d
FB		 1503 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1504 &params);
72246da4 1505 if (ret)
3f89204b 1506 dev_err(dwc->dev, "failed to set STALL on %s\n",
72246da4
FB		 1507 dep->name);
1508 else
1509 dep->flags |= DWC3_EP_STALL;
1510 } else {
ffb80fc6
FB		 1511 if (!(dep->flags & DWC3_EP_STALL))
1512 return 0;
2cd4718d 1513
50c763f8 1514 ret = dwc3_send_clear_stall_ep_cmd(dep);
72246da4 1515 if (ret)
3f89204b 1516 dev_err(dwc->dev, "failed to clear STALL on %s\n",
72246da4
FB		 1517 dep->name);
1518 else
a535d81c 1519 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
72246da4 1520 }
5275455a 1521
72246da4
FB		 1522 return ret;
1523}
1524
1525static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1526{
1527 struct dwc3_ep *dep = to_dwc3_ep(ep);
1528 struct dwc3 *dwc = dep->dwc;
1529
1530 unsigned long flags;
1531
1532 int ret;
1533
1534 spin_lock_irqsave(&dwc->lock, flags);
7a608559 1535 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
72246da4
FB		 1536 spin_unlock_irqrestore(&dwc->lock, flags);
1537
1538 return ret;
1539}
1540
1541static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1542{
1543 struct dwc3_ep *dep = to_dwc3_ep(ep);
249a4569
PZ		 1544 struct dwc3 *dwc = dep->dwc;
1545 unsigned long flags;
95aa4e8d 1546 int ret;
72246da4 1547
249a4569 1548 spin_lock_irqsave(&dwc->lock, flags);
72246da4
FB		 1549 dep->flags |= DWC3_EP_WEDGE;
1550
08f0d966 1551 if (dep->number == 0 || dep->number == 1)
95aa4e8d 1552 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
08f0d966 1553 else
7a608559 1554 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
95aa4e8d
FB		 1555 spin_unlock_irqrestore(&dwc->lock, flags);
1556
1557 return ret;
72246da4
FB		 1558}
1559
1560/* -------------------------------------------------------------------------- */
1561
1562static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1563 .bLength = USB_DT_ENDPOINT_SIZE,
1564 .bDescriptorType = USB_DT_ENDPOINT,
1565 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1566};
1567
1568static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1569 .enable = dwc3_gadget_ep0_enable,
1570 .disable = dwc3_gadget_ep0_disable,
1571 .alloc_request = dwc3_gadget_ep_alloc_request,
1572 .free_request = dwc3_gadget_ep_free_request,
1573 .queue = dwc3_gadget_ep0_queue,
1574 .dequeue = dwc3_gadget_ep_dequeue,
08f0d966 1575 .set_halt = dwc3_gadget_ep0_set_halt,
72246da4
FB		 1576 .set_wedge = dwc3_gadget_ep_set_wedge,
1577};
1578
1579static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1580 .enable = dwc3_gadget_ep_enable,
1581 .disable = dwc3_gadget_ep_disable,
1582 .alloc_request = dwc3_gadget_ep_alloc_request,
1583 .free_request = dwc3_gadget_ep_free_request,
1584 .queue = dwc3_gadget_ep_queue,
1585 .dequeue = dwc3_gadget_ep_dequeue,
1586 .set_halt = dwc3_gadget_ep_set_halt,
1587 .set_wedge = dwc3_gadget_ep_set_wedge,
1588};
1589
1590/* -------------------------------------------------------------------------- */
1591
1592static int dwc3_gadget_get_frame(struct usb_gadget *g)
1593{
1594 struct dwc3 *dwc = gadget_to_dwc(g);
72246da4 1595
6cb2e4e3 1596 return __dwc3_gadget_get_frame(dwc);
72246da4
FB		 1597}
1598
218ef7b6 1599static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
72246da4 1600{
d6011f6f 1601 int retries;
72246da4 1602
218ef7b6 1603 int ret;
72246da4
FB		 1604 u32 reg;
1605
72246da4
FB		 1606 u8 link_state;
1607 u8 speed;
1608
72246da4
FB		 1609 /*
1610 * According to the Databook Remote wakeup request should
1611 * be issued only when the device is in early suspend state.
1612 *
1613 * We can check that via USB Link State bits in DSTS register.
1614 */
1615 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1616
1617 speed = reg & DWC3_DSTS_CONNECTSPD;
ee5cd41c 1618 if ((speed == DWC3_DSTS_SUPERSPEED) ||
5eb30ced 1619 (speed == DWC3_DSTS_SUPERSPEED_PLUS))
6b742899 1620 return 0;
72246da4
FB		 1621
1622 link_state = DWC3_DSTS_USBLNKST(reg);
1623
1624 switch (link_state) {
1625 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
1626 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
1627 break;
1628 default:
218ef7b6 1629 return -EINVAL;
72246da4
FB		 1630 }
1631
8598bde7
FB		 1632 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1633 if (ret < 0) {
1634 dev_err(dwc->dev, "failed to put link in Recovery\n");
218ef7b6 1635 return ret;
8598bde7 1636 }
72246da4 1637
802fde98
PZ		 1638 /* Recent versions do this automatically */
1639 if (dwc->revision < DWC3_REVISION_194A) {
1640 /* write zeroes to Link Change Request */
fcc023c7 1641 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
802fde98
PZ		 1642 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1643 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1644 }
72246da4 1645
1d046793 1646 /* poll until Link State changes to ON */
d6011f6f 1647 retries = 20000;
72246da4 1648
d6011f6f 1649 while (retries--) {
72246da4
FB		 1650 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1651
1652 /* in HS, means ON */
1653 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1654 break;
1655 }
1656
1657 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1658 dev_err(dwc->dev, "failed to send remote wakeup\n");
218ef7b6 1659 return -EINVAL;
72246da4
FB		 1660 }
1661
218ef7b6
FB		 1662 return 0;
1663}
1664
1665static int dwc3_gadget_wakeup(struct usb_gadget *g)
1666{
1667 struct dwc3 *dwc = gadget_to_dwc(g);
1668 unsigned long flags;
1669 int ret;
1670
1671 spin_lock_irqsave(&dwc->lock, flags);
1672 ret = __dwc3_gadget_wakeup(dwc);
72246da4
FB		 1673 spin_unlock_irqrestore(&dwc->lock, flags);
1674
1675 return ret;
1676}
1677
1678static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1679 int is_selfpowered)
1680{
1681 struct dwc3 *dwc = gadget_to_dwc(g);
249a4569 1682 unsigned long flags;
72246da4 1683
249a4569 1684 spin_lock_irqsave(&dwc->lock, flags);
bcdea503 1685 g->is_selfpowered = !!is_selfpowered;
249a4569 1686 spin_unlock_irqrestore(&dwc->lock, flags);
72246da4
FB		 1687
1688 return 0;
1689}
1690
7b2a0368 1691static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
72246da4
FB		 1692{
1693 u32 reg;
61d58242 1694 u32 timeout = 500;
72246da4 1695
fc8bb91b
FB		 1696 if (pm_runtime_suspended(dwc->dev))
1697 return 0;
1698
72246da4 1699 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
8db7ed15 1700 if (is_on) {
802fde98
PZ		 1701 if (dwc->revision <= DWC3_REVISION_187A) {
1702 reg &= ~DWC3_DCTL_TRGTULST_MASK;
1703 reg |= DWC3_DCTL_TRGTULST_RX_DET;
1704 }
1705
1706 if (dwc->revision >= DWC3_REVISION_194A)
1707 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1708 reg |= DWC3_DCTL_RUN_STOP;
7b2a0368
FB		 1709
1710 if (dwc->has_hibernation)
1711 reg |= DWC3_DCTL_KEEP_CONNECT;
1712
9fcb3bd8 1713 dwc->pullups_connected = true;
8db7ed15 1714 } else {
72246da4 1715 reg &= ~DWC3_DCTL_RUN_STOP;
7b2a0368
FB		 1716
1717 if (dwc->has_hibernation && !suspend)
1718 reg &= ~DWC3_DCTL_KEEP_CONNECT;
1719
9fcb3bd8 1720 dwc->pullups_connected = false;
8db7ed15 1721 }
72246da4
FB		 1722
1723 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1724
1725 do {
1726 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
b6d4e16e
FB		 1727 reg &= DWC3_DSTS_DEVCTRLHLT;
1728 } while (--timeout && !(!is_on ^ !reg));
f2df679b
FB		 1729
1730 if (!timeout)
1731 return -ETIMEDOUT;
72246da4 1732
6f17f74b 1733 return 0;
72246da4
FB		 1734}
1735
1736static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1737{
1738 struct dwc3 *dwc = gadget_to_dwc(g);
1739 unsigned long flags;
6f17f74b 1740 int ret;
72246da4
FB		 1741
1742 is_on = !!is_on;
1743
bb014736
BW		 1744 /*
1745 * Per databook, when we want to stop the gadget, if a control transfer
1746 * is still in process, complete it and get the core into setup phase.
1747 */
1748 if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1749 reinit_completion(&dwc->ep0_in_setup);
1750
1751 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1752 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1753 if (ret == 0) {
1754 dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1755 return -ETIMEDOUT;
1756 }
1757 }
1758
72246da4 1759 spin_lock_irqsave(&dwc->lock, flags);
7b2a0368 1760 ret = dwc3_gadget_run_stop(dwc, is_on, false);
72246da4
FB		 1761 spin_unlock_irqrestore(&dwc->lock, flags);
1762
6f17f74b 1763 return ret;
72246da4
FB		 1764}
1765
8698e2ac
FB		 1766static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1767{
1768 u32 reg;
1769
1770 /* Enable all but Start and End of Frame IRQs */
1771 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1772 DWC3_DEVTEN_EVNTOVERFLOWEN |
1773 DWC3_DEVTEN_CMDCMPLTEN |
1774 DWC3_DEVTEN_ERRTICERREN |
1775 DWC3_DEVTEN_WKUPEVTEN |
8698e2ac
FB		 1776 DWC3_DEVTEN_CONNECTDONEEN |
1777 DWC3_DEVTEN_USBRSTEN |
1778 DWC3_DEVTEN_DISCONNEVTEN);
1779
799e9dc8
FB		 1780 if (dwc->revision < DWC3_REVISION_250A)
1781 reg |= DWC3_DEVTEN_ULSTCNGEN;
1782
8698e2ac
FB		 1783 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1784}
1785
1786static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1787{
1788 /* mask all interrupts */
1789 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1790}
1791
1792static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
b15a762f 1793static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
8698e2ac 1794
4e99472b 1795/**
bfad65ee
FB		 1796 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
1797 * @dwc: pointer to our context structure
4e99472b
FB		 1798 *
1799 * The following looks like complex but it's actually very simple. In order to
1800 * calculate the number of packets we can burst at once on OUT transfers, we're
1801 * gonna use RxFIFO size.
1802 *
1803 * To calculate RxFIFO size we need two numbers:
1804 * MDWIDTH = size, in bits, of the internal memory bus
1805 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1806 *
1807 * Given these two numbers, the formula is simple:
1808 *
1809 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1810 *
1811 * 24 bytes is for 3x SETUP packets
1812 * 16 bytes is a clock domain crossing tolerance
1813 *
1814 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1815 */
1816static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1817{
1818 u32 ram2_depth;
1819 u32 mdwidth;
1820 u32 nump;
1821 u32 reg;
1822
1823 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1824 mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1825
1826 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1827 nump = min_t(u32, nump, 16);
1828
1829 /* update NumP */
1830 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1831 reg &= ~DWC3_DCFG_NUMP_MASK;
1832 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1833 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1834}
1835
d7be2952 1836static int __dwc3_gadget_start(struct dwc3 *dwc)
72246da4 1837{
72246da4 1838 struct dwc3_ep *dep;
72246da4
FB		 1839 int ret = 0;
1840 u32 reg;
1841
cf40b86b
JY		 1842 /*
1843 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
1844 * the core supports IMOD, disable it.
1845 */
1846 if (dwc->imod_interval) {
1847 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1848 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1849 } else if (dwc3_has_imod(dwc)) {
1850 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
1851 }
1852
2a58f9c1
FB		 1853 /*
1854 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1855 * field instead of letting dwc3 itself calculate that automatically.
1856 *
1857 * This way, we maximize the chances that we'll be able to get several
1858 * bursts of data without going through any sort of endpoint throttling.
1859 */
1860 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
01b0e2cc
TN		 1861 if (dwc3_is_usb31(dwc))
1862 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
1863 else
1864 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1865
2a58f9c1
FB		 1866 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1867
4e99472b
FB		 1868 dwc3_gadget_setup_nump(dwc);
1869
72246da4
FB		 1870 /* Start with SuperSpeed Default */
1871 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1872
1873 dep = dwc->eps[0];
39ebb05c 1874 ret = __dwc3_gadget_ep_enable(dep, false, false);
72246da4
FB		 1875 if (ret) {
1876 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
d7be2952 1877 goto err0;
72246da4
FB		 1878 }
1879
1880 dep = dwc->eps[1];
39ebb05c 1881 ret = __dwc3_gadget_ep_enable(dep, false, false);
72246da4
FB		 1882 if (ret) {
1883 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
d7be2952 1884 goto err1;
72246da4
FB		 1885 }
1886
1887 /* begin to receive SETUP packets */
c7fcdeb2 1888 dwc->ep0state = EP0_SETUP_PHASE;
72246da4
FB		 1889 dwc3_ep0_out_start(dwc);
1890
8698e2ac
FB		 1891 dwc3_gadget_enable_irq(dwc);
1892
72246da4
FB		 1893 return 0;
1894
b0d7ffd4 1895err1:
d7be2952 1896 __dwc3_gadget_ep_disable(dwc->eps[0]);
b0d7ffd4
FB		 1897
1898err0:
72246da4
FB		 1899 return ret;
1900}
1901
d7be2952
FB		 1902static int dwc3_gadget_start(struct usb_gadget *g,
1903 struct usb_gadget_driver *driver)
72246da4
FB		 1904{
1905 struct dwc3 *dwc = gadget_to_dwc(g);
1906 unsigned long flags;
d7be2952 1907 int ret = 0;
8698e2ac 1908 int irq;
72246da4 1909
9522def4 1910 irq = dwc->irq_gadget;
d7be2952
FB		 1911 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1912 IRQF_SHARED, "dwc3", dwc->ev_buf);
1913 if (ret) {
1914 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1915 irq, ret);
1916 goto err0;
1917 }
1918
72246da4 1919 spin_lock_irqsave(&dwc->lock, flags);
d7be2952
FB		 1920 if (dwc->gadget_driver) {
1921 dev_err(dwc->dev, "%s is already bound to %s\n",
1922 dwc->gadget.name,
1923 dwc->gadget_driver->driver.name);
1924 ret = -EBUSY;
1925 goto err1;
1926 }
1927
1928 dwc->gadget_driver = driver;
1929
fc8bb91b
FB		 1930 if (pm_runtime_active(dwc->dev))
1931 __dwc3_gadget_start(dwc);
1932
d7be2952
FB		 1933 spin_unlock_irqrestore(&dwc->lock, flags);
1934
1935 return 0;
1936
1937err1:
1938 spin_unlock_irqrestore(&dwc->lock, flags);
1939 free_irq(irq, dwc);
1940
1941err0:
1942 return ret;
1943}
72246da4 1944
d7be2952
FB		 1945static void __dwc3_gadget_stop(struct dwc3 *dwc)
1946{
8698e2ac 1947 dwc3_gadget_disable_irq(dwc);
72246da4
FB		 1948 __dwc3_gadget_ep_disable(dwc->eps[0]);
1949 __dwc3_gadget_ep_disable(dwc->eps[1]);
d7be2952 1950}
72246da4 1951
d7be2952
FB		 1952static int dwc3_gadget_stop(struct usb_gadget *g)
1953{
1954 struct dwc3 *dwc = gadget_to_dwc(g);
1955 unsigned long flags;
76a638f8 1956 int epnum;
498f0478 1957 u32 tmo_eps = 0;
72246da4 1958
d7be2952 1959 spin_lock_irqsave(&dwc->lock, flags);
76a638f8
BW		 1960
1961 if (pm_runtime_suspended(dwc->dev))
1962 goto out;
1963
d7be2952 1964 __dwc3_gadget_stop(dwc);
76a638f8
BW		 1965
1966 for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1967 struct dwc3_ep *dep = dwc->eps[epnum];
498f0478 1968 int ret;
76a638f8
BW		 1969
1970 if (!dep)
1971 continue;
1972
1973 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1974 continue;
1975
498f0478
RQ		 1976 ret = wait_event_interruptible_lock_irq_timeout(dep->wait_end_transfer,
1977 !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1978 dwc->lock, msecs_to_jiffies(5));
1979
1980 if (ret <= 0) {
1981 /* Timed out or interrupted! There's nothing much
1982 * we can do so we just log here and print which
1983 * endpoints timed out at the end.
1984 */
1985 tmo_eps |= 1 << epnum;
1986 dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
1987 }
1988 }
1989
1990 if (tmo_eps) {
1991 dev_err(dwc->dev,
1992 "end transfer timed out on endpoints 0x%x [bitmap]\n",
1993 tmo_eps);
76a638f8
BW		 1994 }
1995
1996out:
d7be2952 1997 dwc->gadget_driver = NULL;
72246da4
FB		 1998 spin_unlock_irqrestore(&dwc->lock, flags);
1999
3f308d17 2000 free_irq(dwc->irq_gadget, dwc->ev_buf);
b0d7ffd4 2001
72246da4
FB		 2002 return 0;
2003}
802fde98 2004
7d8d0639
FB		 2005static void dwc3_gadget_set_speed(struct usb_gadget *g,
2006 enum usb_device_speed speed)
2007{
2008 struct dwc3 *dwc = gadget_to_dwc(g);
2009 unsigned long flags;
2010 u32 reg;
2011
2012 spin_lock_irqsave(&dwc->lock, flags);
2013 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2014 reg &= ~(DWC3_DCFG_SPEED_MASK);
2015
2016 /*
2017 * WORKAROUND: DWC3 revision < 2.20a have an issue
2018 * which would cause metastability state on Run/Stop
2019 * bit if we try to force the IP to USB2-only mode.
2020 *
2021 * Because of that, we cannot configure the IP to any
2022 * speed other than the SuperSpeed
2023 *
2024 * Refers to:
2025 *
2026 * STAR#9000525659: Clock Domain Crossing on DCTL in
2027 * USB 2.0 Mode
2028 */
42bf02ec
RQ		 2029 if (dwc->revision < DWC3_REVISION_220A &&
2030 !dwc->dis_metastability_quirk) {
7d8d0639
FB		 2031 reg |= DWC3_DCFG_SUPERSPEED;
2032 } else {
2033 switch (speed) {
2034 case USB_SPEED_LOW:
2035 reg |= DWC3_DCFG_LOWSPEED;
2036 break;
2037 case USB_SPEED_FULL:
2038 reg |= DWC3_DCFG_FULLSPEED;
2039 break;
2040 case USB_SPEED_HIGH:
2041 reg |= DWC3_DCFG_HIGHSPEED;
2042 break;
2043 case USB_SPEED_SUPER:
2044 reg |= DWC3_DCFG_SUPERSPEED;
2045 break;
2046 case USB_SPEED_SUPER_PLUS:
2f3090c6
TN		 2047 if (dwc3_is_usb31(dwc))
2048 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2049 else
2050 reg |= DWC3_DCFG_SUPERSPEED;
7d8d0639
FB		 2051 break;
2052 default:
2053 dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2054
2055 if (dwc->revision & DWC3_REVISION_IS_DWC31)
2056 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2057 else
2058 reg |= DWC3_DCFG_SUPERSPEED;
2059 }
2060 }
2061 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2062
2063 spin_unlock_irqrestore(&dwc->lock, flags);
2064}
2065
72246da4
FB		 2066static const struct usb_gadget_ops dwc3_gadget_ops = {
2067 .get_frame = dwc3_gadget_get_frame,
2068 .wakeup = dwc3_gadget_wakeup,
2069 .set_selfpowered = dwc3_gadget_set_selfpowered,
2070 .pullup = dwc3_gadget_pullup,
2071 .udc_start = dwc3_gadget_start,
2072 .udc_stop = dwc3_gadget_stop,
7d8d0639 2073 .udc_set_speed = dwc3_gadget_set_speed,
72246da4
FB		 2074};
2075
2076/* -------------------------------------------------------------------------- */
2077
46b780d4 2078static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
72246da4
FB		 2079{
2080 struct dwc3_ep *dep;
47d3946e 2081 u8 epnum;
72246da4 2082
f3bcfc7e
BD		 2083 INIT_LIST_HEAD(&dwc->gadget.ep_list);
2084
46b780d4 2085 for (epnum = 0; epnum < total; epnum++) {
47d3946e 2086 bool direction = epnum & 1;
46b780d4 2087 u8 num = epnum >> 1;
72246da4 2088
72246da4 2089 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
734d5a53 2090 if (!dep)
72246da4 2091 return -ENOMEM;
72246da4
FB		 2092
2093 dep->dwc = dwc;
2094 dep->number = epnum;
47d3946e 2095 dep->direction = direction;
2eb88016 2096 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
72246da4
FB		 2097 dwc->eps[epnum] = dep;
2098
46b780d4 2099 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
47d3946e 2100 direction ? "in" : "out");
6a1e3ef4 2101
72246da4 2102 dep->endpoint.name = dep->name;
39ebb05c
JY		 2103
2104 if (!(dep->number > 1)) {
2105 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2106 dep->endpoint.comp_desc = NULL;
2107 }
2108
74674cbf 2109 spin_lock_init(&dep->lock);
72246da4 2110
46b780d4 2111 if (num == 0) {
e117e742 2112 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
6048e4c6 2113 dep->endpoint.maxburst = 1;
72246da4 2114 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
46b780d4 2115 if (!direction)
72246da4 2116 dwc->gadget.ep0 = &dep->endpoint;
28781789
FB		 2117 } else if (direction) {
2118 int mdwidth;
46b780d4 2119 int kbytes;
28781789
FB		 2120 int size;
2121 int ret;
28781789
FB		 2122
2123 mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2124 /* MDWIDTH is represented in bits, we need it in bytes */
2125 mdwidth /= 8;
2126
46b780d4 2127 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num));
d548a617
TN		 2128 if (dwc3_is_usb31(dwc))
2129 size = DWC31_GTXFIFOSIZ_TXFDEF(size);
2130 else
2131 size = DWC3_GTXFIFOSIZ_TXFDEF(size);
28781789
FB		 2132
2133 /* FIFO Depth is in MDWDITH bytes. Multiply */
2134 size *= mdwidth;
2135
46b780d4
AS		 2136 kbytes = size / 1024;
2137 if (kbytes == 0)
2138 kbytes = 1;
28781789
FB		 2139
2140 /*
46b780d4 2141 * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for
28781789
FB		 2142 * internal overhead. We don't really know how these are used,
2143 * but documentation say it exists.
2144 */
46b780d4
AS		 2145 size -= mdwidth * (kbytes + 1);
2146 size /= kbytes;
28781789
FB		 2147
2148 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2149
2150 dep->endpoint.max_streams = 15;
2151 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2152 list_add_tail(&dep->endpoint.ep_list,
2153 &dwc->gadget.ep_list);
2154
2155 ret = dwc3_alloc_trb_pool(dep);
2156 if (ret)
2157 return ret;
72246da4
FB		 2158 } else {
2159 int ret;
2160
e117e742 2161 usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
12d36c16 2162 dep->endpoint.max_streams = 15;
72246da4
FB		 2163 dep->endpoint.ops = &dwc3_gadget_ep_ops;
2164 list_add_tail(&dep->endpoint.ep_list,
2165 &dwc->gadget.ep_list);
2166
2167 ret = dwc3_alloc_trb_pool(dep);
25b8ff68 2168 if (ret)
72246da4 2169 return ret;
72246da4 2170 }
25b8ff68 2171
46b780d4 2172 if (num == 0) {
a474d3b7
RB		 2173 dep->endpoint.caps.type_control = true;
2174 } else {
2175 dep->endpoint.caps.type_iso = true;
2176 dep->endpoint.caps.type_bulk = true;
2177 dep->endpoint.caps.type_int = true;
2178 }
2179
47d3946e 2180 dep->endpoint.caps.dir_in = direction;
a474d3b7
RB		 2181 dep->endpoint.caps.dir_out = !direction;
2182
aa3342c8
FB		 2183 INIT_LIST_HEAD(&dep->pending_list);
2184 INIT_LIST_HEAD(&dep->started_list);
72246da4
FB		 2185 }
2186
2187 return 0;
2188}
2189
2190static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2191{
2192 struct dwc3_ep *dep;
2193 u8 epnum;
2194
2195 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2196 dep = dwc->eps[epnum];
6a1e3ef4
FB		 2197 if (!dep)
2198 continue;
5bf8fae3
GC		 2199 /*
2200 * Physical endpoints 0 and 1 are special; they form the
2201 * bi-directional USB endpoint 0.
2202 *
2203 * For those two physical endpoints, we don't allocate a TRB
2204 * pool nor do we add them the endpoints list. Due to that, we
2205 * shouldn't do these two operations otherwise we would end up
2206 * with all sorts of bugs when removing dwc3.ko.
2207 */
2208 if (epnum != 0 && epnum != 1) {
2209 dwc3_free_trb_pool(dep);
72246da4 2210 list_del(&dep->endpoint.ep_list);
5bf8fae3 2211 }
72246da4
FB		 2212
2213 kfree(dep);
2214 }
2215}
2216
72246da4 2217/* -------------------------------------------------------------------------- */
e5caff68 2218
8f608e8a
FB		 2219static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2220 struct dwc3_request *req, struct dwc3_trb *trb,
2221 const struct dwc3_event_depevt *event, int status, int chain)
72246da4 2222{
72246da4
FB		 2223 unsigned int count;
2224 unsigned int s_pkt = 0;
2225
dc55c67e 2226 dwc3_ep_inc_deq(dep);
a9c3ca5f 2227
2c4cbe6e
FB		 2228 trace_dwc3_complete_trb(dep, trb);
2229
e5b36ae2
FB		 2230 /*
2231 * If we're in the middle of series of chained TRBs and we
2232 * receive a short transfer along the way, DWC3 will skip
2233 * through all TRBs including the last TRB in the chain (the
2234 * where CHN bit is zero. DWC3 will also avoid clearing HWO
2235 * bit and SW has to do it manually.
2236 *
2237 * We're going to do that here to avoid problems of HW trying
2238 * to use bogus TRBs for transfers.
2239 */
2240 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2241 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2242
c6267a51
FB		 2243 /*
2244 * If we're dealing with unaligned size OUT transfer, we will be left
2245 * with one TRB pending in the ring. We need to manually clear HWO bit
2246 * from that TRB.
2247 */
d6e5a549 2248 if ((req->zero || req->unaligned) && (trb->ctrl & DWC3_TRB_CTRL_HWO)) {
c6267a51
FB		 2249 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2250 return 1;
2251 }
2252
e5ba5ec8 2253 count = trb->size & DWC3_TRB_SIZE_MASK;
e62c5bc5 2254 req->remaining += count;
e5ba5ec8 2255
35b2719e
FB		 2256 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2257 return 1;
2258
4d588a14 2259 if (!dep->direction) {
e5ba5ec8
PA		 2260 if (count && (event->status & DEPEVT_STATUS_SHORT))
2261 s_pkt = 1;
2262 }
2263
7c705dfe 2264 if (s_pkt && !chain)
e5ba5ec8 2265 return 1;
f99f53f2 2266
e5ba5ec8
PA		 2267 if ((event->status & DEPEVT_STATUS_IOC) &&
2268 (trb->ctrl & DWC3_TRB_CTRL_IOC))
2269 return 1;
f99f53f2 2270
e5ba5ec8
PA		 2271 return 0;
2272}
2273
d3692953
FB		 2274static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2275 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2276 int status)
2277{
2278 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2279 struct scatterlist *sg = req->sg;
2280 struct scatterlist *s;
2281 unsigned int pending = req->num_pending_sgs;
2282 unsigned int i;
2283 int ret = 0;
2284
2285 for_each_sg(sg, s, pending, i) {
2286 trb = &dep->trb_pool[dep->trb_dequeue];
2287
2288 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2289 break;
2290
2291 req->sg = sg_next(s);
2292 req->num_pending_sgs--;
2293
2294 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2295 trb, event, status, true);
2296 if (ret)
2297 break;
2298 }
2299
2300 return ret;
2301}
2302
2303static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2304 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2305 int status)
2306{
2307 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2308
2309 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2310 event, status, false);
2311}
2312
12a3a4ad 2313static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
8f608e8a 2314 const struct dwc3_event_depevt *event, int status)
e5ba5ec8 2315{
31162af4 2316 struct dwc3_request *req, *n;
e62c5bc5 2317 int ret = 0;
e5ba5ec8 2318
31162af4 2319 list_for_each_entry_safe(req, n, &dep->started_list, list) {
1f512119 2320 unsigned length;
e5b36ae2 2321
1f512119 2322 length = req->request.length;
cf35fc33 2323 if (req->num_pending_sgs)
d3692953
FB		 2324 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2325 status);
2326 else
2327 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2328 status);
d115d705 2329
d6e5a549 2330 if (req->unaligned || req->zero) {
8b3b7b66
FB		 2331 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2332 status);
c6267a51 2333 req->unaligned = false;
d6e5a549 2334 req->zero = false;
c6267a51
FB		 2335 }
2336
e62c5bc5 2337 req->request.actual = length - req->remaining;
1f512119 2338
c96e6725
AKV		 2339 if (req->request.actual < length || req->num_pending_sgs) {
2340 /*
2341 * There could be a scenario where the whole req can't
2342 * be mapped into available TRB's. In that case, we need
2343 * to kick transfer again if (req->num_pending_sgs > 0)
2344 */
2345 if (req->num_pending_sgs) {
8f608e8a 2346 dev_WARN_ONCE(dep->dwc->dev,
c96e6725
AKV		 2347 (req->request.actual == length),
2348 "There are some pending sg's that needs to be queued again\n");
12a3a4ad
FB		 2349 __dwc3_gadget_kick_transfer(dep);
2350 return;
c96e6725
AKV		 2351 }
2352 }
1f512119 2353
d115d705 2354 dwc3_gadget_giveback(dep, req, status);
e5ba5ec8 2355
58f0218a 2356 if (ret)
72246da4 2357 break;
31162af4 2358 }
72246da4
FB		 2359}
2360
ee3638b8
FB		 2361static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2362 const struct dwc3_event_depevt *event)
2363{
2364 u32 cur_uf, mask;
2365
2366 mask = ~(dep->interval - 1);
2367 cur_uf = event->parameters & mask;
2368 dep->frame_number = cur_uf;
2369}
2370
8f608e8a
FB		 2371static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2372 const struct dwc3_event_depevt *event)
72246da4 2373{
8f608e8a 2374 struct dwc3 *dwc = dep->dwc;
72246da4 2375 unsigned status = 0;
6d8a0196 2376 bool stop = false;
72246da4 2377
ee3638b8
FB		 2378 dwc3_gadget_endpoint_frame_from_event(dep, event);
2379
72246da4
FB		 2380 if (event->status & DEPEVT_STATUS_BUSERR)
2381 status = -ECONNRESET;
2382
6d8a0196
FB		 2383 if (event->status & DEPEVT_STATUS_MISSED_ISOC) {
2384 status = -EXDEV;
2385 stop = true;
2386 }
2387
5f2e7975 2388 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
fae2b904 2389
6d8a0196
FB		 2390 if (stop) {
2391 dwc3_stop_active_transfer(dep, true);
2392 dep->flags = DWC3_EP_ENABLED;
2393 }
2394
fae2b904
FB		 2395 /*
2396 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2397 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2398 */
2399 if (dwc->revision < DWC3_REVISION_183A) {
2400 u32 reg;
2401 int i;
2402
2403 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
348e026f 2404 dep = dwc->eps[i];
fae2b904
FB		 2405
2406 if (!(dep->flags & DWC3_EP_ENABLED))
2407 continue;
2408
aa3342c8 2409 if (!list_empty(&dep->started_list))
fae2b904
FB		 2410 return;
2411 }
2412
2413 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2414 reg |= dwc->u1u2;
2415 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2416
2417 dwc->u1u2 = 0;
2418 }
72246da4
FB		 2419}
2420
8f608e8a
FB		 2421static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
2422 const struct dwc3_event_depevt *event)
32033865 2423{
ee3638b8 2424 dwc3_gadget_endpoint_frame_from_event(dep, event);
5828cada 2425 __dwc3_gadget_start_isoc(dep);
32033865
FB		 2426}
2427
72246da4
FB		 2428static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2429 const struct dwc3_event_depevt *event)
2430{
2431 struct dwc3_ep *dep;
2432 u8 epnum = event->endpoint_number;
76a638f8 2433 u8 cmd;
72246da4
FB		 2434
2435 dep = dwc->eps[epnum];
2436
d7fd41c6
JD		 2437 if (!(dep->flags & DWC3_EP_ENABLED)) {
2438 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2439 return;
2440
2441 /* Handle only EPCMDCMPLT when EP disabled */
2442 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2443 return;
2444 }
3336abb5 2445
72246da4
FB		 2446 if (epnum == 0 || epnum == 1) {
2447 dwc3_ep0_interrupt(dwc, event);
2448 return;
2449 }
2450
2451 switch (event->endpoint_event) {
72246da4 2452 case DWC3_DEPEVT_XFERINPROGRESS:
8f608e8a 2453 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
72246da4
FB		 2454 break;
2455 case DWC3_DEPEVT_XFERNOTREADY:
8f608e8a 2456 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
879631aa 2457 break;
72246da4 2458 case DWC3_DEPEVT_EPCMDCMPLT:
76a638f8
BW		 2459 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2460
2461 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2462 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2463 wake_up(&dep->wait_end_transfer);
2464 }
2465 break;
a24a6ab1 2466 case DWC3_DEPEVT_STREAMEVT:
742a4fff 2467 case DWC3_DEPEVT_XFERCOMPLETE:
76a638f8 2468 case DWC3_DEPEVT_RXTXFIFOEVT:
72246da4
FB		 2469 break;
2470 }
2471}
2472
2473static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2474{
2475 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2476 spin_unlock(&dwc->lock);
2477 dwc->gadget_driver->disconnect(&dwc->gadget);
2478 spin_lock(&dwc->lock);
2479 }
2480}
2481
bc5ba2e0
FB		 2482static void dwc3_suspend_gadget(struct dwc3 *dwc)
2483{
73a30bfc 2484 if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
bc5ba2e0
FB		 2485 spin_unlock(&dwc->lock);
2486 dwc->gadget_driver->suspend(&dwc->gadget);
2487 spin_lock(&dwc->lock);
2488 }
2489}
2490
2491static void dwc3_resume_gadget(struct dwc3 *dwc)
2492{
73a30bfc 2493 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
bc5ba2e0
FB		 2494 spin_unlock(&dwc->lock);
2495 dwc->gadget_driver->resume(&dwc->gadget);
5c7b3b02 2496 spin_lock(&dwc->lock);
8e74475b
FB		 2497 }
2498}
2499
2500static void dwc3_reset_gadget(struct dwc3 *dwc)
2501{
2502 if (!dwc->gadget_driver)
2503 return;
2504
2505 if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2506 spin_unlock(&dwc->lock);
2507 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
bc5ba2e0
FB		 2508 spin_lock(&dwc->lock);
2509 }
2510}
2511
8f608e8a 2512static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force)
72246da4 2513{
8f608e8a 2514 struct dwc3 *dwc = dep->dwc;
72246da4
FB		 2515 struct dwc3_gadget_ep_cmd_params params;
2516 u32 cmd;
2517 int ret;
2518
76a638f8
BW		 2519 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
2520 !dep->resource_index)
3daf74d7
PA		 2521 return;
2522
57911504
PA		 2523 /*
2524 * NOTICE: We are violating what the Databook says about the
2525 * EndTransfer command. Ideally we would _always_ wait for the
2526 * EndTransfer Command Completion IRQ, but that's causing too
2527 * much trouble synchronizing between us and gadget driver.
2528 *
2529 * We have discussed this with the IP Provider and it was
2530 * suggested to giveback all requests here, but give HW some
2531 * extra time to synchronize with the interconnect. We're using
dc93b41a 2532 * an arbitrary 100us delay for that.
57911504
PA		 2533 *
2534 * Note also that a similar handling was tested by Synopsys
2535 * (thanks a lot Paul) and nothing bad has come out of it.
2536 * In short, what we're doing is:
2537 *
2538 * - Issue EndTransfer WITH CMDIOC bit set
2539 * - Wait 100us
06281d46
JY		 2540 *
2541 * As of IP version 3.10a of the DWC_usb3 IP, the controller
2542 * supports a mode to work around the above limitation. The
2543 * software can poll the CMDACT bit in the DEPCMD register
2544 * after issuing a EndTransfer command. This mode is enabled
2545 * by writing GUCTL2[14]. This polling is already done in the
2546 * dwc3_send_gadget_ep_cmd() function so if the mode is
2547 * enabled, the EndTransfer command will have completed upon
2548 * returning from this function and we don't need to delay for
2549 * 100us.
2550 *
2551 * This mode is NOT available on the DWC_usb31 IP.
57911504
PA		 2552 */
2553
3daf74d7 2554 cmd = DWC3_DEPCMD_ENDTRANSFER;
b992e681
PZ		 2555 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2556 cmd |= DWC3_DEPCMD_CMDIOC;
b4996a86 2557 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
3daf74d7 2558 memset(&params, 0, sizeof(params));
2cd4718d 2559 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
3daf74d7 2560 WARN_ON_ONCE(ret);
b4996a86 2561 dep->resource_index = 0;
06281d46 2562
76a638f8
BW		 2563 if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
2564 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
06281d46 2565 udelay(100);
76a638f8 2566 }
72246da4
FB		 2567}
2568
72246da4
FB		 2569static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2570{
2571 u32 epnum;
2572
2573 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2574 struct dwc3_ep *dep;
72246da4
FB		 2575 int ret;
2576
2577 dep = dwc->eps[epnum];
6a1e3ef4
FB		 2578 if (!dep)
2579 continue;
72246da4
FB		 2580
2581 if (!(dep->flags & DWC3_EP_STALL))
2582 continue;
2583
2584 dep->flags &= ~DWC3_EP_STALL;
2585
50c763f8 2586 ret = dwc3_send_clear_stall_ep_cmd(dep);
72246da4
FB		 2587 WARN_ON_ONCE(ret);
2588 }
2589}
2590
2591static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2592{
c4430a26
FB		 2593 int reg;
2594
72246da4
FB		 2595 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2596 reg &= ~DWC3_DCTL_INITU1ENA;
2597 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2598
2599 reg &= ~DWC3_DCTL_INITU2ENA;
2600 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
72246da4 2601
72246da4
FB		 2602 dwc3_disconnect_gadget(dwc);
2603
2604 dwc->gadget.speed = USB_SPEED_UNKNOWN;
df62df56 2605 dwc->setup_packet_pending = false;
06a374ed 2606 usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
fc8bb91b
FB		 2607
2608 dwc->connected = false;
72246da4
FB		 2609}
2610
72246da4
FB		 2611static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2612{
2613 u32 reg;
2614
fc8bb91b
FB		 2615 dwc->connected = true;
2616
df62df56
FB		 2617 /*
2618 * WORKAROUND: DWC3 revisions <1.88a have an issue which
2619 * would cause a missing Disconnect Event if there's a
2620 * pending Setup Packet in the FIFO.
2621 *
2622 * There's no suggested workaround on the official Bug
2623 * report, which states that "unless the driver/application
2624 * is doing any special handling of a disconnect event,
2625 * there is no functional issue".
2626 *
2627 * Unfortunately, it turns out that we _do_ some special
2628 * handling of a disconnect event, namely complete all
2629 * pending transfers, notify gadget driver of the
2630 * disconnection, and so on.
2631 *
2632 * Our suggested workaround is to follow the Disconnect
2633 * Event steps here, instead, based on a setup_packet_pending
b5d335e5
FB		 2634 * flag. Such flag gets set whenever we have a SETUP_PENDING
2635 * status for EP0 TRBs and gets cleared on XferComplete for the
df62df56
FB		 2636 * same endpoint.
2637 *
2638 * Refers to:
2639 *
2640 * STAR#9000466709: RTL: Device : Disconnect event not
2641 * generated if setup packet pending in FIFO
2642 */
2643 if (dwc->revision < DWC3_REVISION_188A) {
2644 if (dwc->setup_packet_pending)
2645 dwc3_gadget_disconnect_interrupt(dwc);
2646 }
2647
8e74475b 2648 dwc3_reset_gadget(dwc);
72246da4
FB		 2649
2650 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2651 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2652 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3b637367 2653 dwc->test_mode = false;
72246da4
FB		 2654 dwc3_clear_stall_all_ep(dwc);
2655
2656 /* Reset device address to zero */
2657 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2658 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2659 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
72246da4
FB		 2660}
2661
72246da4
FB		 2662static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2663{
72246da4
FB		 2664 struct dwc3_ep *dep;
2665 int ret;
2666 u32 reg;
2667 u8 speed;
2668
72246da4
FB		 2669 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2670 speed = reg & DWC3_DSTS_CONNECTSPD;
2671 dwc->speed = speed;
2672
5fb6fdaf
JY		 2673 /*
2674 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2675 * each time on Connect Done.
2676 *
2677 * Currently we always use the reset value. If any platform
2678 * wants to set this to a different value, we need to add a
2679 * setting and update GCTL.RAMCLKSEL here.
2680 */
72246da4
FB		 2681
2682 switch (speed) {
2da9ad76 2683 case DWC3_DSTS_SUPERSPEED_PLUS:
7580862b
JY		 2684 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2685 dwc->gadget.ep0->maxpacket = 512;
2686 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2687 break;
2da9ad76 2688 case DWC3_DSTS_SUPERSPEED:
05870c5b
FB		 2689 /*
2690 * WORKAROUND: DWC3 revisions <1.90a have an issue which
2691 * would cause a missing USB3 Reset event.
2692 *
2693 * In such situations, we should force a USB3 Reset
2694 * event by calling our dwc3_gadget_reset_interrupt()
2695 * routine.
2696 *
2697 * Refers to:
2698 *
2699 * STAR#9000483510: RTL: SS : USB3 reset event may
2700 * not be generated always when the link enters poll
2701 */
2702 if (dwc->revision < DWC3_REVISION_190A)
2703 dwc3_gadget_reset_interrupt(dwc);
2704
72246da4
FB		 2705 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2706 dwc->gadget.ep0->maxpacket = 512;
2707 dwc->gadget.speed = USB_SPEED_SUPER;
2708 break;
2da9ad76 2709 case DWC3_DSTS_HIGHSPEED:
72246da4
FB		 2710 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2711 dwc->gadget.ep0->maxpacket = 64;
2712 dwc->gadget.speed = USB_SPEED_HIGH;
2713 break;
9418ee15 2714 case DWC3_DSTS_FULLSPEED:
72246da4
FB		 2715 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2716 dwc->gadget.ep0->maxpacket = 64;
2717 dwc->gadget.speed = USB_SPEED_FULL;
2718 break;
2da9ad76 2719 case DWC3_DSTS_LOWSPEED:
72246da4
FB		 2720 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2721 dwc->gadget.ep0->maxpacket = 8;
2722 dwc->gadget.speed = USB_SPEED_LOW;
2723 break;
2724 }
2725
61800263
TN		 2726 dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2727
2b758350
PA		 2728 /* Enable USB2 LPM Capability */
2729
ee5cd41c 2730 if ((dwc->revision > DWC3_REVISION_194A) &&
2da9ad76
JY		 2731 (speed != DWC3_DSTS_SUPERSPEED) &&
2732 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2b758350
PA		 2733 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2734 reg |= DWC3_DCFG_LPM_CAP;
2735 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2736
2737 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2738 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2739
460d098c 2740 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2b758350 2741
80caf7d2
HR		 2742 /*
2743 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2744 * DCFG.LPMCap is set, core responses with an ACK and the
2745 * BESL value in the LPM token is less than or equal to LPM
2746 * NYET threshold.
2747 */
2748 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2749 && dwc->has_lpm_erratum,
9165dabb 2750 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
80caf7d2
HR		 2751
2752 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2753 reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2754
356363bf
FB		 2755 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2756 } else {
2757 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2758 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2b758350
PA		 2759 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2760 }
2761
72246da4 2762 dep = dwc->eps[0];
39ebb05c 2763 ret = __dwc3_gadget_ep_enable(dep, true, false);
72246da4
FB		 2764 if (ret) {
2765 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2766 return;
2767 }
2768
2769 dep = dwc->eps[1];
39ebb05c 2770 ret = __dwc3_gadget_ep_enable(dep, true, false);
72246da4
FB		 2771 if (ret) {
2772 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2773 return;
2774 }
2775
2776 /*
2777 * Configure PHY via GUSB3PIPECTLn if required.
2778 *
2779 * Update GTXFIFOSIZn
2780 *
2781 * In both cases reset values should be sufficient.
2782 */
2783}
2784
2785static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2786{
72246da4
FB		 2787 /*
2788 * TODO take core out of low power mode when that's
2789 * implemented.
2790 */
2791
ad14d4e0
JL		 2792 if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2793 spin_unlock(&dwc->lock);
2794 dwc->gadget_driver->resume(&dwc->gadget);
2795 spin_lock(&dwc->lock);
2796 }
72246da4
FB		 2797}
2798
2799static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2800 unsigned int evtinfo)
2801{
fae2b904 2802 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
0b0cc1cd
FB		 2803 unsigned int pwropt;
2804
2805 /*
2806 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2807 * Hibernation mode enabled which would show up when device detects
2808 * host-initiated U3 exit.
2809 *
2810 * In that case, device will generate a Link State Change Interrupt
2811 * from U3 to RESUME which is only necessary if Hibernation is
2812 * configured in.
2813 *
2814 * There are no functional changes due to such spurious event and we
2815 * just need to ignore it.
2816 *
2817 * Refers to:
2818 *
2819 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2820 * operational mode
2821 */
2822 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2823 if ((dwc->revision < DWC3_REVISION_250A) &&
2824 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2825 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2826 (next == DWC3_LINK_STATE_RESUME)) {
0b0cc1cd
FB		 2827 return;
2828 }
2829 }
fae2b904
FB		 2830
2831 /*
2832 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2833 * on the link partner, the USB session might do multiple entry/exit
2834 * of low power states before a transfer takes place.
2835 *
2836 * Due to this problem, we might experience lower throughput. The
2837 * suggested workaround is to disable DCTL[12:9] bits if we're
2838 * transitioning from U1/U2 to U0 and enable those bits again
2839 * after a transfer completes and there are no pending transfers
2840 * on any of the enabled endpoints.
2841 *
2842 * This is the first half of that workaround.
2843 *
2844 * Refers to:
2845 *
2846 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2847 * core send LGO_Ux entering U0
2848 */
2849 if (dwc->revision < DWC3_REVISION_183A) {
2850 if (next == DWC3_LINK_STATE_U0) {
2851 u32 u1u2;
2852 u32 reg;
2853
2854 switch (dwc->link_state) {
2855 case DWC3_LINK_STATE_U1:
2856 case DWC3_LINK_STATE_U2:
2857 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2858 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2859 | DWC3_DCTL_ACCEPTU2ENA
2860 | DWC3_DCTL_INITU1ENA
2861 | DWC3_DCTL_ACCEPTU1ENA);
2862
2863 if (!dwc->u1u2)
2864 dwc->u1u2 = reg & u1u2;
2865
2866 reg &= ~u1u2;
2867
2868 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2869 break;
2870 default:
2871 /* do nothing */
2872 break;
2873 }
2874 }
2875 }
2876
bc5ba2e0
FB		 2877 switch (next) {
2878 case DWC3_LINK_STATE_U1:
2879 if (dwc->speed == USB_SPEED_SUPER)
2880 dwc3_suspend_gadget(dwc);
2881 break;
2882 case DWC3_LINK_STATE_U2:
2883 case DWC3_LINK_STATE_U3:
2884 dwc3_suspend_gadget(dwc);
2885 break;
2886 case DWC3_LINK_STATE_RESUME:
2887 dwc3_resume_gadget(dwc);
2888 break;
2889 default:
2890 /* do nothing */
2891 break;
2892 }
2893
e57ebc1d 2894 dwc->link_state = next;
72246da4
FB		 2895}
2896
72704f87
BW		 2897static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
2898 unsigned int evtinfo)
2899{
2900 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
2901
2902 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
2903 dwc3_suspend_gadget(dwc);
2904
2905 dwc->link_state = next;
2906}
2907
e1dadd3b
FB		 2908static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2909 unsigned int evtinfo)
2910{
2911 unsigned int is_ss = evtinfo & BIT(4);
2912
bfad65ee 2913 /*
e1dadd3b
FB		 2914 * WORKAROUND: DWC3 revison 2.20a with hibernation support
2915 * have a known issue which can cause USB CV TD.9.23 to fail
2916 * randomly.
2917 *
2918 * Because of this issue, core could generate bogus hibernation
2919 * events which SW needs to ignore.
2920 *
2921 * Refers to:
2922 *
2923 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2924 * Device Fallback from SuperSpeed
2925 */
2926 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2927 return;
2928
2929 /* enter hibernation here */
2930}
2931
72246da4
FB		 2932static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2933 const struct dwc3_event_devt *event)
2934{
2935 switch (event->type) {
2936 case DWC3_DEVICE_EVENT_DISCONNECT:
2937 dwc3_gadget_disconnect_interrupt(dwc);
2938 break;
2939 case DWC3_DEVICE_EVENT_RESET:
2940 dwc3_gadget_reset_interrupt(dwc);
2941 break;
2942 case DWC3_DEVICE_EVENT_CONNECT_DONE:
2943 dwc3_gadget_conndone_interrupt(dwc);
2944 break;
2945 case DWC3_DEVICE_EVENT_WAKEUP:
2946 dwc3_gadget_wakeup_interrupt(dwc);
2947 break;
e1dadd3b
FB		 2948 case DWC3_DEVICE_EVENT_HIBER_REQ:
2949 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
2950 "unexpected hibernation event\n"))
2951 break;
2952
2953 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2954 break;
72246da4
FB		 2955 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2956 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2957 break;
2958 case DWC3_DEVICE_EVENT_EOPF:
72704f87 2959 /* It changed to be suspend event for version 2.30a and above */
5eb30ced 2960 if (dwc->revision >= DWC3_REVISION_230A) {
72704f87
BW		 2961 /*
2962 * Ignore suspend event until the gadget enters into
2963 * USB_STATE_CONFIGURED state.
2964 */
2965 if (dwc->gadget.state >= USB_STATE_CONFIGURED)
2966 dwc3_gadget_suspend_interrupt(dwc,
2967 event->event_info);
2968 }
72246da4
FB		 2969 break;
2970 case DWC3_DEVICE_EVENT_SOF:
72246da4 2971 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
72246da4 2972 case DWC3_DEVICE_EVENT_CMD_CMPL:
72246da4 2973 case DWC3_DEVICE_EVENT_OVERFLOW:
72246da4
FB		 2974 break;
2975 default:
e9f2aa87 2976 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
72246da4
FB		 2977 }
2978}
2979
2980static void dwc3_process_event_entry(struct dwc3 *dwc,
2981 const union dwc3_event *event)
2982{
43c96be1 2983 trace_dwc3_event(event->raw, dwc);
2c4cbe6e 2984
dfc5e805
FB		 2985 if (!event->type.is_devspec)
2986 dwc3_endpoint_interrupt(dwc, &event->depevt);
2987 else if (event->type.type == DWC3_EVENT_TYPE_DEV)
72246da4 2988 dwc3_gadget_interrupt(dwc, &event->devt);
dfc5e805 2989 else
72246da4 2990 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
72246da4
FB		 2991}
2992
dea520a4 2993static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
b15a762f 2994{
dea520a4 2995 struct dwc3 *dwc = evt->dwc;
b15a762f 2996 irqreturn_t ret = IRQ_NONE;
f42f2447 2997 int left;
e8adfc30 2998 u32 reg;
b15a762f 2999
f42f2447 3000 left = evt->count;
b15a762f 3001
f42f2447
FB		 3002 if (!(evt->flags & DWC3_EVENT_PENDING))
3003 return IRQ_NONE;
b15a762f 3004
f42f2447
FB		 3005 while (left > 0) {
3006 union dwc3_event event;
b15a762f 3007
ebbb2d59 3008 event.raw = *(u32 *) (evt->cache + evt->lpos);
b15a762f 3009
f42f2447 3010 dwc3_process_event_entry(dwc, &event);
b15a762f 3011
f42f2447
FB		 3012 /*
3013 * FIXME we wrap around correctly to the next entry as
3014 * almost all entries are 4 bytes in size. There is one
3015 * entry which has 12 bytes which is a regular entry
3016 * followed by 8 bytes data. ATM I don't know how
3017 * things are organized if we get next to the a
3018 * boundary so I worry about that once we try to handle
3019 * that.
3020 */
caefe6c7 3021 evt->lpos = (evt->lpos + 4) % evt->length;
f42f2447 3022 left -= 4;
f42f2447 3023 }
b15a762f 3024
f42f2447
FB		 3025 evt->count = 0;
3026 evt->flags &= ~DWC3_EVENT_PENDING;
3027 ret = IRQ_HANDLED;
b15a762f 3028
f42f2447 3029 /* Unmask interrupt */
660e9bde 3030 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
f42f2447 3031 reg &= ~DWC3_GEVNTSIZ_INTMASK;
660e9bde 3032 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
b15a762f 3033
cf40b86b
JY		 3034 if (dwc->imod_interval) {
3035 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3036 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3037 }
3038
f42f2447
FB		 3039 return ret;
3040}
e8adfc30 3041
dea520a4 3042static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
f42f2447 3043{
dea520a4
FB		 3044 struct dwc3_event_buffer *evt = _evt;
3045 struct dwc3 *dwc = evt->dwc;
e5f68b4a 3046 unsigned long flags;
f42f2447 3047 irqreturn_t ret = IRQ_NONE;
f42f2447 3048
e5f68b4a 3049 spin_lock_irqsave(&dwc->lock, flags);
dea520a4 3050 ret = dwc3_process_event_buf(evt);
e5f68b4a 3051 spin_unlock_irqrestore(&dwc->lock, flags);
b15a762f
FB		 3052
3053 return ret;
3054}
3055
dea520a4 3056static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
72246da4 3057{
dea520a4 3058 struct dwc3 *dwc = evt->dwc;
ebbb2d59 3059 u32 amount;
72246da4 3060 u32 count;
e8adfc30 3061 u32 reg;
72246da4 3062
fc8bb91b
FB		 3063 if (pm_runtime_suspended(dwc->dev)) {
3064 pm_runtime_get(dwc->dev);
3065 disable_irq_nosync(dwc->irq_gadget);
3066 dwc->pending_events = true;
3067 return IRQ_HANDLED;
3068 }
3069
d325a1de
TN		 3070 /*
3071 * With PCIe legacy interrupt, test shows that top-half irq handler can
3072 * be called again after HW interrupt deassertion. Check if bottom-half
3073 * irq event handler completes before caching new event to prevent
3074 * losing events.
3075 */
3076 if (evt->flags & DWC3_EVENT_PENDING)
3077 return IRQ_HANDLED;
3078
660e9bde 3079 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
72246da4
FB		 3080 count &= DWC3_GEVNTCOUNT_MASK;
3081 if (!count)
3082 return IRQ_NONE;
3083
b15a762f
FB		 3084 evt->count = count;
3085 evt->flags |= DWC3_EVENT_PENDING;
72246da4 3086
e8adfc30 3087 /* Mask interrupt */
660e9bde 3088 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
e8adfc30 3089 reg |= DWC3_GEVNTSIZ_INTMASK;
660e9bde 3090 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
e8adfc30 3091
ebbb2d59
JY		 3092 amount = min(count, evt->length - evt->lpos);
3093 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3094
3095 if (amount < count)
3096 memcpy(evt->cache, evt->buf, count - amount);
3097
65aca320
JY		 3098 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3099
b15a762f 3100 return IRQ_WAKE_THREAD;
72246da4
FB		 3101}
3102
dea520a4 3103static irqreturn_t dwc3_interrupt(int irq, void *_evt)
72246da4 3104{
dea520a4 3105 struct dwc3_event_buffer *evt = _evt;
72246da4 3106
dea520a4 3107 return dwc3_check_event_buf(evt);
72246da4
FB		 3108}
3109
6db3812e
FB		 3110static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3111{
3112 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3113 int irq;
3114
3115 irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
3116 if (irq > 0)
3117 goto out;
3118
3119 if (irq == -EPROBE_DEFER)
3120 goto out;
3121
3122 irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
3123 if (irq > 0)
3124 goto out;
3125
3126 if (irq == -EPROBE_DEFER)
3127 goto out;
3128
3129 irq = platform_get_irq(dwc3_pdev, 0);
3130 if (irq > 0)
3131 goto out;
3132
3133 if (irq != -EPROBE_DEFER)
3134 dev_err(dwc->dev, "missing peripheral IRQ\n");
3135
3136 if (!irq)
3137 irq = -EINVAL;
3138
3139out:
3140 return irq;
3141}
3142
72246da4 3143/**
bfad65ee 3144 * dwc3_gadget_init - initializes gadget related registers
1d046793 3145 * @dwc: pointer to our controller context structure
72246da4
FB		 3146 *
3147 * Returns 0 on success otherwise negative errno.
3148 */
41ac7b3a 3149int dwc3_gadget_init(struct dwc3 *dwc)
72246da4 3150{
6db3812e
FB		 3151 int ret;
3152 int irq;
9522def4 3153
6db3812e
FB		 3154 irq = dwc3_gadget_get_irq(dwc);
3155 if (irq < 0) {
3156 ret = irq;
3157 goto err0;
9522def4
RQ		 3158 }
3159
3160 dwc->irq_gadget = irq;
72246da4 3161
d64ff406
AB		 3162 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3163 sizeof(*dwc->ep0_trb) * 2,
3164 &dwc->ep0_trb_addr, GFP_KERNEL);
72246da4
FB		 3165 if (!dwc->ep0_trb) {
3166 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3167 ret = -ENOMEM;
7d5e650a 3168 goto err0;
72246da4
FB		 3169 }
3170
4199c5f8 3171 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
72246da4 3172 if (!dwc->setup_buf) {
72246da4 3173 ret = -ENOMEM;
7d5e650a 3174 goto err1;
72246da4
FB		 3175 }
3176
905dc04e
FB		 3177 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3178 &dwc->bounce_addr, GFP_KERNEL);
3179 if (!dwc->bounce) {
3180 ret = -ENOMEM;
d6e5a549 3181 goto err2;
905dc04e
FB		 3182 }
3183
bb014736
BW		 3184 init_completion(&dwc->ep0_in_setup);
3185
72246da4 3186 dwc->gadget.ops = &dwc3_gadget_ops;
72246da4 3187 dwc->gadget.speed = USB_SPEED_UNKNOWN;
eeb720fb 3188 dwc->gadget.sg_supported = true;
72246da4 3189 dwc->gadget.name = "dwc3-gadget";
6a4290cc 3190 dwc->gadget.is_otg = dwc->dr_mode == USB_DR_MODE_OTG;
72246da4 3191
b9e51b2b
BM		 3192 /*
3193 * FIXME We might be setting max_speed to <SUPER, however versions
3194 * <2.20a of dwc3 have an issue with metastability (documented
3195 * elsewhere in this driver) which tells us we can't set max speed to
3196 * anything lower than SUPER.
3197 *
3198 * Because gadget.max_speed is only used by composite.c and function
3199 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3200 * to happen so we avoid sending SuperSpeed Capability descriptor
3201 * together with our BOS descriptor as that could confuse host into
3202 * thinking we can handle super speed.
3203 *
3204 * Note that, in fact, we won't even support GetBOS requests when speed
3205 * is less than super speed because we don't have means, yet, to tell
3206 * composite.c that we are USB 2.0 + LPM ECN.
3207 */
42bf02ec
RQ		 3208 if (dwc->revision < DWC3_REVISION_220A &&
3209 !dwc->dis_metastability_quirk)
5eb30ced 3210 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
b9e51b2b
BM		 3211 dwc->revision);
3212
3213 dwc->gadget.max_speed = dwc->maximum_speed;
3214
72246da4
FB		 3215 /*
3216 * REVISIT: Here we should clear all pending IRQs to be
3217 * sure we're starting from a well known location.
3218 */
3219
f3bcfc7e 3220 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
72246da4 3221 if (ret)
d6e5a549 3222 goto err3;
72246da4 3223
72246da4
FB		 3224 ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3225 if (ret) {
3226 dev_err(dwc->dev, "failed to register udc\n");
d6e5a549 3227 goto err4;
72246da4
FB		 3228 }
3229
3230 return 0;
3231
7d5e650a 3232err4:
d6e5a549 3233 dwc3_gadget_free_endpoints(dwc);
04c03d10 3234
7d5e650a 3235err3:
d6e5a549
FB		 3236 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3237 dwc->bounce_addr);
5812b1c2 3238
7d5e650a 3239err2:
0fc9a1be 3240 kfree(dwc->setup_buf);
72246da4 3241
7d5e650a 3242err1:
d64ff406 3243 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
72246da4
FB		 3244 dwc->ep0_trb, dwc->ep0_trb_addr);
3245
72246da4
FB		 3246err0:
3247 return ret;
3248}
3249
7415f17c
FB		 3250/* -------------------------------------------------------------------------- */
3251
72246da4
FB		 3252void dwc3_gadget_exit(struct dwc3 *dwc)
3253{
72246da4 3254 usb_del_gadget_udc(&dwc->gadget);
72246da4 3255 dwc3_gadget_free_endpoints(dwc);
905dc04e 3256 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
d6e5a549 3257 dwc->bounce_addr);
0fc9a1be 3258 kfree(dwc->setup_buf);
d64ff406 3259 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
d6e5a549 3260 dwc->ep0_trb, dwc->ep0_trb_addr);
72246da4 3261}
7415f17c 3262
0b0231aa 3263int dwc3_gadget_suspend(struct dwc3 *dwc)
7415f17c 3264{
9772b47a
RQ		 3265 if (!dwc->gadget_driver)
3266 return 0;
3267
1551e35e 3268 dwc3_gadget_run_stop(dwc, false, false);
9f8a67b6
FB		 3269 dwc3_disconnect_gadget(dwc);
3270 __dwc3_gadget_stop(dwc);
7415f17c
FB		 3271
3272 return 0;
3273}
3274
3275int dwc3_gadget_resume(struct dwc3 *dwc)
3276{
7415f17c
FB		 3277 int ret;
3278
9772b47a
RQ		 3279 if (!dwc->gadget_driver)
3280 return 0;
3281
9f8a67b6
FB		 3282 ret = __dwc3_gadget_start(dwc);
3283 if (ret < 0)
7415f17c
FB		 3284 goto err0;
3285
9f8a67b6
FB		 3286 ret = dwc3_gadget_run_stop(dwc, true, false);
3287 if (ret < 0)
7415f17c
FB		 3288 goto err1;
3289
7415f17c
FB		 3290 return 0;
3291
3292err1:
9f8a67b6 3293 __dwc3_gadget_stop(dwc);
7415f17c
FB		 3294
3295err0:
3296 return ret;
3297}
fc8bb91b
FB		 3298
3299void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3300{
3301 if (dwc->pending_events) {
3302 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3303 dwc->pending_events = false;
3304 enable_irq(dwc->irq_gadget);
3305 }
3306}
Ubuntu Hirsute Linux kernel mirror