]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/scsi/ufs/ufshcd.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: lpfc: Update lpfc version to 12.8.0.0
[mirror_ubuntu-jammy-kernel.git] / drivers / scsi / ufs / ufshcd.c
CommitLineData
7a3e97b0 1/*
e0eca63e 2 * Universal Flash Storage Host controller driver Core
7a3e97b0
SY		 3 *
4 * This code is based on drivers/scsi/ufs/ufshcd.c
3b1d0580 5 * Copyright (C) 2011-2013 Samsung India Software Operations
52ac95fe 6 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7a3e97b0 7 *
3b1d0580
VH		 8 * Authors:
9 * Santosh Yaraganavi <santosh.sy@samsung.com>
10 * Vinayak Holikatti <h.vinayak@samsung.com>
7a3e97b0
SY		 11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
3b1d0580
VH		 16 * See the COPYING file in the top-level directory or visit
17 * <http://www.gnu.org/licenses/gpl-2.0.html>
7a3e97b0
SY		 18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 *
3b1d0580
VH		 24 * This program is provided "AS IS" and "WITH ALL FAULTS" and
25 * without warranty of any kind. You are solely responsible for
26 * determining the appropriateness of using and distributing
27 * the program and assume all risks associated with your exercise
28 * of rights with respect to the program, including but not limited
29 * to infringement of third party rights, the risks and costs of
30 * program errors, damage to or loss of data, programs or equipment,
31 * and unavailability or interruption of operations. Under no
32 * circumstances will the contributor of this Program be liable for
33 * any damages of any kind arising from your use or distribution of
34 * this program.
5c0c28a8
SRT		 35 *
36 * The Linux Foundation chooses to take subject only to the GPLv2
37 * license terms, and distributes only under these terms.
7a3e97b0
SY		 38 */
39
6ccf44fe 40#include <linux/async.h>
856b3483 41#include <linux/devfreq.h>
b573d484 42#include <linux/nls.h>
54b879b7 43#include <linux/of.h>
ad448378 44#include <linux/bitfield.h>
fb276f77 45#include <linux/blk-pm.h>
e0eca63e 46#include "ufshcd.h"
c58ab7aa 47#include "ufs_quirks.h"
53b3d9c3 48#include "unipro.h"
cbb6813e 49#include "ufs-sysfs.h"
df032bf2 50#include "ufs_bsg.h"
7a3e97b0 51
7ff5ab47
SJ		 52#define CREATE_TRACE_POINTS
53#include <trace/events/ufs.h>
54
2fbd009b
SJ		 55#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
56 UTP_TASK_REQ_COMPL |\
57 UFSHCD_ERROR_MASK)
6ccf44fe
SJ		 58/* UIC command timeout, unit: ms */
59#define UIC_CMD_TIMEOUT 500
2fbd009b 60
5a0b0cb9
SRT		 61/* NOP OUT retries waiting for NOP IN response */
62#define NOP_OUT_RETRIES 10
63/* Timeout after 30 msecs if NOP OUT hangs without response */
64#define NOP_OUT_TIMEOUT 30 /* msecs */
65
68078d5c 66/* Query request retries */
10fe5888 67#define QUERY_REQ_RETRIES 3
68078d5c 68/* Query request timeout */
10fe5888 69#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
68078d5c 70
e2933132
SRT		 71/* Task management command timeout */
72#define TM_CMD_TIMEOUT 100 /* msecs */
73
64238fbd
YG		 74/* maximum number of retries for a general UIC command */
75#define UFS_UIC_COMMAND_RETRIES 3
76
1d337ec2
SRT		 77/* maximum number of link-startup retries */
78#define DME_LINKSTARTUP_RETRIES 3
79
87d0b4a6
YG		 80/* Maximum retries for Hibern8 enter */
81#define UIC_HIBERN8_ENTER_RETRIES 3
82
1d337ec2
SRT		 83/* maximum number of reset retries before giving up */
84#define MAX_HOST_RESET_RETRIES 5
85
68078d5c
DR		 86/* Expose the flag value from utp_upiu_query.value */
87#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
88
7d568652
SJ		 89/* Interrupt aggregation default timeout, unit: 40us */
90#define INT_AGGR_DEF_TO 0x02
91
49615ba1
SC		 92/* default delay of autosuspend: 2000 ms */
93#define RPM_AUTOSUSPEND_DELAY_MS 2000
94
09f17791
CG		 95/* Default value of wait time before gating device ref clock */
96#define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
97
aa497613
SRT		 98#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
99 ({ \
100 int _ret; \
101 if (_on) \
102 _ret = ufshcd_enable_vreg(_dev, _vreg); \
103 else \
104 _ret = ufshcd_disable_vreg(_dev, _vreg); \
105 _ret; \
106 })
107
ba80917d
TW		 108#define ufshcd_hex_dump(prefix_str, buf, len) do { \
109 size_t __len = (len); \
110 print_hex_dump(KERN_ERR, prefix_str, \
111 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
112 16, 4, buf, __len, false); \
113} while (0)
114
115int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
116 const char *prefix)
117{
d6724756
MG		 118 u32 *regs;
119 size_t pos;
120
121 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
122 return -EINVAL;
ba80917d 123
cddaebaf 124 regs = kzalloc(len, GFP_ATOMIC);
ba80917d
TW		 125 if (!regs)
126 return -ENOMEM;
127
d6724756
MG		 128 for (pos = 0; pos < len; pos += 4)
129 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
130
ba80917d
TW		 131 ufshcd_hex_dump(prefix, regs, len);
132 kfree(regs);
133
134 return 0;
135}
136EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
66cc820f 137
7a3e97b0
SY		 138enum {
139 UFSHCD_MAX_CHANNEL = 0,
140 UFSHCD_MAX_ID = 1,
7a3e97b0
SY		 141 UFSHCD_CMD_PER_LUN = 32,
142 UFSHCD_CAN_QUEUE = 32,
143};
144
145/* UFSHCD states */
146enum {
7a3e97b0
SY		 147 UFSHCD_STATE_RESET,
148 UFSHCD_STATE_ERROR,
3441da7d 149 UFSHCD_STATE_OPERATIONAL,
141f8165 150 UFSHCD_STATE_EH_SCHEDULED,
3441da7d
SRT		 151};
152
153/* UFSHCD error handling flags */
154enum {
155 UFSHCD_EH_IN_PROGRESS = (1 << 0),
7a3e97b0
SY		 156};
157
e8e7f271
SRT		 158/* UFSHCD UIC layer error flags */
159enum {
160 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
9a47ec7c
YG		 161 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
162 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
163 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
164 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
165 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
e8e7f271
SRT		 166};
167
3441da7d 168#define ufshcd_set_eh_in_progress(h) \
9c490d2d 169 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
3441da7d 170#define ufshcd_eh_in_progress(h) \
9c490d2d 171 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
3441da7d 172#define ufshcd_clear_eh_in_progress(h) \
9c490d2d 173 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
3441da7d 174
57d104c1
SJ		 175#define ufshcd_set_ufs_dev_active(h) \
176 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
177#define ufshcd_set_ufs_dev_sleep(h) \
178 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
179#define ufshcd_set_ufs_dev_poweroff(h) \
180 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
181#define ufshcd_is_ufs_dev_active(h) \
182 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
183#define ufshcd_is_ufs_dev_sleep(h) \
184 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
185#define ufshcd_is_ufs_dev_poweroff(h) \
186 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
187
cbb6813e 188struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
57d104c1
SJ		 189 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
190 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
191 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
192 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
193 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
194 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
195};
196
197static inline enum ufs_dev_pwr_mode
198ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
199{
200 return ufs_pm_lvl_states[lvl].dev_state;
201}
202
203static inline enum uic_link_state
204ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
205{
206 return ufs_pm_lvl_states[lvl].link_state;
207}
208
0c8f7586
SJ		 209static inline enum ufs_pm_level
210ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
211 enum uic_link_state link_state)
212{
213 enum ufs_pm_level lvl;
214
215 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
216 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
217 (ufs_pm_lvl_states[lvl].link_state == link_state))
218 return lvl;
219 }
220
221 /* if no match found, return the level 0 */
222 return UFS_PM_LVL_0;
223}
224
56d4a186
SJ		 225static struct ufs_dev_fix ufs_fixups[] = {
226 /* UFS cards deviations table */
227 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
228 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
56d4a186
SJ		 229 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
230 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
56d4a186
SJ		 231 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
232 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
233 UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
234 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
235 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
236 UFS_DEVICE_QUIRK_PA_TACTIVATE),
237 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
238 UFS_DEVICE_QUIRK_PA_TACTIVATE),
56d4a186
SJ		 239 UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
240 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
8e4829c6
WL		 241 UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
242 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
56d4a186
SJ		 243
244 END_FIX
245};
246
9333d775 247static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
3441da7d 248static void ufshcd_async_scan(void *data, async_cookie_t cookie);
e8e7f271 249static int ufshcd_reset_and_restore(struct ufs_hba *hba);
e7d38257 250static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
e8e7f271 251static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
1d337ec2 252static void ufshcd_hba_exit(struct ufs_hba *hba);
1b9e2141 253static int ufshcd_probe_hba(struct ufs_hba *hba, bool async);
1ab27c9c
ST		 254static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
255 bool skip_ref_clk);
256static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
1ab27c9c 257static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
cad2e03d 258static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
57d104c1 259static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
fcb0c4b0
ST		 260static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
261static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
401f1e44 262static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
fcb0c4b0 263static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
57d104c1 264static irqreturn_t ufshcd_intr(int irq, void *__hba);
874237f7
YG		 265static int ufshcd_change_power_mode(struct ufs_hba *hba,
266 struct ufs_pa_layer_attr *pwr_mode);
14497328
YG		 267static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
268{
269 return tag >= 0 && tag < hba->nutrs;
270}
57d104c1 271
5231d38c 272static inline void ufshcd_enable_irq(struct ufs_hba *hba)
57d104c1 273{
57d104c1 274 if (!hba->is_irq_enabled) {
5231d38c 275 enable_irq(hba->irq);
57d104c1
SJ		 276 hba->is_irq_enabled = true;
277 }
57d104c1
SJ		 278}
279
280static inline void ufshcd_disable_irq(struct ufs_hba *hba)
281{
282 if (hba->is_irq_enabled) {
5231d38c 283 disable_irq(hba->irq);
57d104c1
SJ		 284 hba->is_irq_enabled = false;
285 }
286}
3441da7d 287
38135535
SJ		 288static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
289{
290 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
291 scsi_unblock_requests(hba->host);
292}
293
294static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
295{
296 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
297 scsi_block_requests(hba->host);
298}
299
6667e6d9
OS		 300static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
301 const char *str)
302{
303 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
304
305 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
306}
307
308static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
309 const char *str)
310{
311 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
312
313 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
314}
315
316static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
317 const char *str)
318{
6667e6d9 319 int off = (int)tag - hba->nutrs;
391e388f 320 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
6667e6d9 321
391e388f
CH		 322 trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
323 &descp->input_param1);
6667e6d9
OS		 324}
325
1a07f2d9
LS		 326static void ufshcd_add_command_trace(struct ufs_hba *hba,
327 unsigned int tag, const char *str)
328{
329 sector_t lba = -1;
330 u8 opcode = 0;
331 u32 intr, doorbell;
e7c3b379 332 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
e4d2add7 333 struct scsi_cmnd *cmd = lrbp->cmd;
1a07f2d9
LS		 334 int transfer_len = -1;
335
e7c3b379
OS		 336 if (!trace_ufshcd_command_enabled()) {
337 /* trace UPIU W/O tracing command */
e4d2add7 338 if (cmd)
e7c3b379 339 ufshcd_add_cmd_upiu_trace(hba, tag, str);
1a07f2d9 340 return;
e7c3b379 341 }
1a07f2d9 342
e4d2add7 343 if (cmd) { /* data phase exists */
e7c3b379
OS		 344 /* trace UPIU also */
345 ufshcd_add_cmd_upiu_trace(hba, tag, str);
e4d2add7 346 opcode = cmd->cmnd[0];
1a07f2d9
LS		 347 if ((opcode == READ_10) || (opcode == WRITE_10)) {
348 /*
349 * Currently we only fully trace read(10) and write(10)
350 * commands
351 */
e4d2add7
BVA		 352 if (cmd->request && cmd->request->bio)
353 lba = cmd->request->bio->bi_iter.bi_sector;
1a07f2d9
LS		 354 transfer_len = be32_to_cpu(
355 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
356 }
357 }
358
359 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
360 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
361 trace_ufshcd_command(dev_name(hba->dev), str, tag,
362 doorbell, transfer_len, intr, lba, opcode);
363}
364
ff8e20c6
DR		 365static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
366{
367 struct ufs_clk_info *clki;
368 struct list_head *head = &hba->clk_list_head;
369
566ec9ad 370 if (list_empty(head))
ff8e20c6
DR		 371 return;
372
373 list_for_each_entry(clki, head, list) {
374 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
375 clki->max_freq)
376 dev_err(hba->dev, "clk: %s, rate: %u\n",
377 clki->name, clki->curr_freq);
378 }
379}
380
48d5b973
SC		 381static void ufshcd_print_err_hist(struct ufs_hba *hba,
382 struct ufs_err_reg_hist *err_hist,
383 char *err_name)
ff8e20c6
DR		 384{
385 int i;
27752647 386 bool found = false;
ff8e20c6 387
48d5b973
SC		 388 for (i = 0; i < UFS_ERR_REG_HIST_LENGTH; i++) {
389 int p = (i + err_hist->pos) % UFS_ERR_REG_HIST_LENGTH;
ff8e20c6 390
645728a6 391 if (err_hist->tstamp[p] == 0)
ff8e20c6 392 continue;
c5397f13 393 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
ff8e20c6 394 err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
27752647 395 found = true;
ff8e20c6 396 }
27752647
SC		 397
398 if (!found)
fd1fb4d5 399 dev_err(hba->dev, "No record of %s\n", err_name);
ff8e20c6
DR		 400}
401
66cc820f
DR		 402static void ufshcd_print_host_regs(struct ufs_hba *hba)
403{
ba80917d 404 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
66cc820f
DR		 405 dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n",
406 hba->ufs_version, hba->capabilities);
407 dev_err(hba->dev,
408 "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n",
409 (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks);
ff8e20c6
DR		 410 dev_err(hba->dev,
411 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n",
412 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
413 hba->ufs_stats.hibern8_exit_cnt);
414
48d5b973
SC		 415 ufshcd_print_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
416 ufshcd_print_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
417 ufshcd_print_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
418 ufshcd_print_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
419 ufshcd_print_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
d3c615bf
SC		 420 ufshcd_print_err_hist(hba, &hba->ufs_stats.auto_hibern8_err,
421 "auto_hibern8_err");
8808b4e9
SC		 422 ufshcd_print_err_hist(hba, &hba->ufs_stats.fatal_err, "fatal_err");
423 ufshcd_print_err_hist(hba, &hba->ufs_stats.link_startup_err,
424 "link_startup_fail");
425 ufshcd_print_err_hist(hba, &hba->ufs_stats.resume_err, "resume_fail");
426 ufshcd_print_err_hist(hba, &hba->ufs_stats.suspend_err,
427 "suspend_fail");
428 ufshcd_print_err_hist(hba, &hba->ufs_stats.dev_reset, "dev_reset");
429 ufshcd_print_err_hist(hba, &hba->ufs_stats.host_reset, "host_reset");
430 ufshcd_print_err_hist(hba, &hba->ufs_stats.task_abort, "task_abort");
ff8e20c6
DR		 431
432 ufshcd_print_clk_freqs(hba);
433
7c486d91 434 ufshcd_vops_dbg_register_dump(hba);
66cc820f
DR		 435}
436
437static
438void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
439{
440 struct ufshcd_lrb *lrbp;
7fabb77b 441 int prdt_length;
66cc820f
DR		 442 int tag;
443
444 for_each_set_bit(tag, &bitmap, hba->nutrs) {
445 lrbp = &hba->lrb[tag];
446
ff8e20c6
DR		 447 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
448 tag, ktime_to_us(lrbp->issue_time_stamp));
09017188
ZL		 449 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
450 tag, ktime_to_us(lrbp->compl_time_stamp));
ff8e20c6
DR		 451 dev_err(hba->dev,
452 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
453 tag, (u64)lrbp->utrd_dma_addr);
454
66cc820f
DR		 455 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
456 sizeof(struct utp_transfer_req_desc));
ff8e20c6
DR		 457 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
458 (u64)lrbp->ucd_req_dma_addr);
66cc820f
DR		 459 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
460 sizeof(struct utp_upiu_req));
ff8e20c6
DR		 461 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
462 (u64)lrbp->ucd_rsp_dma_addr);
66cc820f
DR		 463 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
464 sizeof(struct utp_upiu_rsp));
66cc820f 465
7fabb77b
GB		 466 prdt_length = le16_to_cpu(
467 lrbp->utr_descriptor_ptr->prd_table_length);
468 dev_err(hba->dev,
469 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
470 tag, prdt_length,
471 (u64)lrbp->ucd_prdt_dma_addr);
472
473 if (pr_prdt)
66cc820f 474 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
7fabb77b 475 sizeof(struct ufshcd_sg_entry) * prdt_length);
66cc820f
DR		 476 }
477}
478
479static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
480{
66cc820f
DR		 481 int tag;
482
483 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
391e388f
CH		 484 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
485
66cc820f 486 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
391e388f 487 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
66cc820f
DR		 488 }
489}
490
6ba65588
GB		 491static void ufshcd_print_host_state(struct ufs_hba *hba)
492{
493 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
7252a360
BVA		 494 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
495 hba->outstanding_reqs, hba->outstanding_tasks);
6ba65588
GB		 496 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
497 hba->saved_err, hba->saved_uic_err);
498 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
499 hba->curr_dev_pwr_mode, hba->uic_link_state);
500 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
501 hba->pm_op_in_progress, hba->is_sys_suspended);
502 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
503 hba->auto_bkops_enabled, hba->host->host_self_blocked);
504 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
505 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
506 hba->eh_flags, hba->req_abort_count);
507 dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n",
508 hba->capabilities, hba->caps);
509 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
510 hba->dev_quirks);
511}
512
ff8e20c6
DR		 513/**
514 * ufshcd_print_pwr_info - print power params as saved in hba
515 * power info
516 * @hba: per-adapter instance
517 */
518static void ufshcd_print_pwr_info(struct ufs_hba *hba)
519{
520 static const char * const names[] = {
521 "INVALID MODE",
522 "FAST MODE",
523 "SLOW_MODE",
524 "INVALID MODE",
525 "FASTAUTO_MODE",
526 "SLOWAUTO_MODE",
527 "INVALID MODE",
528 };
529
530 dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
531 __func__,
532 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
533 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
534 names[hba->pwr_info.pwr_rx],
535 names[hba->pwr_info.pwr_tx],
536 hba->pwr_info.hs_rate);
537}
538
5c955c10
SC		 539void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
540{
541 if (!us)
542 return;
543
544 if (us < 10)
545 udelay(us);
546 else
547 usleep_range(us, us + tolerance);
548}
549EXPORT_SYMBOL_GPL(ufshcd_delay_us);
550
5a0b0cb9
SRT		 551/*
552 * ufshcd_wait_for_register - wait for register value to change
553 * @hba - per-adapter interface
554 * @reg - mmio register offset
555 * @mask - mask to apply to read register value
556 * @val - wait condition
557 * @interval_us - polling interval in microsecs
558 * @timeout_ms - timeout in millisecs
596585a2 559 * @can_sleep - perform sleep or just spin
5a0b0cb9
SRT		 560 *
561 * Returns -ETIMEDOUT on error, zero on success
562 */
596585a2
YG		 563int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
564 u32 val, unsigned long interval_us,
565 unsigned long timeout_ms, bool can_sleep)
5a0b0cb9
SRT		 566{
567 int err = 0;
568 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
569
570 /* ignore bits that we don't intend to wait on */
571 val = val & mask;
572
573 while ((ufshcd_readl(hba, reg) & mask) != val) {
596585a2
YG		 574 if (can_sleep)
575 usleep_range(interval_us, interval_us + 50);
576 else
577 udelay(interval_us);
5a0b0cb9
SRT		 578 if (time_after(jiffies, timeout)) {
579 if ((ufshcd_readl(hba, reg) & mask) != val)
580 err = -ETIMEDOUT;
581 break;
582 }
583 }
584
585 return err;
586}
587
2fbd009b
SJ		 588/**
589 * ufshcd_get_intr_mask - Get the interrupt bit mask
8aa29f19 590 * @hba: Pointer to adapter instance
2fbd009b
SJ		 591 *
592 * Returns interrupt bit mask per version
593 */
594static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
595{
c01848c6
YG		 596 u32 intr_mask = 0;
597
598 switch (hba->ufs_version) {
599 case UFSHCI_VERSION_10:
600 intr_mask = INTERRUPT_MASK_ALL_VER_10;
601 break;
c01848c6
YG		 602 case UFSHCI_VERSION_11:
603 case UFSHCI_VERSION_20:
604 intr_mask = INTERRUPT_MASK_ALL_VER_11;
605 break;
c01848c6
YG		 606 case UFSHCI_VERSION_21:
607 default:
608 intr_mask = INTERRUPT_MASK_ALL_VER_21;
031d1e0f 609 break;
c01848c6
YG		 610 }
611
612 return intr_mask;
2fbd009b
SJ		 613}
614
7a3e97b0
SY		 615/**
616 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
8aa29f19 617 * @hba: Pointer to adapter instance
7a3e97b0
SY		 618 *
619 * Returns UFSHCI version supported by the controller
620 */
621static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
622{
0263bcd0
YG		 623 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
624 return ufshcd_vops_get_ufs_hci_version(hba);
9949e702 625
b873a275 626 return ufshcd_readl(hba, REG_UFS_VERSION);
7a3e97b0
SY		 627}
628
629/**
630 * ufshcd_is_device_present - Check if any device connected to
631 * the host controller
5c0c28a8 632 * @hba: pointer to adapter instance
7a3e97b0 633 *
c9e6010b 634 * Returns true if device present, false if no device detected
7a3e97b0 635 */
c9e6010b 636static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
7a3e97b0 637{
5c0c28a8 638 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
c9e6010b 639 DEVICE_PRESENT) ? true : false;
7a3e97b0
SY		 640}
641
642/**
643 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
8aa29f19 644 * @lrbp: pointer to local command reference block
7a3e97b0
SY		 645 *
646 * This function is used to get the OCS field from UTRD
647 * Returns the OCS field in the UTRD
648 */
649static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
650{
e8c8e82a 651 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
7a3e97b0
SY		 652}
653
7a3e97b0
SY		 654/**
655 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
656 * @hba: per adapter instance
657 * @pos: position of the bit to be cleared
658 */
659static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
660{
49200199 661 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
1399c5b0
AA		 662}
663
664/**
665 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
666 * @hba: per adapter instance
667 * @pos: position of the bit to be cleared
668 */
669static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
670{
49200199 671 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
7a3e97b0
SY		 672}
673
a48353f6
YG		 674/**
675 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
676 * @hba: per adapter instance
677 * @tag: position of the bit to be cleared
678 */
679static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
680{
681 __clear_bit(tag, &hba->outstanding_reqs);
682}
683
7a3e97b0
SY		 684/**
685 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
686 * @reg: Register value of host controller status
687 *
688 * Returns integer, 0 on Success and positive value if failed
689 */
690static inline int ufshcd_get_lists_status(u32 reg)
691{
6cf16115 692 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
7a3e97b0
SY		 693}
694
695/**
696 * ufshcd_get_uic_cmd_result - Get the UIC command result
697 * @hba: Pointer to adapter instance
698 *
699 * This function gets the result of UIC command completion
700 * Returns 0 on success, non zero value on error
701 */
702static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
703{
b873a275 704 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
7a3e97b0
SY		 705 MASK_UIC_COMMAND_RESULT;
706}
707
12b4fdb4
SJ		 708/**
709 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
710 * @hba: Pointer to adapter instance
711 *
712 * This function gets UIC command argument3
713 * Returns 0 on success, non zero value on error
714 */
715static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
716{
717 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
718}
719
7a3e97b0 720/**
5a0b0cb9 721 * ufshcd_get_req_rsp - returns the TR response transaction type
7a3e97b0 722 * @ucd_rsp_ptr: pointer to response UPIU
7a3e97b0
SY		 723 */
724static inline int
5a0b0cb9 725ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
7a3e97b0 726{
5a0b0cb9 727 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
7a3e97b0
SY		 728}
729
730/**
731 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
732 * @ucd_rsp_ptr: pointer to response UPIU
733 *
734 * This function gets the response status and scsi_status from response UPIU
735 * Returns the response result code.
736 */
737static inline int
738ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
739{
740 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
741}
742
1c2623c5
SJ		 743/*
744 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
745 * from response UPIU
746 * @ucd_rsp_ptr: pointer to response UPIU
747 *
748 * Return the data segment length.
749 */
750static inline unsigned int
751ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
752{
753 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
754 MASK_RSP_UPIU_DATA_SEG_LEN;
755}
756
66ec6d59
SRT		 757/**
758 * ufshcd_is_exception_event - Check if the device raised an exception event
759 * @ucd_rsp_ptr: pointer to response UPIU
760 *
761 * The function checks if the device raised an exception event indicated in
762 * the Device Information field of response UPIU.
763 *
764 * Returns true if exception is raised, false otherwise.
765 */
766static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
767{
768 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
769 MASK_RSP_EXCEPTION_EVENT ? true : false;
770}
771
7a3e97b0 772/**
7d568652 773 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
7a3e97b0 774 * @hba: per adapter instance
7a3e97b0
SY		 775 */
776static inline void
7d568652 777ufshcd_reset_intr_aggr(struct ufs_hba *hba)
7a3e97b0 778{
7d568652
SJ		 779 ufshcd_writel(hba, INT_AGGR_ENABLE |
780 INT_AGGR_COUNTER_AND_TIMER_RESET,
781 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
782}
783
784/**
785 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
786 * @hba: per adapter instance
787 * @cnt: Interrupt aggregation counter threshold
788 * @tmout: Interrupt aggregation timeout value
789 */
790static inline void
791ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
792{
793 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
794 INT_AGGR_COUNTER_THLD_VAL(cnt) |
795 INT_AGGR_TIMEOUT_VAL(tmout),
796 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
7a3e97b0
SY		 797}
798
b852190e
YG		 799/**
800 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
801 * @hba: per adapter instance
802 */
803static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
804{
805 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
806}
807
7a3e97b0
SY		 808/**
809 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
810 * When run-stop registers are set to 1, it indicates the
811 * host controller that it can process the requests
812 * @hba: per adapter instance
813 */
814static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
815{
b873a275
SJ		 816 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
817 REG_UTP_TASK_REQ_LIST_RUN_STOP);
818 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
819 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
7a3e97b0
SY		 820}
821
7a3e97b0
SY		 822/**
823 * ufshcd_hba_start - Start controller initialization sequence
824 * @hba: per adapter instance
825 */
826static inline void ufshcd_hba_start(struct ufs_hba *hba)
827{
b873a275 828 ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
7a3e97b0
SY		 829}
830
831/**
832 * ufshcd_is_hba_active - Get controller state
833 * @hba: per adapter instance
834 *
c9e6010b 835 * Returns false if controller is active, true otherwise
7a3e97b0 836 */
c9e6010b 837static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
7a3e97b0 838{
4a8eec2b
TK		 839 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
840 ? false : true;
7a3e97b0
SY		 841}
842
37113106
YG		 843u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
844{
845 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
846 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
847 (hba->ufs_version == UFSHCI_VERSION_11))
848 return UFS_UNIPRO_VER_1_41;
849 else
850 return UFS_UNIPRO_VER_1_6;
851}
852EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
853
854static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
855{
856 /*
857 * If both host and device support UniPro ver1.6 or later, PA layer
858 * parameters tuning happens during link startup itself.
859 *
860 * We can manually tune PA layer parameters if either host or device
861 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
862 * logic simple, we will only do manual tuning if local unipro version
863 * doesn't support ver1.6 or later.
864 */
865 if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
866 return true;
867 else
868 return false;
869}
870
a3cd5ec5
SJ		 871static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
872{
873 int ret = 0;
874 struct ufs_clk_info *clki;
875 struct list_head *head = &hba->clk_list_head;
876 ktime_t start = ktime_get();
877 bool clk_state_changed = false;
878
566ec9ad 879 if (list_empty(head))
a3cd5ec5
SJ		 880 goto out;
881
882 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
883 if (ret)
884 return ret;
885
886 list_for_each_entry(clki, head, list) {
887 if (!IS_ERR_OR_NULL(clki->clk)) {
888 if (scale_up && clki->max_freq) {
889 if (clki->curr_freq == clki->max_freq)
890 continue;
891
892 clk_state_changed = true;
893 ret = clk_set_rate(clki->clk, clki->max_freq);
894 if (ret) {
895 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
896 __func__, clki->name,
897 clki->max_freq, ret);
898 break;
899 }
900 trace_ufshcd_clk_scaling(dev_name(hba->dev),
901 "scaled up", clki->name,
902 clki->curr_freq,
903 clki->max_freq);
904
905 clki->curr_freq = clki->max_freq;
906
907 } else if (!scale_up && clki->min_freq) {
908 if (clki->curr_freq == clki->min_freq)
909 continue;
910
911 clk_state_changed = true;
912 ret = clk_set_rate(clki->clk, clki->min_freq);
913 if (ret) {
914 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
915 __func__, clki->name,
916 clki->min_freq, ret);
917 break;
918 }
919 trace_ufshcd_clk_scaling(dev_name(hba->dev),
920 "scaled down", clki->name,
921 clki->curr_freq,
922 clki->min_freq);
923 clki->curr_freq = clki->min_freq;
924 }
925 }
926 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
927 clki->name, clk_get_rate(clki->clk));
928 }
929
930 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
931
932out:
933 if (clk_state_changed)
934 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
935 (scale_up ? "up" : "down"),
936 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
937 return ret;
938}
939
940/**
941 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
942 * @hba: per adapter instance
943 * @scale_up: True if scaling up and false if scaling down
944 *
945 * Returns true if scaling is required, false otherwise.
946 */
947static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
948 bool scale_up)
949{
950 struct ufs_clk_info *clki;
951 struct list_head *head = &hba->clk_list_head;
952
566ec9ad 953 if (list_empty(head))
a3cd5ec5
SJ		 954 return false;
955
956 list_for_each_entry(clki, head, list) {
957 if (!IS_ERR_OR_NULL(clki->clk)) {
958 if (scale_up && clki->max_freq) {
959 if (clki->curr_freq == clki->max_freq)
960 continue;
961 return true;
962 } else if (!scale_up && clki->min_freq) {
963 if (clki->curr_freq == clki->min_freq)
964 continue;
965 return true;
966 }
967 }
968 }
969
970 return false;
971}
972
973static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
974 u64 wait_timeout_us)
975{
976 unsigned long flags;
977 int ret = 0;
978 u32 tm_doorbell;
979 u32 tr_doorbell;
980 bool timeout = false, do_last_check = false;
981 ktime_t start;
982
983 ufshcd_hold(hba, false);
984 spin_lock_irqsave(hba->host->host_lock, flags);
985 /*
986 * Wait for all the outstanding tasks/transfer requests.
987 * Verify by checking the doorbell registers are clear.
988 */
989 start = ktime_get();
990 do {
991 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
992 ret = -EBUSY;
993 goto out;
994 }
995
996 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
997 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
998 if (!tm_doorbell && !tr_doorbell) {
999 timeout = false;
1000 break;
1001 } else if (do_last_check) {
1002 break;
1003 }
1004
1005 spin_unlock_irqrestore(hba->host->host_lock, flags);
1006 schedule();
1007 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1008 wait_timeout_us) {
1009 timeout = true;
1010 /*
1011 * We might have scheduled out for long time so make
1012 * sure to check if doorbells are cleared by this time
1013 * or not.
1014 */
1015 do_last_check = true;
1016 }
1017 spin_lock_irqsave(hba->host->host_lock, flags);
1018 } while (tm_doorbell || tr_doorbell);
1019
1020 if (timeout) {
1021 dev_err(hba->dev,
1022 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1023 __func__, tm_doorbell, tr_doorbell);
1024 ret = -EBUSY;
1025 }
1026out:
1027 spin_unlock_irqrestore(hba->host->host_lock, flags);
1028 ufshcd_release(hba);
1029 return ret;
1030}
1031
1032/**
1033 * ufshcd_scale_gear - scale up/down UFS gear
1034 * @hba: per adapter instance
1035 * @scale_up: True for scaling up gear and false for scaling down
1036 *
1037 * Returns 0 for success,
1038 * Returns -EBUSY if scaling can't happen at this time
1039 * Returns non-zero for any other errors
1040 */
1041static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1042{
1043 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1044 int ret = 0;
1045 struct ufs_pa_layer_attr new_pwr_info;
1046
1047 if (scale_up) {
1048 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1049 sizeof(struct ufs_pa_layer_attr));
1050 } else {
1051 memcpy(&new_pwr_info, &hba->pwr_info,
1052 sizeof(struct ufs_pa_layer_attr));
1053
1054 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1055 || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1056 /* save the current power mode */
1057 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1058 &hba->pwr_info,
1059 sizeof(struct ufs_pa_layer_attr));
1060
1061 /* scale down gear */
1062 new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1063 new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1064 }
1065 }
1066
1067 /* check if the power mode needs to be changed or not? */
1068 ret = ufshcd_change_power_mode(hba, &new_pwr_info);
1069
1070 if (ret)
1071 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1072 __func__, ret,
1073 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1074 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1075
1076 return ret;
1077}
1078
1079static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1080{
1081 #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
1082 int ret = 0;
1083 /*
1084 * make sure that there are no outstanding requests when
1085 * clock scaling is in progress
1086 */
38135535 1087 ufshcd_scsi_block_requests(hba);
a3cd5ec5
SJ		 1088 down_write(&hba->clk_scaling_lock);
1089 if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1090 ret = -EBUSY;
1091 up_write(&hba->clk_scaling_lock);
38135535 1092 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5
SJ		 1093 }
1094
1095 return ret;
1096}
1097
1098static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1099{
1100 up_write(&hba->clk_scaling_lock);
38135535 1101 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5
SJ		 1102}
1103
1104/**
1105 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1106 * @hba: per adapter instance
1107 * @scale_up: True for scaling up and false for scalin down
1108 *
1109 * Returns 0 for success,
1110 * Returns -EBUSY if scaling can't happen at this time
1111 * Returns non-zero for any other errors
1112 */
1113static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1114{
1115 int ret = 0;
1116
401f1e44
SJ		 1117 /* let's not get into low power until clock scaling is completed */
1118 ufshcd_hold(hba, false);
1119
a3cd5ec5
SJ		 1120 ret = ufshcd_clock_scaling_prepare(hba);
1121 if (ret)
1122 return ret;
1123
1124 /* scale down the gear before scaling down clocks */
1125 if (!scale_up) {
1126 ret = ufshcd_scale_gear(hba, false);
1127 if (ret)
1128 goto out;
1129 }
1130
1131 ret = ufshcd_scale_clks(hba, scale_up);
1132 if (ret) {
1133 if (!scale_up)
1134 ufshcd_scale_gear(hba, true);
1135 goto out;
1136 }
1137
1138 /* scale up the gear after scaling up clocks */
1139 if (scale_up) {
1140 ret = ufshcd_scale_gear(hba, true);
1141 if (ret) {
1142 ufshcd_scale_clks(hba, false);
1143 goto out;
1144 }
1145 }
1146
1147 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1148
1149out:
1150 ufshcd_clock_scaling_unprepare(hba);
401f1e44 1151 ufshcd_release(hba);
a3cd5ec5
SJ		 1152 return ret;
1153}
1154
401f1e44
SJ		 1155static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1156{
1157 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1158 clk_scaling.suspend_work);
1159 unsigned long irq_flags;
1160
1161 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1162 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1163 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1164 return;
1165 }
1166 hba->clk_scaling.is_suspended = true;
1167 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1168
1169 __ufshcd_suspend_clkscaling(hba);
1170}
1171
1172static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1173{
1174 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1175 clk_scaling.resume_work);
1176 unsigned long irq_flags;
1177
1178 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1179 if (!hba->clk_scaling.is_suspended) {
1180 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1181 return;
1182 }
1183 hba->clk_scaling.is_suspended = false;
1184 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1185
1186 devfreq_resume_device(hba->devfreq);
1187}
1188
a3cd5ec5
SJ		 1189static int ufshcd_devfreq_target(struct device *dev,
1190 unsigned long *freq, u32 flags)
1191{
1192 int ret = 0;
1193 struct ufs_hba *hba = dev_get_drvdata(dev);
1194 ktime_t start;
401f1e44 1195 bool scale_up, sched_clk_scaling_suspend_work = false;
092b4558
BA		 1196 struct list_head *clk_list = &hba->clk_list_head;
1197 struct ufs_clk_info *clki;
a3cd5ec5
SJ		 1198 unsigned long irq_flags;
1199
1200 if (!ufshcd_is_clkscaling_supported(hba))
1201 return -EINVAL;
1202
91831d33
AD		 1203 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1204 /* Override with the closest supported frequency */
1205 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
a3cd5ec5
SJ		 1206 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1207 if (ufshcd_eh_in_progress(hba)) {
1208 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1209 return 0;
1210 }
1211
401f1e44
SJ		 1212 if (!hba->clk_scaling.active_reqs)
1213 sched_clk_scaling_suspend_work = true;
1214
092b4558
BA		 1215 if (list_empty(clk_list)) {
1216 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1217 goto out;
1218 }
1219
91831d33 1220 /* Decide based on the rounded-off frequency and update */
092b4558 1221 scale_up = (*freq == clki->max_freq) ? true : false;
91831d33
AD		 1222 if (!scale_up)
1223 *freq = clki->min_freq;
1224 /* Update the frequency */
401f1e44
SJ		 1225 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1226 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1227 ret = 0;
1228 goto out; /* no state change required */
a3cd5ec5
SJ		 1229 }
1230 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1231
1232 start = ktime_get();
a3cd5ec5
SJ		 1233 ret = ufshcd_devfreq_scale(hba, scale_up);
1234
a3cd5ec5
SJ		 1235 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1236 (scale_up ? "up" : "down"),
1237 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1238
401f1e44
SJ		 1239out:
1240 if (sched_clk_scaling_suspend_work)
1241 queue_work(hba->clk_scaling.workq,
1242 &hba->clk_scaling.suspend_work);
1243
a3cd5ec5
SJ		 1244 return ret;
1245}
1246
7252a360
BVA		 1247static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1248{
1249 int *busy = priv;
1250
1251 WARN_ON_ONCE(reserved);
1252 (*busy)++;
1253 return false;
1254}
1255
1256/* Whether or not any tag is in use by a request that is in progress. */
1257static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1258{
1259 struct request_queue *q = hba->cmd_queue;
1260 int busy = 0;
1261
1262 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1263 return busy;
1264}
a3cd5ec5
SJ		 1265
1266static int ufshcd_devfreq_get_dev_status(struct device *dev,
1267 struct devfreq_dev_status *stat)
1268{
1269 struct ufs_hba *hba = dev_get_drvdata(dev);
1270 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1271 unsigned long flags;
91831d33
AD		 1272 struct list_head *clk_list = &hba->clk_list_head;
1273 struct ufs_clk_info *clki;
a3cd5ec5
SJ		 1274
1275 if (!ufshcd_is_clkscaling_supported(hba))
1276 return -EINVAL;
1277
1278 memset(stat, 0, sizeof(*stat));
1279
1280 spin_lock_irqsave(hba->host->host_lock, flags);
1281 if (!scaling->window_start_t)
1282 goto start_window;
1283
91831d33
AD		 1284 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1285 /*
1286 * If current frequency is 0, then the ondemand governor considers
1287 * there's no initial frequency set. And it always requests to set
1288 * to max. frequency.
1289 */
1290 stat->current_frequency = clki->curr_freq;
a3cd5ec5
SJ		 1291 if (scaling->is_busy_started)
1292 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1293 scaling->busy_start_t));
1294
1295 stat->total_time = jiffies_to_usecs((long)jiffies -
1296 (long)scaling->window_start_t);
1297 stat->busy_time = scaling->tot_busy_t;
1298start_window:
1299 scaling->window_start_t = jiffies;
1300 scaling->tot_busy_t = 0;
1301
1302 if (hba->outstanding_reqs) {
1303 scaling->busy_start_t = ktime_get();
1304 scaling->is_busy_started = true;
1305 } else {
1306 scaling->busy_start_t = 0;
1307 scaling->is_busy_started = false;
1308 }
1309 spin_unlock_irqrestore(hba->host->host_lock, flags);
1310 return 0;
1311}
1312
1313static struct devfreq_dev_profile ufs_devfreq_profile = {
1314 .polling_ms = 100,
1315 .target = ufshcd_devfreq_target,
1316 .get_dev_status = ufshcd_devfreq_get_dev_status,
1317};
1318
2c75f9a5
AD		 1319#if IS_ENABLED(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND)
1320static struct devfreq_simple_ondemand_data ufs_ondemand_data = {
1321 .upthreshold = 70,
1322 .downdifferential = 5,
1323};
1324
1325static void *gov_data = &ufs_ondemand_data;
1326#else
1327static void *gov_data; /* NULL */
1328#endif
1329
deac444f
BA		 1330static int ufshcd_devfreq_init(struct ufs_hba *hba)
1331{
092b4558
BA		 1332 struct list_head *clk_list = &hba->clk_list_head;
1333 struct ufs_clk_info *clki;
deac444f
BA		 1334 struct devfreq *devfreq;
1335 int ret;
1336
092b4558
BA		 1337 /* Skip devfreq if we don't have any clocks in the list */
1338 if (list_empty(clk_list))
1339 return 0;
1340
1341 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1342 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1343 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1344
2c75f9a5
AD		 1345 ufshcd_vops_config_scaling_param(hba, &ufs_devfreq_profile,
1346 gov_data);
092b4558 1347 devfreq = devfreq_add_device(hba->dev,
deac444f
BA		 1348 &ufs_devfreq_profile,
1349 DEVFREQ_GOV_SIMPLE_ONDEMAND,
2c75f9a5 1350 gov_data);
deac444f
BA		 1351 if (IS_ERR(devfreq)) {
1352 ret = PTR_ERR(devfreq);
1353 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
092b4558
BA		 1354
1355 dev_pm_opp_remove(hba->dev, clki->min_freq);
1356 dev_pm_opp_remove(hba->dev, clki->max_freq);
deac444f
BA		 1357 return ret;
1358 }
1359
1360 hba->devfreq = devfreq;
1361
1362 return 0;
1363}
1364
092b4558
BA		 1365static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1366{
1367 struct list_head *clk_list = &hba->clk_list_head;
1368 struct ufs_clk_info *clki;
1369
1370 if (!hba->devfreq)
1371 return;
1372
1373 devfreq_remove_device(hba->devfreq);
1374 hba->devfreq = NULL;
1375
1376 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1377 dev_pm_opp_remove(hba->dev, clki->min_freq);
1378 dev_pm_opp_remove(hba->dev, clki->max_freq);
1379}
1380
401f1e44
SJ		 1381static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1382{
1383 unsigned long flags;
1384
1385 devfreq_suspend_device(hba->devfreq);
1386 spin_lock_irqsave(hba->host->host_lock, flags);
1387 hba->clk_scaling.window_start_t = 0;
1388 spin_unlock_irqrestore(hba->host->host_lock, flags);
1389}
a3cd5ec5 1390
a508253d
GB		 1391static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1392{
401f1e44
SJ		 1393 unsigned long flags;
1394 bool suspend = false;
1395
fcb0c4b0
ST		 1396 if (!ufshcd_is_clkscaling_supported(hba))
1397 return;
1398
401f1e44
SJ		 1399 spin_lock_irqsave(hba->host->host_lock, flags);
1400 if (!hba->clk_scaling.is_suspended) {
1401 suspend = true;
1402 hba->clk_scaling.is_suspended = true;
1403 }
1404 spin_unlock_irqrestore(hba->host->host_lock, flags);
1405
1406 if (suspend)
1407 __ufshcd_suspend_clkscaling(hba);
a508253d
GB		 1408}
1409
1410static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1411{
401f1e44
SJ		 1412 unsigned long flags;
1413 bool resume = false;
1414
1415 if (!ufshcd_is_clkscaling_supported(hba))
1416 return;
1417
1418 spin_lock_irqsave(hba->host->host_lock, flags);
1419 if (hba->clk_scaling.is_suspended) {
1420 resume = true;
1421 hba->clk_scaling.is_suspended = false;
1422 }
1423 spin_unlock_irqrestore(hba->host->host_lock, flags);
1424
1425 if (resume)
1426 devfreq_resume_device(hba->devfreq);
fcb0c4b0
ST		 1427}
1428
1429static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1430 struct device_attribute *attr, char *buf)
1431{
1432 struct ufs_hba *hba = dev_get_drvdata(dev);
1433
1434 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1435}
1436
1437static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1438 struct device_attribute *attr, const char *buf, size_t count)
1439{
1440 struct ufs_hba *hba = dev_get_drvdata(dev);
1441 u32 value;
1442 int err;
1443
1444 if (kstrtou32(buf, 0, &value))
1445 return -EINVAL;
1446
1447 value = !!value;
1448 if (value == hba->clk_scaling.is_allowed)
1449 goto out;
1450
1451 pm_runtime_get_sync(hba->dev);
1452 ufshcd_hold(hba, false);
1453
401f1e44
SJ		 1454 cancel_work_sync(&hba->clk_scaling.suspend_work);
1455 cancel_work_sync(&hba->clk_scaling.resume_work);
1456
1457 hba->clk_scaling.is_allowed = value;
1458
fcb0c4b0
ST		 1459 if (value) {
1460 ufshcd_resume_clkscaling(hba);
1461 } else {
1462 ufshcd_suspend_clkscaling(hba);
a3cd5ec5 1463 err = ufshcd_devfreq_scale(hba, true);
fcb0c4b0
ST		 1464 if (err)
1465 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1466 __func__, err);
1467 }
fcb0c4b0
ST		 1468
1469 ufshcd_release(hba);
1470 pm_runtime_put_sync(hba->dev);
1471out:
1472 return count;
a508253d
GB		 1473}
1474
a3cd5ec5
SJ		 1475static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1476{
1477 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1478 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1479 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1480 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1481 hba->clk_scaling.enable_attr.attr.mode = 0644;
1482 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1483 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1484}
1485
1ab27c9c
ST		 1486static void ufshcd_ungate_work(struct work_struct *work)
1487{
1488 int ret;
1489 unsigned long flags;
1490 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1491 clk_gating.ungate_work);
1492
1493 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1494
1495 spin_lock_irqsave(hba->host->host_lock, flags);
1496 if (hba->clk_gating.state == CLKS_ON) {
1497 spin_unlock_irqrestore(hba->host->host_lock, flags);
1498 goto unblock_reqs;
1499 }
1500
1501 spin_unlock_irqrestore(hba->host->host_lock, flags);
1502 ufshcd_setup_clocks(hba, true);
1503
8b0bbf00
SC		 1504 ufshcd_enable_irq(hba);
1505
1ab27c9c
ST		 1506 /* Exit from hibern8 */
1507 if (ufshcd_can_hibern8_during_gating(hba)) {
1508 /* Prevent gating in this path */
1509 hba->clk_gating.is_suspended = true;
1510 if (ufshcd_is_link_hibern8(hba)) {
1511 ret = ufshcd_uic_hibern8_exit(hba);
1512 if (ret)
1513 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1514 __func__, ret);
1515 else
1516 ufshcd_set_link_active(hba);
1517 }
1518 hba->clk_gating.is_suspended = false;
1519 }
1520unblock_reqs:
38135535 1521 ufshcd_scsi_unblock_requests(hba);
1ab27c9c
ST		 1522}
1523
1524/**
1525 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1526 * Also, exit from hibern8 mode and set the link as active.
1527 * @hba: per adapter instance
1528 * @async: This indicates whether caller should ungate clocks asynchronously.
1529 */
1530int ufshcd_hold(struct ufs_hba *hba, bool async)
1531{
1532 int rc = 0;
1533 unsigned long flags;
1534
1535 if (!ufshcd_is_clkgating_allowed(hba))
1536 goto out;
1ab27c9c
ST		 1537 spin_lock_irqsave(hba->host->host_lock, flags);
1538 hba->clk_gating.active_reqs++;
1539
53c12d0e
YG		 1540 if (ufshcd_eh_in_progress(hba)) {
1541 spin_unlock_irqrestore(hba->host->host_lock, flags);
1542 return 0;
1543 }
1544
856b3483 1545start:
1ab27c9c
ST		 1546 switch (hba->clk_gating.state) {
1547 case CLKS_ON:
f2a785ac
VG		 1548 /*
1549 * Wait for the ungate work to complete if in progress.
1550 * Though the clocks may be in ON state, the link could
1551 * still be in hibner8 state if hibern8 is allowed
1552 * during clock gating.
1553 * Make sure we exit hibern8 state also in addition to
1554 * clocks being ON.
1555 */
1556 if (ufshcd_can_hibern8_during_gating(hba) &&
1557 ufshcd_is_link_hibern8(hba)) {
c63d6099
CG		 1558 if (async) {
1559 rc = -EAGAIN;
1560 hba->clk_gating.active_reqs--;
1561 break;
1562 }
f2a785ac
VG		 1563 spin_unlock_irqrestore(hba->host->host_lock, flags);
1564 flush_work(&hba->clk_gating.ungate_work);
1565 spin_lock_irqsave(hba->host->host_lock, flags);
1566 goto start;
1567 }
1ab27c9c
ST		 1568 break;
1569 case REQ_CLKS_OFF:
1570 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1571 hba->clk_gating.state = CLKS_ON;
7ff5ab47
SJ		 1572 trace_ufshcd_clk_gating(dev_name(hba->dev),
1573 hba->clk_gating.state);
1ab27c9c
ST		 1574 break;
1575 }
1576 /*
9c490d2d 1577 * If we are here, it means gating work is either done or
1ab27c9c
ST		 1578 * currently running. Hence, fall through to cancel gating
1579 * work and to enable clocks.
1580 */
30eb2e4c 1581 /* fallthrough */
1ab27c9c 1582 case CLKS_OFF:
38135535 1583 ufshcd_scsi_block_requests(hba);
1ab27c9c 1584 hba->clk_gating.state = REQ_CLKS_ON;
7ff5ab47
SJ		 1585 trace_ufshcd_clk_gating(dev_name(hba->dev),
1586 hba->clk_gating.state);
10e5e375
VV		 1587 queue_work(hba->clk_gating.clk_gating_workq,
1588 &hba->clk_gating.ungate_work);
1ab27c9c
ST		 1589 /*
1590 * fall through to check if we should wait for this
1591 * work to be done or not.
1592 */
30eb2e4c 1593 /* fallthrough */
1ab27c9c
ST		 1594 case REQ_CLKS_ON:
1595 if (async) {
1596 rc = -EAGAIN;
1597 hba->clk_gating.active_reqs--;
1598 break;
1599 }
1600
1601 spin_unlock_irqrestore(hba->host->host_lock, flags);
1602 flush_work(&hba->clk_gating.ungate_work);
1603 /* Make sure state is CLKS_ON before returning */
856b3483 1604 spin_lock_irqsave(hba->host->host_lock, flags);
1ab27c9c
ST		 1605 goto start;
1606 default:
1607 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1608 __func__, hba->clk_gating.state);
1609 break;
1610 }
1611 spin_unlock_irqrestore(hba->host->host_lock, flags);
1612out:
1613 return rc;
1614}
6e3fd44d 1615EXPORT_SYMBOL_GPL(ufshcd_hold);
1ab27c9c
ST		 1616
1617static void ufshcd_gate_work(struct work_struct *work)
1618{
1619 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1620 clk_gating.gate_work.work);
1621 unsigned long flags;
1622
1623 spin_lock_irqsave(hba->host->host_lock, flags);
3f0c06de
VG		 1624 /*
1625 * In case you are here to cancel this work the gating state
1626 * would be marked as REQ_CLKS_ON. In this case save time by
1627 * skipping the gating work and exit after changing the clock
1628 * state to CLKS_ON.
1629 */
1630 if (hba->clk_gating.is_suspended ||
18f01374 1631 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1ab27c9c 1632 hba->clk_gating.state = CLKS_ON;
7ff5ab47
SJ		 1633 trace_ufshcd_clk_gating(dev_name(hba->dev),
1634 hba->clk_gating.state);
1ab27c9c
ST		 1635 goto rel_lock;
1636 }
1637
1638 if (hba->clk_gating.active_reqs
1639 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
7252a360 1640 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1ab27c9c
ST		 1641 || hba->active_uic_cmd || hba->uic_async_done)
1642 goto rel_lock;
1643
1644 spin_unlock_irqrestore(hba->host->host_lock, flags);
1645
1646 /* put the link into hibern8 mode before turning off clocks */
1647 if (ufshcd_can_hibern8_during_gating(hba)) {
1648 if (ufshcd_uic_hibern8_enter(hba)) {
1649 hba->clk_gating.state = CLKS_ON;
7ff5ab47
SJ		 1650 trace_ufshcd_clk_gating(dev_name(hba->dev),
1651 hba->clk_gating.state);
1ab27c9c
ST		 1652 goto out;
1653 }
1654 ufshcd_set_link_hibern8(hba);
1655 }
1656
8b0bbf00
SC		 1657 ufshcd_disable_irq(hba);
1658
1ab27c9c
ST		 1659 if (!ufshcd_is_link_active(hba))
1660 ufshcd_setup_clocks(hba, false);
1661 else
1662 /* If link is active, device ref_clk can't be switched off */
1663 __ufshcd_setup_clocks(hba, false, true);
1664
1665 /*
1666 * In case you are here to cancel this work the gating state
1667 * would be marked as REQ_CLKS_ON. In this case keep the state
1668 * as REQ_CLKS_ON which would anyway imply that clocks are off
1669 * and a request to turn them on is pending. By doing this way,
1670 * we keep the state machine in tact and this would ultimately
1671 * prevent from doing cancel work multiple times when there are
1672 * new requests arriving before the current cancel work is done.
1673 */
1674 spin_lock_irqsave(hba->host->host_lock, flags);
7ff5ab47 1675 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1ab27c9c 1676 hba->clk_gating.state = CLKS_OFF;
7ff5ab47
SJ		 1677 trace_ufshcd_clk_gating(dev_name(hba->dev),
1678 hba->clk_gating.state);
1679 }
1ab27c9c
ST		 1680rel_lock:
1681 spin_unlock_irqrestore(hba->host->host_lock, flags);
1682out:
1683 return;
1684}
1685
1686/* host lock must be held before calling this variant */
1687static void __ufshcd_release(struct ufs_hba *hba)
1688{
1689 if (!ufshcd_is_clkgating_allowed(hba))
1690 return;
1691
1692 hba->clk_gating.active_reqs--;
1693
1694 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
1695 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
7252a360 1696 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
53c12d0e
YG		 1697 || hba->active_uic_cmd || hba->uic_async_done
1698 || ufshcd_eh_in_progress(hba))
1ab27c9c
ST		 1699 return;
1700
1701 hba->clk_gating.state = REQ_CLKS_OFF;
7ff5ab47 1702 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
f4bb7704
EG		 1703 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1704 &hba->clk_gating.gate_work,
1705 msecs_to_jiffies(hba->clk_gating.delay_ms));
1ab27c9c
ST		 1706}
1707
1708void ufshcd_release(struct ufs_hba *hba)
1709{
1710 unsigned long flags;
1711
1712 spin_lock_irqsave(hba->host->host_lock, flags);
1713 __ufshcd_release(hba);
1714 spin_unlock_irqrestore(hba->host->host_lock, flags);
1715}
6e3fd44d 1716EXPORT_SYMBOL_GPL(ufshcd_release);
1ab27c9c
ST		 1717
1718static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1719 struct device_attribute *attr, char *buf)
1720{
1721 struct ufs_hba *hba = dev_get_drvdata(dev);
1722
1723 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1724}
1725
1726static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1727 struct device_attribute *attr, const char *buf, size_t count)
1728{
1729 struct ufs_hba *hba = dev_get_drvdata(dev);
1730 unsigned long flags, value;
1731
1732 if (kstrtoul(buf, 0, &value))
1733 return -EINVAL;
1734
1735 spin_lock_irqsave(hba->host->host_lock, flags);
1736 hba->clk_gating.delay_ms = value;
1737 spin_unlock_irqrestore(hba->host->host_lock, flags);
1738 return count;
1739}
1740
b427411a
ST		 1741static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1742 struct device_attribute *attr, char *buf)
1743{
1744 struct ufs_hba *hba = dev_get_drvdata(dev);
1745
1746 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1747}
1748
1749static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1750 struct device_attribute *attr, const char *buf, size_t count)
1751{
1752 struct ufs_hba *hba = dev_get_drvdata(dev);
1753 unsigned long flags;
1754 u32 value;
1755
1756 if (kstrtou32(buf, 0, &value))
1757 return -EINVAL;
1758
1759 value = !!value;
1760 if (value == hba->clk_gating.is_enabled)
1761 goto out;
1762
1763 if (value) {
1764 ufshcd_release(hba);
1765 } else {
1766 spin_lock_irqsave(hba->host->host_lock, flags);
1767 hba->clk_gating.active_reqs++;
1768 spin_unlock_irqrestore(hba->host->host_lock, flags);
1769 }
1770
1771 hba->clk_gating.is_enabled = value;
1772out:
1773 return count;
1774}
1775
eebcc196
VG		 1776static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1777{
1778 char wq_name[sizeof("ufs_clkscaling_00")];
1779
1780 if (!ufshcd_is_clkscaling_supported(hba))
1781 return;
1782
1783 INIT_WORK(&hba->clk_scaling.suspend_work,
1784 ufshcd_clk_scaling_suspend_work);
1785 INIT_WORK(&hba->clk_scaling.resume_work,
1786 ufshcd_clk_scaling_resume_work);
1787
1788 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1789 hba->host->host_no);
1790 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1791
1792 ufshcd_clkscaling_init_sysfs(hba);
1793}
1794
1795static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1796{
1797 if (!ufshcd_is_clkscaling_supported(hba))
1798 return;
1799
1800 destroy_workqueue(hba->clk_scaling.workq);
1801 ufshcd_devfreq_remove(hba);
1802}
1803
1ab27c9c
ST		 1804static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1805{
10e5e375
VV		 1806 char wq_name[sizeof("ufs_clk_gating_00")];
1807
1ab27c9c
ST		 1808 if (!ufshcd_is_clkgating_allowed(hba))
1809 return;
1810
1811 hba->clk_gating.delay_ms = 150;
1812 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1813 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1814
10e5e375
VV		 1815 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1816 hba->host->host_no);
1817 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1818 WQ_MEM_RECLAIM);
1819
b427411a
ST		 1820 hba->clk_gating.is_enabled = true;
1821
1ab27c9c
ST		 1822 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1823 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1824 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1825 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
b427411a 1826 hba->clk_gating.delay_attr.attr.mode = 0644;
1ab27c9c
ST		 1827 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1828 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
b427411a
ST		 1829
1830 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1831 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1832 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1833 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1834 hba->clk_gating.enable_attr.attr.mode = 0644;
1835 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1836 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1ab27c9c
ST		 1837}
1838
1839static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1840{
1841 if (!ufshcd_is_clkgating_allowed(hba))
1842 return;
1843 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
b427411a 1844 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
97cd6805
AM		 1845 cancel_work_sync(&hba->clk_gating.ungate_work);
1846 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
10e5e375 1847 destroy_workqueue(hba->clk_gating.clk_gating_workq);
1ab27c9c
ST		 1848}
1849
856b3483
ST		 1850/* Must be called with host lock acquired */
1851static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1852{
401f1e44
SJ		 1853 bool queue_resume_work = false;
1854
fcb0c4b0 1855 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1856 return;
1857
401f1e44
SJ		 1858 if (!hba->clk_scaling.active_reqs++)
1859 queue_resume_work = true;
1860
1861 if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1862 return;
1863
1864 if (queue_resume_work)
1865 queue_work(hba->clk_scaling.workq,
1866 &hba->clk_scaling.resume_work);
1867
1868 if (!hba->clk_scaling.window_start_t) {
1869 hba->clk_scaling.window_start_t = jiffies;
1870 hba->clk_scaling.tot_busy_t = 0;
1871 hba->clk_scaling.is_busy_started = false;
1872 }
1873
856b3483
ST		 1874 if (!hba->clk_scaling.is_busy_started) {
1875 hba->clk_scaling.busy_start_t = ktime_get();
1876 hba->clk_scaling.is_busy_started = true;
1877 }
1878}
1879
1880static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1881{
1882 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1883
fcb0c4b0 1884 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1885 return;
1886
1887 if (!hba->outstanding_reqs && scaling->is_busy_started) {
1888 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1889 scaling->busy_start_t));
8b0e1953 1890 scaling->busy_start_t = 0;
856b3483
ST		 1891 scaling->is_busy_started = false;
1892 }
1893}
7a3e97b0
SY		 1894/**
1895 * ufshcd_send_command - Send SCSI or device management commands
1896 * @hba: per adapter instance
1897 * @task_tag: Task tag of the command
1898 */
1899static inline
1900void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1901{
ff8e20c6 1902 hba->lrb[task_tag].issue_time_stamp = ktime_get();
09017188 1903 hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0);
eacf36f5 1904 ufshcd_add_command_trace(hba, task_tag, "send");
856b3483 1905 ufshcd_clk_scaling_start_busy(hba);
7a3e97b0 1906 __set_bit(task_tag, &hba->outstanding_reqs);
b873a275 1907 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
ad1a1b9c
GB		 1908 /* Make sure that doorbell is committed immediately */
1909 wmb();
7a3e97b0
SY		 1910}
1911
1912/**
1913 * ufshcd_copy_sense_data - Copy sense data in case of check condition
8aa29f19 1914 * @lrbp: pointer to local reference block
7a3e97b0
SY		 1915 */
1916static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1917{
1918 int len;
1c2623c5
SJ		 1919 if (lrbp->sense_buffer &&
1920 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
e3ce73d6
YG		 1921 int len_to_copy;
1922
5a0b0cb9 1923 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
09a5a24f 1924 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
e3ce73d6 1925
09a5a24f
AA		 1926 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
1927 len_to_copy);
7a3e97b0
SY		 1928 }
1929}
1930
68078d5c
DR		 1931/**
1932 * ufshcd_copy_query_response() - Copy the Query Response and the data
1933 * descriptor
1934 * @hba: per adapter instance
8aa29f19 1935 * @lrbp: pointer to local reference block
68078d5c
DR		 1936 */
1937static
c6d4a831 1938int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
68078d5c
DR		 1939{
1940 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1941
68078d5c 1942 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
68078d5c 1943
68078d5c 1944 /* Get the descriptor */
1c90836f
AA		 1945 if (hba->dev_cmd.query.descriptor &&
1946 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
d44a5f98 1947 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
68078d5c 1948 GENERAL_UPIU_REQUEST_SIZE;
c6d4a831
DR		 1949 u16 resp_len;
1950 u16 buf_len;
68078d5c
DR		 1951
1952 /* data segment length */
c6d4a831 1953 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
68078d5c 1954 MASK_QUERY_DATA_SEG_LEN;
ea2aab24
SRT		 1955 buf_len = be16_to_cpu(
1956 hba->dev_cmd.query.request.upiu_req.length);
c6d4a831
DR		 1957 if (likely(buf_len >= resp_len)) {
1958 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
1959 } else {
1960 dev_warn(hba->dev,
3d4881d1
BH		 1961 "%s: rsp size %d is bigger than buffer size %d",
1962 __func__, resp_len, buf_len);
c6d4a831
DR		 1963 return -EINVAL;
1964 }
68078d5c 1965 }
c6d4a831
DR		 1966
1967 return 0;
68078d5c
DR		 1968}
1969
7a3e97b0
SY		 1970/**
1971 * ufshcd_hba_capabilities - Read controller capabilities
1972 * @hba: per adapter instance
1973 */
1974static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
1975{
b873a275 1976 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
7a3e97b0
SY		 1977
1978 /* nutrs and nutmrs are 0 based values */
1979 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
1980 hba->nutmrs =
1981 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
1982}
1983
1984/**
6ccf44fe
SJ		 1985 * ufshcd_ready_for_uic_cmd - Check if controller is ready
1986 * to accept UIC commands
7a3e97b0 1987 * @hba: per adapter instance
6ccf44fe
SJ		 1988 * Return true on success, else false
1989 */
1990static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
1991{
1992 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
1993 return true;
1994 else
1995 return false;
1996}
1997
53b3d9c3
SJ		 1998/**
1999 * ufshcd_get_upmcrs - Get the power mode change request status
2000 * @hba: Pointer to adapter instance
2001 *
2002 * This function gets the UPMCRS field of HCS register
2003 * Returns value of UPMCRS field
2004 */
2005static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2006{
2007 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2008}
2009
6ccf44fe
SJ		 2010/**
2011 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
2012 * @hba: per adapter instance
2013 * @uic_cmd: UIC command
2014 *
2015 * Mutex must be held.
7a3e97b0
SY		 2016 */
2017static inline void
6ccf44fe 2018ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
7a3e97b0 2019{
6ccf44fe
SJ		 2020 WARN_ON(hba->active_uic_cmd);
2021
2022 hba->active_uic_cmd = uic_cmd;
2023
7a3e97b0 2024 /* Write Args */
6ccf44fe
SJ		 2025 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2026 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2027 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
7a3e97b0
SY		 2028
2029 /* Write UIC Cmd */
6ccf44fe 2030 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
b873a275 2031 REG_UIC_COMMAND);
7a3e97b0
SY		 2032}
2033
6ccf44fe
SJ		 2034/**
2035 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2036 * @hba: per adapter instance
8aa29f19 2037 * @uic_cmd: UIC command
6ccf44fe
SJ		 2038 *
2039 * Must be called with mutex held.
2040 * Returns 0 only if success.
2041 */
2042static int
2043ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2044{
2045 int ret;
2046 unsigned long flags;
2047
2048 if (wait_for_completion_timeout(&uic_cmd->done,
2049 msecs_to_jiffies(UIC_CMD_TIMEOUT)))
2050 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2051 else
2052 ret = -ETIMEDOUT;
2053
2054 spin_lock_irqsave(hba->host->host_lock, flags);
2055 hba->active_uic_cmd = NULL;
2056 spin_unlock_irqrestore(hba->host->host_lock, flags);
2057
2058 return ret;
2059}
2060
2061/**
2062 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2063 * @hba: per adapter instance
2064 * @uic_cmd: UIC command
d75f7fe4 2065 * @completion: initialize the completion only if this is set to true
6ccf44fe
SJ		 2066 *
2067 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
57d104c1 2068 * with mutex held and host_lock locked.
6ccf44fe
SJ		 2069 * Returns 0 only if success.
2070 */
2071static int
d75f7fe4
YG		 2072__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2073 bool completion)
6ccf44fe 2074{
6ccf44fe
SJ		 2075 if (!ufshcd_ready_for_uic_cmd(hba)) {
2076 dev_err(hba->dev,
2077 "Controller not ready to accept UIC commands\n");
2078 return -EIO;
2079 }
2080
d75f7fe4
YG		 2081 if (completion)
2082 init_completion(&uic_cmd->done);
6ccf44fe 2083
6ccf44fe 2084 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
6ccf44fe 2085
57d104c1 2086 return 0;
6ccf44fe
SJ		 2087}
2088
2089/**
2090 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2091 * @hba: per adapter instance
2092 * @uic_cmd: UIC command
2093 *
2094 * Returns 0 only if success.
2095 */
e77044c5 2096int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
6ccf44fe
SJ		 2097{
2098 int ret;
57d104c1 2099 unsigned long flags;
6ccf44fe 2100
1ab27c9c 2101 ufshcd_hold(hba, false);
6ccf44fe 2102 mutex_lock(&hba->uic_cmd_mutex);
cad2e03d
YG		 2103 ufshcd_add_delay_before_dme_cmd(hba);
2104
57d104c1 2105 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 2106 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
57d104c1
SJ		 2107 spin_unlock_irqrestore(hba->host->host_lock, flags);
2108 if (!ret)
2109 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2110
6ccf44fe
SJ		 2111 mutex_unlock(&hba->uic_cmd_mutex);
2112
1ab27c9c 2113 ufshcd_release(hba);
6ccf44fe
SJ		 2114 return ret;
2115}
2116
7a3e97b0
SY		 2117/**
2118 * ufshcd_map_sg - Map scatter-gather list to prdt
8aa29f19
BVA		 2119 * @hba: per adapter instance
2120 * @lrbp: pointer to local reference block
7a3e97b0
SY		 2121 *
2122 * Returns 0 in case of success, non-zero value in case of failure
2123 */
75b1cc4a 2124static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0
SY		 2125{
2126 struct ufshcd_sg_entry *prd_table;
2127 struct scatterlist *sg;
2128 struct scsi_cmnd *cmd;
2129 int sg_segments;
2130 int i;
2131
2132 cmd = lrbp->cmd;
2133 sg_segments = scsi_dma_map(cmd);
2134 if (sg_segments < 0)
2135 return sg_segments;
2136
2137 if (sg_segments) {
49200199
CH		 2138 lrbp->utr_descriptor_ptr->prd_table_length =
2139 cpu_to_le16((u16)sg_segments);
7a3e97b0
SY		 2140
2141 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2142
2143 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2144 prd_table[i].size =
2145 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2146 prd_table[i].base_addr =
2147 cpu_to_le32(lower_32_bits(sg->dma_address));
2148 prd_table[i].upper_addr =
2149 cpu_to_le32(upper_32_bits(sg->dma_address));
52ac95fe 2150 prd_table[i].reserved = 0;
7a3e97b0
SY		 2151 }
2152 } else {
2153 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2154 }
2155
2156 return 0;
2157}
2158
2159/**
2fbd009b 2160 * ufshcd_enable_intr - enable interrupts
7a3e97b0 2161 * @hba: per adapter instance
2fbd009b 2162 * @intrs: interrupt bits
7a3e97b0 2163 */
2fbd009b 2164static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
7a3e97b0 2165{
2fbd009b
SJ		 2166 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2167
2168 if (hba->ufs_version == UFSHCI_VERSION_10) {
2169 u32 rw;
2170 rw = set & INTERRUPT_MASK_RW_VER_10;
2171 set = rw | ((set ^ intrs) & intrs);
2172 } else {
2173 set |= intrs;
2174 }
2175
2176 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2177}
2178
2179/**
2180 * ufshcd_disable_intr - disable interrupts
2181 * @hba: per adapter instance
2182 * @intrs: interrupt bits
2183 */
2184static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2185{
2186 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2187
2188 if (hba->ufs_version == UFSHCI_VERSION_10) {
2189 u32 rw;
2190 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2191 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2192 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2193
2194 } else {
2195 set &= ~intrs;
7a3e97b0 2196 }
2fbd009b
SJ		 2197
2198 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
7a3e97b0
SY		 2199}
2200
5a0b0cb9
SRT		 2201/**
2202 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2203 * descriptor according to request
2204 * @lrbp: pointer to local reference block
2205 * @upiu_flags: flags required in the header
2206 * @cmd_dir: requests data direction
2207 */
2208static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
300bb13f 2209 u32 *upiu_flags, enum dma_data_direction cmd_dir)
5a0b0cb9
SRT		 2210{
2211 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2212 u32 data_direction;
2213 u32 dword_0;
2214
2215 if (cmd_dir == DMA_FROM_DEVICE) {
2216 data_direction = UTP_DEVICE_TO_HOST;
2217 *upiu_flags = UPIU_CMD_FLAGS_READ;
2218 } else if (cmd_dir == DMA_TO_DEVICE) {
2219 data_direction = UTP_HOST_TO_DEVICE;
2220 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2221 } else {
2222 data_direction = UTP_NO_DATA_TRANSFER;
2223 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2224 }
2225
2226 dword_0 = data_direction | (lrbp->command_type
2227 << UPIU_COMMAND_TYPE_OFFSET);
2228 if (lrbp->intr_cmd)
2229 dword_0 |= UTP_REQ_DESC_INT_CMD;
2230
2231 /* Transfer request descriptor header fields */
2232 req_desc->header.dword_0 = cpu_to_le32(dword_0);
52ac95fe
YG		 2233 /* dword_1 is reserved, hence it is set to 0 */
2234 req_desc->header.dword_1 = 0;
5a0b0cb9
SRT		 2235 /*
2236 * assigning invalid value for command status. Controller
2237 * updates OCS on command completion, with the command
2238 * status
2239 */
2240 req_desc->header.dword_2 =
2241 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
52ac95fe
YG		 2242 /* dword_3 is reserved, hence it is set to 0 */
2243 req_desc->header.dword_3 = 0;
51047266
YG		 2244
2245 req_desc->prd_table_length = 0;
5a0b0cb9
SRT		 2246}
2247
2248/**
2249 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2250 * for scsi commands
8aa29f19
BVA		 2251 * @lrbp: local reference block pointer
2252 * @upiu_flags: flags
5a0b0cb9
SRT		 2253 */
2254static
2255void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
2256{
1b21b8f0 2257 struct scsi_cmnd *cmd = lrbp->cmd;
5a0b0cb9 2258 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
52ac95fe 2259 unsigned short cdb_len;
5a0b0cb9
SRT		 2260
2261 /* command descriptor fields */
2262 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2263 UPIU_TRANSACTION_COMMAND, upiu_flags,
2264 lrbp->lun, lrbp->task_tag);
2265 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2266 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2267
2268 /* Total EHS length and Data segment length will be zero */
2269 ucd_req_ptr->header.dword_2 = 0;
2270
1b21b8f0 2271 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
5a0b0cb9 2272
1b21b8f0 2273 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
a851b2bd 2274 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
1b21b8f0 2275 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
52ac95fe
YG		 2276
2277 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2278}
2279
68078d5c
DR		 2280/**
2281 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2282 * for query requsts
2283 * @hba: UFS hba
2284 * @lrbp: local reference block pointer
2285 * @upiu_flags: flags
2286 */
2287static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2288 struct ufshcd_lrb *lrbp, u32 upiu_flags)
2289{
2290 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2291 struct ufs_query *query = &hba->dev_cmd.query;
e8c8e82a 2292 u16 len = be16_to_cpu(query->request.upiu_req.length);
68078d5c
DR		 2293
2294 /* Query request header */
2295 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2296 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2297 lrbp->lun, lrbp->task_tag);
2298 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2299 0, query->request.query_func, 0, 0);
2300
6861285c
ZL		 2301 /* Data segment length only need for WRITE_DESC */
2302 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2303 ucd_req_ptr->header.dword_2 =
2304 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2305 else
2306 ucd_req_ptr->header.dword_2 = 0;
68078d5c
DR		 2307
2308 /* Copy the Query Request buffer as is */
2309 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2310 QUERY_OSF_SIZE);
68078d5c
DR		 2311
2312 /* Copy the Descriptor */
c6d4a831 2313 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
220d17a6 2314 memcpy(ucd_req_ptr + 1, query->descriptor, len);
c6d4a831 2315
51047266 2316 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
68078d5c
DR		 2317}
2318
5a0b0cb9
SRT		 2319static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2320{
2321 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2322
2323 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2324
2325 /* command descriptor fields */
2326 ucd_req_ptr->header.dword_0 =
2327 UPIU_HEADER_DWORD(
2328 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
51047266
YG		 2329 /* clear rest of the fields of basic header */
2330 ucd_req_ptr->header.dword_1 = 0;
2331 ucd_req_ptr->header.dword_2 = 0;
2332
2333 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2334}
2335
7a3e97b0 2336/**
300bb13f
JP		 2337 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2338 * for Device Management Purposes
8aa29f19
BVA		 2339 * @hba: per adapter instance
2340 * @lrbp: pointer to local reference block
7a3e97b0 2341 */
300bb13f 2342static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0 2343{
7a3e97b0 2344 u32 upiu_flags;
5a0b0cb9 2345 int ret = 0;
7a3e97b0 2346
83dc7e3d 2347 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2348 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2349 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
83dc7e3d 2350 else
2351 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2352
2353 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2354 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2355 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2356 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2357 ufshcd_prepare_utp_nop_upiu(lrbp);
2358 else
2359 ret = -EINVAL;
2360
2361 return ret;
2362}
2363
2364/**
2365 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2366 * for SCSI Purposes
8aa29f19
BVA		 2367 * @hba: per adapter instance
2368 * @lrbp: pointer to local reference block
300bb13f
JP		 2369 */
2370static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2371{
2372 u32 upiu_flags;
2373 int ret = 0;
2374
83dc7e3d 2375 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2376 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2377 lrbp->command_type = UTP_CMD_TYPE_SCSI;
83dc7e3d 2378 else
2379 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2380
2381 if (likely(lrbp->cmd)) {
2382 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2383 lrbp->cmd->sc_data_direction);
2384 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2385 } else {
2386 ret = -EINVAL;
2387 }
5a0b0cb9
SRT		 2388
2389 return ret;
7a3e97b0
SY		 2390}
2391
2a8fa600
SJ		 2392/**
2393 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
8aa29f19 2394 * @upiu_wlun_id: UPIU W-LUN id
2a8fa600
SJ		 2395 *
2396 * Returns SCSI W-LUN id
2397 */
2398static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2399{
2400 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2401}
2402
4d2b8d40
BVA		 2403static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2404{
2405 struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
2406 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2407 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2408 i * sizeof(struct utp_transfer_cmd_desc);
2409 u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2410 response_upiu);
2411 u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2412
2413 lrb->utr_descriptor_ptr = utrdlp + i;
2414 lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2415 i * sizeof(struct utp_transfer_req_desc);
2416 lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
2417 lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2418 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2419 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2420 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
2421 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2422}
2423
7a3e97b0
SY		 2424/**
2425 * ufshcd_queuecommand - main entry point for SCSI requests
8aa29f19 2426 * @host: SCSI host pointer
7a3e97b0 2427 * @cmd: command from SCSI Midlayer
7a3e97b0
SY		 2428 *
2429 * Returns 0 for success, non-zero in case of failure
2430 */
2431static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2432{
2433 struct ufshcd_lrb *lrbp;
2434 struct ufs_hba *hba;
2435 unsigned long flags;
2436 int tag;
2437 int err = 0;
2438
2439 hba = shost_priv(host);
2440
2441 tag = cmd->request->tag;
14497328
YG		 2442 if (!ufshcd_valid_tag(hba, tag)) {
2443 dev_err(hba->dev,
2444 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2445 __func__, tag, cmd, cmd->request);
2446 BUG();
2447 }
7a3e97b0 2448
a3cd5ec5
SJ		 2449 if (!down_read_trylock(&hba->clk_scaling_lock))
2450 return SCSI_MLQUEUE_HOST_BUSY;
2451
3441da7d
SRT		 2452 spin_lock_irqsave(hba->host->host_lock, flags);
2453 switch (hba->ufshcd_state) {
2454 case UFSHCD_STATE_OPERATIONAL:
2455 break;
141f8165 2456 case UFSHCD_STATE_EH_SCHEDULED:
3441da7d 2457 case UFSHCD_STATE_RESET:
7a3e97b0 2458 err = SCSI_MLQUEUE_HOST_BUSY;
3441da7d
SRT		 2459 goto out_unlock;
2460 case UFSHCD_STATE_ERROR:
2461 set_host_byte(cmd, DID_ERROR);
2462 cmd->scsi_done(cmd);
2463 goto out_unlock;
2464 default:
2465 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2466 __func__, hba->ufshcd_state);
2467 set_host_byte(cmd, DID_BAD_TARGET);
2468 cmd->scsi_done(cmd);
2469 goto out_unlock;
7a3e97b0 2470 }
53c12d0e
YG		 2471
2472 /* if error handling is in progress, don't issue commands */
2473 if (ufshcd_eh_in_progress(hba)) {
2474 set_host_byte(cmd, DID_ERROR);
2475 cmd->scsi_done(cmd);
2476 goto out_unlock;
2477 }
3441da7d 2478 spin_unlock_irqrestore(hba->host->host_lock, flags);
7a3e97b0 2479
7fabb77b
GB		 2480 hba->req_abort_count = 0;
2481
1ab27c9c
ST		 2482 err = ufshcd_hold(hba, true);
2483 if (err) {
2484 err = SCSI_MLQUEUE_HOST_BUSY;
1ab27c9c
ST		 2485 goto out;
2486 }
2487 WARN_ON(hba->clk_gating.state != CLKS_ON);
2488
7a3e97b0
SY		 2489 lrbp = &hba->lrb[tag];
2490
5a0b0cb9 2491 WARN_ON(lrbp->cmd);
7a3e97b0 2492 lrbp->cmd = cmd;
09a5a24f 2493 lrbp->sense_bufflen = UFS_SENSE_SIZE;
7a3e97b0
SY		 2494 lrbp->sense_buffer = cmd->sense_buffer;
2495 lrbp->task_tag = tag;
0ce147d4 2496 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
b852190e 2497 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
e0b299e3 2498 lrbp->req_abort_skip = false;
7a3e97b0 2499
300bb13f
JP		 2500 ufshcd_comp_scsi_upiu(hba, lrbp);
2501
75b1cc4a 2502 err = ufshcd_map_sg(hba, lrbp);
5a0b0cb9
SRT		 2503 if (err) {
2504 lrbp->cmd = NULL;
17c7d35f 2505 ufshcd_release(hba);
7a3e97b0 2506 goto out;
5a0b0cb9 2507 }
ad1a1b9c
GB		 2508 /* Make sure descriptors are ready before ringing the doorbell */
2509 wmb();
7a3e97b0
SY		 2510
2511 /* issue command to the controller */
2512 spin_lock_irqsave(hba->host->host_lock, flags);
5905d464 2513 ufshcd_vops_setup_xfer_req(hba, tag, true);
7a3e97b0 2514 ufshcd_send_command(hba, tag);
3441da7d 2515out_unlock:
7a3e97b0
SY		 2516 spin_unlock_irqrestore(hba->host->host_lock, flags);
2517out:
a3cd5ec5 2518 up_read(&hba->clk_scaling_lock);
7a3e97b0
SY		 2519 return err;
2520}
2521
5a0b0cb9
SRT		 2522static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2523 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2524{
2525 lrbp->cmd = NULL;
2526 lrbp->sense_bufflen = 0;
2527 lrbp->sense_buffer = NULL;
2528 lrbp->task_tag = tag;
2529 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
5a0b0cb9
SRT		 2530 lrbp->intr_cmd = true; /* No interrupt aggregation */
2531 hba->dev_cmd.type = cmd_type;
2532
300bb13f 2533 return ufshcd_comp_devman_upiu(hba, lrbp);
5a0b0cb9
SRT		 2534}
2535
2536static int
2537ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2538{
2539 int err = 0;
2540 unsigned long flags;
2541 u32 mask = 1 << tag;
2542
2543 /* clear outstanding transaction before retry */
2544 spin_lock_irqsave(hba->host->host_lock, flags);
2545 ufshcd_utrl_clear(hba, tag);
2546 spin_unlock_irqrestore(hba->host->host_lock, flags);
2547
2548 /*
2549 * wait for for h/w to clear corresponding bit in door-bell.
2550 * max. wait is 1 sec.
2551 */
2552 err = ufshcd_wait_for_register(hba,
2553 REG_UTP_TRANSFER_REQ_DOOR_BELL,
596585a2 2554 mask, ~mask, 1000, 1000, true);
5a0b0cb9
SRT		 2555
2556 return err;
2557}
2558
c6d4a831
DR		 2559static int
2560ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2561{
2562 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2563
2564 /* Get the UPIU response */
2565 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2566 UPIU_RSP_CODE_OFFSET;
2567 return query_res->response;
2568}
2569
5a0b0cb9
SRT		 2570/**
2571 * ufshcd_dev_cmd_completion() - handles device management command responses
2572 * @hba: per adapter instance
2573 * @lrbp: pointer to local reference block
2574 */
2575static int
2576ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2577{
2578 int resp;
2579 int err = 0;
2580
ff8e20c6 2581 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 2582 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2583
2584 switch (resp) {
2585 case UPIU_TRANSACTION_NOP_IN:
2586 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2587 err = -EINVAL;
2588 dev_err(hba->dev, "%s: unexpected response %x\n",
2589 __func__, resp);
2590 }
2591 break;
68078d5c 2592 case UPIU_TRANSACTION_QUERY_RSP:
c6d4a831
DR		 2593 err = ufshcd_check_query_response(hba, lrbp);
2594 if (!err)
2595 err = ufshcd_copy_query_response(hba, lrbp);
68078d5c 2596 break;
5a0b0cb9
SRT		 2597 case UPIU_TRANSACTION_REJECT_UPIU:
2598 /* TODO: handle Reject UPIU Response */
2599 err = -EPERM;
2600 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2601 __func__);
2602 break;
2603 default:
2604 err = -EINVAL;
2605 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2606 __func__, resp);
2607 break;
2608 }
2609
2610 return err;
2611}
2612
2613static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2614 struct ufshcd_lrb *lrbp, int max_timeout)
2615{
2616 int err = 0;
2617 unsigned long time_left;
2618 unsigned long flags;
2619
2620 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2621 msecs_to_jiffies(max_timeout));
2622
ad1a1b9c
GB		 2623 /* Make sure descriptors are ready before ringing the doorbell */
2624 wmb();
5a0b0cb9
SRT		 2625 spin_lock_irqsave(hba->host->host_lock, flags);
2626 hba->dev_cmd.complete = NULL;
2627 if (likely(time_left)) {
2628 err = ufshcd_get_tr_ocs(lrbp);
2629 if (!err)
2630 err = ufshcd_dev_cmd_completion(hba, lrbp);
2631 }
2632 spin_unlock_irqrestore(hba->host->host_lock, flags);
2633
2634 if (!time_left) {
2635 err = -ETIMEDOUT;
a48353f6
YG		 2636 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2637 __func__, lrbp->task_tag);
5a0b0cb9 2638 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
a48353f6 2639 /* successfully cleared the command, retry if needed */
5a0b0cb9 2640 err = -EAGAIN;
a48353f6
YG		 2641 /*
2642 * in case of an error, after clearing the doorbell,
2643 * we also need to clear the outstanding_request
2644 * field in hba
2645 */
2646 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
5a0b0cb9
SRT		 2647 }
2648
2649 return err;
2650}
2651
5a0b0cb9
SRT		 2652/**
2653 * ufshcd_exec_dev_cmd - API for sending device management requests
8aa29f19
BVA		 2654 * @hba: UFS hba
2655 * @cmd_type: specifies the type (NOP, Query...)
2656 * @timeout: time in seconds
5a0b0cb9 2657 *
68078d5c
DR		 2658 * NOTE: Since there is only one available tag for device management commands,
2659 * it is expected you hold the hba->dev_cmd.lock mutex.
5a0b0cb9
SRT		 2660 */
2661static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2662 enum dev_cmd_type cmd_type, int timeout)
2663{
7252a360
BVA		 2664 struct request_queue *q = hba->cmd_queue;
2665 struct request *req;
5a0b0cb9
SRT		 2666 struct ufshcd_lrb *lrbp;
2667 int err;
2668 int tag;
2669 struct completion wait;
2670 unsigned long flags;
2671
a3cd5ec5
SJ		 2672 down_read(&hba->clk_scaling_lock);
2673
5a0b0cb9
SRT		 2674 /*
2675 * Get free slot, sleep if slots are unavailable.
2676 * Even though we use wait_event() which sleeps indefinitely,
2677 * the maximum wait time is bounded by SCSI request timeout.
2678 */
7252a360 2679 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
bb14dd15
DC		 2680 if (IS_ERR(req)) {
2681 err = PTR_ERR(req);
2682 goto out_unlock;
2683 }
7252a360
BVA		 2684 tag = req->tag;
2685 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5a0b0cb9
SRT		 2686
2687 init_completion(&wait);
2688 lrbp = &hba->lrb[tag];
2689 WARN_ON(lrbp->cmd);
2690 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2691 if (unlikely(err))
2692 goto out_put_tag;
2693
2694 hba->dev_cmd.complete = &wait;
2695
6667e6d9 2696 ufshcd_add_query_upiu_trace(hba, tag, "query_send");
e3dfdc53
YG		 2697 /* Make sure descriptors are ready before ringing the doorbell */
2698 wmb();
5a0b0cb9 2699 spin_lock_irqsave(hba->host->host_lock, flags);
5905d464 2700 ufshcd_vops_setup_xfer_req(hba, tag, false);
5a0b0cb9
SRT		 2701 ufshcd_send_command(hba, tag);
2702 spin_unlock_irqrestore(hba->host->host_lock, flags);
2703
2704 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2705
6667e6d9
OS		 2706 ufshcd_add_query_upiu_trace(hba, tag,
2707 err ? "query_complete_err" : "query_complete");
2708
5a0b0cb9 2709out_put_tag:
7252a360 2710 blk_put_request(req);
bb14dd15 2711out_unlock:
a3cd5ec5 2712 up_read(&hba->clk_scaling_lock);
5a0b0cb9
SRT		 2713 return err;
2714}
2715
d44a5f98
DR		 2716/**
2717 * ufshcd_init_query() - init the query response and request parameters
2718 * @hba: per-adapter instance
2719 * @request: address of the request pointer to be initialized
2720 * @response: address of the response pointer to be initialized
2721 * @opcode: operation to perform
2722 * @idn: flag idn to access
2723 * @index: LU number to access
2724 * @selector: query/flag/descriptor further identification
2725 */
2726static inline void ufshcd_init_query(struct ufs_hba *hba,
2727 struct ufs_query_req **request, struct ufs_query_res **response,
2728 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2729{
2730 *request = &hba->dev_cmd.query.request;
2731 *response = &hba->dev_cmd.query.response;
2732 memset(*request, 0, sizeof(struct ufs_query_req));
2733 memset(*response, 0, sizeof(struct ufs_query_res));
2734 (*request)->upiu_req.opcode = opcode;
2735 (*request)->upiu_req.idn = idn;
2736 (*request)->upiu_req.index = index;
2737 (*request)->upiu_req.selector = selector;
2738}
2739
dc3c8d3a
YG		 2740static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2741 enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
2742{
2743 int ret;
2744 int retries;
2745
2746 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2747 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
2748 if (ret)
2749 dev_dbg(hba->dev,
2750 "%s: failed with error %d, retries %d\n",
2751 __func__, ret, retries);
2752 else
2753 break;
2754 }
2755
2756 if (ret)
2757 dev_err(hba->dev,
2758 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2759 __func__, opcode, idn, ret, retries);
2760 return ret;
2761}
2762
68078d5c
DR		 2763/**
2764 * ufshcd_query_flag() - API function for sending flag query requests
8aa29f19
BVA		 2765 * @hba: per-adapter instance
2766 * @opcode: flag query to perform
2767 * @idn: flag idn to access
2768 * @flag_res: the flag value after the query request completes
68078d5c
DR		 2769 *
2770 * Returns 0 for success, non-zero in case of failure
2771 */
dc3c8d3a 2772int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
68078d5c
DR		 2773 enum flag_idn idn, bool *flag_res)
2774{
d44a5f98
DR		 2775 struct ufs_query_req *request = NULL;
2776 struct ufs_query_res *response = NULL;
2777 int err, index = 0, selector = 0;
e5ad406c 2778 int timeout = QUERY_REQ_TIMEOUT;
68078d5c
DR		 2779
2780 BUG_ON(!hba);
2781
1ab27c9c 2782 ufshcd_hold(hba, false);
68078d5c 2783 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2784 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2785 selector);
68078d5c
DR		 2786
2787 switch (opcode) {
2788 case UPIU_QUERY_OPCODE_SET_FLAG:
2789 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2790 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2791 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2792 break;
2793 case UPIU_QUERY_OPCODE_READ_FLAG:
2794 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2795 if (!flag_res) {
2796 /* No dummy reads */
2797 dev_err(hba->dev, "%s: Invalid argument for read request\n",
2798 __func__);
2799 err = -EINVAL;
2800 goto out_unlock;
2801 }
2802 break;
2803 default:
2804 dev_err(hba->dev,
2805 "%s: Expected query flag opcode but got = %d\n",
2806 __func__, opcode);
2807 err = -EINVAL;
2808 goto out_unlock;
2809 }
68078d5c 2810
e5ad406c 2811 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
68078d5c
DR		 2812
2813 if (err) {
2814 dev_err(hba->dev,
2815 "%s: Sending flag query for idn %d failed, err = %d\n",
2816 __func__, idn, err);
2817 goto out_unlock;
2818 }
2819
2820 if (flag_res)
e8c8e82a 2821 *flag_res = (be32_to_cpu(response->upiu_res.value) &
68078d5c
DR		 2822 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2823
2824out_unlock:
2825 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 2826 ufshcd_release(hba);
68078d5c
DR		 2827 return err;
2828}
2829
66ec6d59
SRT		 2830/**
2831 * ufshcd_query_attr - API function for sending attribute requests
8aa29f19
BVA		 2832 * @hba: per-adapter instance
2833 * @opcode: attribute opcode
2834 * @idn: attribute idn to access
2835 * @index: index field
2836 * @selector: selector field
2837 * @attr_val: the attribute value after the query request completes
66ec6d59
SRT		 2838 *
2839 * Returns 0 for success, non-zero in case of failure
2840*/
ec92b59c
SN		 2841int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2842 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
66ec6d59 2843{
d44a5f98
DR		 2844 struct ufs_query_req *request = NULL;
2845 struct ufs_query_res *response = NULL;
66ec6d59
SRT		 2846 int err;
2847
2848 BUG_ON(!hba);
2849
1ab27c9c 2850 ufshcd_hold(hba, false);
66ec6d59
SRT		 2851 if (!attr_val) {
2852 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2853 __func__, opcode);
2854 err = -EINVAL;
2855 goto out;
2856 }
2857
2858 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2859 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2860 selector);
66ec6d59
SRT		 2861
2862 switch (opcode) {
2863 case UPIU_QUERY_OPCODE_WRITE_ATTR:
2864 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
e8c8e82a 2865 request->upiu_req.value = cpu_to_be32(*attr_val);
66ec6d59
SRT		 2866 break;
2867 case UPIU_QUERY_OPCODE_READ_ATTR:
2868 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2869 break;
2870 default:
2871 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2872 __func__, opcode);
2873 err = -EINVAL;
2874 goto out_unlock;
2875 }
2876
d44a5f98 2877 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
66ec6d59
SRT		 2878
2879 if (err) {
4b761b58
YG		 2880 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2881 __func__, opcode, idn, index, err);
66ec6d59
SRT		 2882 goto out_unlock;
2883 }
2884
e8c8e82a 2885 *attr_val = be32_to_cpu(response->upiu_res.value);
66ec6d59
SRT		 2886
2887out_unlock:
2888 mutex_unlock(&hba->dev_cmd.lock);
2889out:
1ab27c9c 2890 ufshcd_release(hba);
66ec6d59
SRT		 2891 return err;
2892}
2893
5e86ae44
YG		 2894/**
2895 * ufshcd_query_attr_retry() - API function for sending query
2896 * attribute with retries
2897 * @hba: per-adapter instance
2898 * @opcode: attribute opcode
2899 * @idn: attribute idn to access
2900 * @index: index field
2901 * @selector: selector field
2902 * @attr_val: the attribute value after the query request
2903 * completes
2904 *
2905 * Returns 0 for success, non-zero in case of failure
2906*/
2907static int ufshcd_query_attr_retry(struct ufs_hba *hba,
2908 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
2909 u32 *attr_val)
2910{
2911 int ret = 0;
2912 u32 retries;
2913
68c9fcfd 2914 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
5e86ae44
YG		 2915 ret = ufshcd_query_attr(hba, opcode, idn, index,
2916 selector, attr_val);
2917 if (ret)
2918 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
2919 __func__, ret, retries);
2920 else
2921 break;
2922 }
2923
2924 if (ret)
2925 dev_err(hba->dev,
2926 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
2927 __func__, idn, ret, QUERY_REQ_RETRIES);
2928 return ret;
2929}
2930
a70e91b8 2931static int __ufshcd_query_descriptor(struct ufs_hba *hba,
d44a5f98
DR		 2932 enum query_opcode opcode, enum desc_idn idn, u8 index,
2933 u8 selector, u8 *desc_buf, int *buf_len)
2934{
2935 struct ufs_query_req *request = NULL;
2936 struct ufs_query_res *response = NULL;
2937 int err;
2938
2939 BUG_ON(!hba);
2940
1ab27c9c 2941 ufshcd_hold(hba, false);
d44a5f98
DR		 2942 if (!desc_buf) {
2943 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
2944 __func__, opcode);
2945 err = -EINVAL;
2946 goto out;
2947 }
2948
a4b0e8a4 2949 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
d44a5f98
DR		 2950 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
2951 __func__, *buf_len);
2952 err = -EINVAL;
2953 goto out;
2954 }
2955
2956 mutex_lock(&hba->dev_cmd.lock);
2957 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2958 selector);
2959 hba->dev_cmd.query.descriptor = desc_buf;
ea2aab24 2960 request->upiu_req.length = cpu_to_be16(*buf_len);
d44a5f98
DR		 2961
2962 switch (opcode) {
2963 case UPIU_QUERY_OPCODE_WRITE_DESC:
2964 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2965 break;
2966 case UPIU_QUERY_OPCODE_READ_DESC:
2967 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2968 break;
2969 default:
2970 dev_err(hba->dev,
2971 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
2972 __func__, opcode);
2973 err = -EINVAL;
2974 goto out_unlock;
2975 }
2976
2977 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2978
2979 if (err) {
4b761b58
YG		 2980 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2981 __func__, opcode, idn, index, err);
d44a5f98
DR		 2982 goto out_unlock;
2983 }
2984
ea2aab24 2985 *buf_len = be16_to_cpu(response->upiu_res.length);
d44a5f98
DR		 2986
2987out_unlock:
cfcbae38 2988 hba->dev_cmd.query.descriptor = NULL;
d44a5f98
DR		 2989 mutex_unlock(&hba->dev_cmd.lock);
2990out:
1ab27c9c 2991 ufshcd_release(hba);
d44a5f98
DR		 2992 return err;
2993}
2994
a70e91b8 2995/**
8aa29f19
BVA		 2996 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
2997 * @hba: per-adapter instance
2998 * @opcode: attribute opcode
2999 * @idn: attribute idn to access
3000 * @index: index field
3001 * @selector: selector field
3002 * @desc_buf: the buffer that contains the descriptor
3003 * @buf_len: length parameter passed to the device
a70e91b8
YG		 3004 *
3005 * Returns 0 for success, non-zero in case of failure.
3006 * The buf_len parameter will contain, on return, the length parameter
3007 * received on the response.
3008 */
2238d31c
SN		 3009int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3010 enum query_opcode opcode,
3011 enum desc_idn idn, u8 index,
3012 u8 selector,
3013 u8 *desc_buf, int *buf_len)
a70e91b8
YG		 3014{
3015 int err;
3016 int retries;
3017
3018 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3019 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3020 selector, desc_buf, buf_len);
3021 if (!err || err == -EINVAL)
3022 break;
3023 }
3024
3025 return err;
3026}
a70e91b8 3027
a4b0e8a4
PM		 3028/**
3029 * ufshcd_read_desc_length - read the specified descriptor length from header
3030 * @hba: Pointer to adapter instance
3031 * @desc_id: descriptor idn value
3032 * @desc_index: descriptor index
3033 * @desc_length: pointer to variable to read the length of descriptor
3034 *
3035 * Return 0 in case of success, non-zero otherwise
3036 */
3037static int ufshcd_read_desc_length(struct ufs_hba *hba,
3038 enum desc_idn desc_id,
3039 int desc_index,
3040 int *desc_length)
3041{
3042 int ret;
3043 u8 header[QUERY_DESC_HDR_SIZE];
3044 int header_len = QUERY_DESC_HDR_SIZE;
3045
3046 if (desc_id >= QUERY_DESC_IDN_MAX)
3047 return -EINVAL;
3048
3049 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3050 desc_id, desc_index, 0, header,
3051 &header_len);
3052
3053 if (ret) {
3054 dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
3055 __func__, desc_id);
3056 return ret;
3057 } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
3058 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
3059 __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
3060 desc_id);
3061 ret = -EINVAL;
3062 }
3063
3064 *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
3065 return ret;
3066
3067}
3068
3069/**
3070 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3071 * @hba: Pointer to adapter instance
3072 * @desc_id: descriptor idn value
3073 * @desc_len: mapped desc length (out)
3074 *
3075 * Return 0 in case of success, non-zero otherwise
3076 */
3077int ufshcd_map_desc_id_to_length(struct ufs_hba *hba,
3078 enum desc_idn desc_id, int *desc_len)
3079{
3080 switch (desc_id) {
3081 case QUERY_DESC_IDN_DEVICE:
3082 *desc_len = hba->desc_size.dev_desc;
3083 break;
3084 case QUERY_DESC_IDN_POWER:
3085 *desc_len = hba->desc_size.pwr_desc;
3086 break;
3087 case QUERY_DESC_IDN_GEOMETRY:
3088 *desc_len = hba->desc_size.geom_desc;
3089 break;
3090 case QUERY_DESC_IDN_CONFIGURATION:
3091 *desc_len = hba->desc_size.conf_desc;
3092 break;
3093 case QUERY_DESC_IDN_UNIT:
3094 *desc_len = hba->desc_size.unit_desc;
3095 break;
3096 case QUERY_DESC_IDN_INTERCONNECT:
3097 *desc_len = hba->desc_size.interc_desc;
3098 break;
3099 case QUERY_DESC_IDN_STRING:
3100 *desc_len = QUERY_DESC_MAX_SIZE;
3101 break;
c648c2d2
SN		 3102 case QUERY_DESC_IDN_HEALTH:
3103 *desc_len = hba->desc_size.hlth_desc;
3104 break;
a4b0e8a4
PM		 3105 case QUERY_DESC_IDN_RFU_0:
3106 case QUERY_DESC_IDN_RFU_1:
3107 *desc_len = 0;
3108 break;
3109 default:
3110 *desc_len = 0;
3111 return -EINVAL;
3112 }
3113 return 0;
3114}
3115EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3116
da461cec
SJ		 3117/**
3118 * ufshcd_read_desc_param - read the specified descriptor parameter
3119 * @hba: Pointer to adapter instance
3120 * @desc_id: descriptor idn value
3121 * @desc_index: descriptor index
3122 * @param_offset: offset of the parameter to read
3123 * @param_read_buf: pointer to buffer where parameter would be read
3124 * @param_size: sizeof(param_read_buf)
3125 *
3126 * Return 0 in case of success, non-zero otherwise
3127 */
45bced87
SN		 3128int ufshcd_read_desc_param(struct ufs_hba *hba,
3129 enum desc_idn desc_id,
3130 int desc_index,
3131 u8 param_offset,
3132 u8 *param_read_buf,
3133 u8 param_size)
da461cec
SJ		 3134{
3135 int ret;
3136 u8 *desc_buf;
a4b0e8a4 3137 int buff_len;
da461cec
SJ		 3138 bool is_kmalloc = true;
3139
a4b0e8a4
PM		 3140 /* Safety check */
3141 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
da461cec
SJ		 3142 return -EINVAL;
3143
a4b0e8a4
PM		 3144 /* Get the max length of descriptor from structure filled up at probe
3145 * time.
3146 */
3147 ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
da461cec 3148
a4b0e8a4
PM		 3149 /* Sanity checks */
3150 if (ret || !buff_len) {
3151 dev_err(hba->dev, "%s: Failed to get full descriptor length",
3152 __func__);
3153 return ret;
3154 }
3155
3156 /* Check whether we need temp memory */
3157 if (param_offset != 0 || param_size < buff_len) {
da461cec
SJ		 3158 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3159 if (!desc_buf)
3160 return -ENOMEM;
a4b0e8a4
PM		 3161 } else {
3162 desc_buf = param_read_buf;
3163 is_kmalloc = false;
da461cec
SJ		 3164 }
3165
a4b0e8a4 3166 /* Request for full descriptor */
a70e91b8 3167 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
a4b0e8a4
PM		 3168 desc_id, desc_index, 0,
3169 desc_buf, &buff_len);
da461cec 3170
bde44bb6
SJ		 3171 if (ret) {
3172 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3173 __func__, desc_id, desc_index, param_offset, ret);
da461cec
SJ		 3174 goto out;
3175 }
3176
bde44bb6
SJ		 3177 /* Sanity check */
3178 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3179 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3180 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3181 ret = -EINVAL;
3182 goto out;
3183 }
3184
a4b0e8a4
PM		 3185 /* Check wherher we will not copy more data, than available */
3186 if (is_kmalloc && param_size > buff_len)
3187 param_size = buff_len;
bde44bb6 3188
da461cec
SJ		 3189 if (is_kmalloc)
3190 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3191out:
3192 if (is_kmalloc)
3193 kfree(desc_buf);
3194 return ret;
3195}
3196
3197static inline int ufshcd_read_desc(struct ufs_hba *hba,
3198 enum desc_idn desc_id,
3199 int desc_index,
4b828fe1 3200 void *buf,
da461cec
SJ		 3201 u32 size)
3202{
3203 return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
3204}
3205
b573d484 3206
4b828fe1
TW		 3207/**
3208 * struct uc_string_id - unicode string
3209 *
3210 * @len: size of this descriptor inclusive
3211 * @type: descriptor type
3212 * @uc: unicode string character
3213 */
3214struct uc_string_id {
3215 u8 len;
3216 u8 type;
3217 wchar_t uc[0];
3218} __packed;
3219
3220/* replace non-printable or non-ASCII characters with spaces */
3221static inline char ufshcd_remove_non_printable(u8 ch)
3222{
3223 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3224}
3225
b573d484
YG		 3226/**
3227 * ufshcd_read_string_desc - read string descriptor
3228 * @hba: pointer to adapter instance
3229 * @desc_index: descriptor index
4b828fe1
TW		 3230 * @buf: pointer to buffer where descriptor would be read,
3231 * the caller should free the memory.
b573d484 3232 * @ascii: if true convert from unicode to ascii characters
4b828fe1 3233 * null terminated string.
b573d484 3234 *
4b828fe1
TW		 3235 * Return:
3236 * * string size on success.
3237 * * -ENOMEM: on allocation failure
3238 * * -EINVAL: on a wrong parameter
b573d484 3239 */
4b828fe1
TW		 3240int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3241 u8 **buf, bool ascii)
b573d484 3242{
4b828fe1
TW		 3243 struct uc_string_id *uc_str;
3244 u8 *str;
3245 int ret;
b573d484 3246
4b828fe1
TW		 3247 if (!buf)
3248 return -EINVAL;
b573d484 3249
4b828fe1
TW		 3250 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3251 if (!uc_str)
3252 return -ENOMEM;
b573d484 3253
4b828fe1
TW		 3254 ret = ufshcd_read_desc(hba, QUERY_DESC_IDN_STRING,
3255 desc_index, uc_str,
3256 QUERY_DESC_MAX_SIZE);
3257 if (ret < 0) {
3258 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3259 QUERY_REQ_RETRIES, ret);
3260 str = NULL;
3261 goto out;
3262 }
3263
3264 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3265 dev_dbg(hba->dev, "String Desc is of zero length\n");
3266 str = NULL;
3267 ret = 0;
b573d484
YG		 3268 goto out;
3269 }
3270
3271 if (ascii) {
4b828fe1 3272 ssize_t ascii_len;
b573d484 3273 int i;
b573d484 3274 /* remove header and divide by 2 to move from UTF16 to UTF8 */
4b828fe1
TW		 3275 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3276 str = kzalloc(ascii_len, GFP_KERNEL);
3277 if (!str) {
3278 ret = -ENOMEM;
fcbefc3b 3279 goto out;
b573d484
YG		 3280 }
3281
3282 /*
3283 * the descriptor contains string in UTF16 format
3284 * we need to convert to utf-8 so it can be displayed
3285 */
4b828fe1
TW		 3286 ret = utf16s_to_utf8s(uc_str->uc,
3287 uc_str->len - QUERY_DESC_HDR_SIZE,
3288 UTF16_BIG_ENDIAN, str, ascii_len);
b573d484
YG		 3289
3290 /* replace non-printable or non-ASCII characters with spaces */
4b828fe1
TW		 3291 for (i = 0; i < ret; i++)
3292 str[i] = ufshcd_remove_non_printable(str[i]);
b573d484 3293
4b828fe1
TW		 3294 str[ret++] = '\0';
3295
3296 } else {
5f57704d 3297 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
4b828fe1
TW		 3298 if (!str) {
3299 ret = -ENOMEM;
3300 goto out;
3301 }
4b828fe1 3302 ret = uc_str->len;
b573d484
YG		 3303 }
3304out:
4b828fe1
TW		 3305 *buf = str;
3306 kfree(uc_str);
3307 return ret;
b573d484 3308}
b573d484 3309
da461cec
SJ		 3310/**
3311 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3312 * @hba: Pointer to adapter instance
3313 * @lun: lun id
3314 * @param_offset: offset of the parameter to read
3315 * @param_read_buf: pointer to buffer where parameter would be read
3316 * @param_size: sizeof(param_read_buf)
3317 *
3318 * Return 0 in case of success, non-zero otherwise
3319 */
3320static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3321 int lun,
3322 enum unit_desc_param param_offset,
3323 u8 *param_read_buf,
3324 u32 param_size)
3325{
3326 /*
3327 * Unit descriptors are only available for general purpose LUs (LUN id
3328 * from 0 to 7) and RPMB Well known LU.
3329 */
1baa8011 3330 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun))
da461cec
SJ		 3331 return -EOPNOTSUPP;
3332
3333 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3334 param_offset, param_read_buf, param_size);
3335}
3336
09f17791
CG		 3337static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3338{
3339 int err = 0;
3340 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3341
3342 if (hba->dev_info.wspecversion >= 0x300) {
3343 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3344 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3345 &gating_wait);
3346 if (err)
3347 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3348 err, gating_wait);
3349
3350 if (gating_wait == 0) {
3351 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3352 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3353 gating_wait);
3354 }
3355
3356 hba->dev_info.clk_gating_wait_us = gating_wait;
3357 }
3358
3359 return err;
3360}
3361
7a3e97b0
SY		 3362/**
3363 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3364 * @hba: per adapter instance
3365 *
3366 * 1. Allocate DMA memory for Command Descriptor array
3367 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3368 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3369 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3370 * (UTMRDL)
3371 * 4. Allocate memory for local reference block(lrb).
3372 *
3373 * Returns 0 for success, non-zero in case of failure
3374 */
3375static int ufshcd_memory_alloc(struct ufs_hba *hba)
3376{
3377 size_t utmrdl_size, utrdl_size, ucdl_size;
3378
3379 /* Allocate memory for UTP command descriptors */
3380 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2953f850
SJ		 3381 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3382 ucdl_size,
3383 &hba->ucdl_dma_addr,
3384 GFP_KERNEL);
7a3e97b0
SY		 3385
3386 /*
3387 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3388 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3389 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3390 * be aligned to 128 bytes as well
3391 */
3392 if (!hba->ucdl_base_addr ||
3393 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3394 dev_err(hba->dev,
7a3e97b0
SY		 3395 "Command Descriptor Memory allocation failed\n");
3396 goto out;
3397 }
3398
3399 /*
3400 * Allocate memory for UTP Transfer descriptors
3401 * UFSHCI requires 1024 byte alignment of UTRD
3402 */
3403 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2953f850
SJ		 3404 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3405 utrdl_size,
3406 &hba->utrdl_dma_addr,
3407 GFP_KERNEL);
7a3e97b0
SY		 3408 if (!hba->utrdl_base_addr ||
3409 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3410 dev_err(hba->dev,
7a3e97b0
SY		 3411 "Transfer Descriptor Memory allocation failed\n");
3412 goto out;
3413 }
3414
3415 /*
3416 * Allocate memory for UTP Task Management descriptors
3417 * UFSHCI requires 1024 byte alignment of UTMRD
3418 */
3419 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2953f850
SJ		 3420 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3421 utmrdl_size,
3422 &hba->utmrdl_dma_addr,
3423 GFP_KERNEL);
7a3e97b0
SY		 3424 if (!hba->utmrdl_base_addr ||
3425 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3426 dev_err(hba->dev,
7a3e97b0
SY		 3427 "Task Management Descriptor Memory allocation failed\n");
3428 goto out;
3429 }
3430
3431 /* Allocate memory for local reference block */
a86854d0
KC		 3432 hba->lrb = devm_kcalloc(hba->dev,
3433 hba->nutrs, sizeof(struct ufshcd_lrb),
2953f850 3434 GFP_KERNEL);
7a3e97b0 3435 if (!hba->lrb) {
3b1d0580 3436 dev_err(hba->dev, "LRB Memory allocation failed\n");
7a3e97b0
SY		 3437 goto out;
3438 }
3439 return 0;
3440out:
7a3e97b0
SY		 3441 return -ENOMEM;
3442}
3443
3444/**
3445 * ufshcd_host_memory_configure - configure local reference block with
3446 * memory offsets
3447 * @hba: per adapter instance
3448 *
3449 * Configure Host memory space
3450 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3451 * address.
3452 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3453 * and PRDT offset.
3454 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3455 * into local reference block.
3456 */
3457static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3458{
7a3e97b0
SY		 3459 struct utp_transfer_req_desc *utrdlp;
3460 dma_addr_t cmd_desc_dma_addr;
3461 dma_addr_t cmd_desc_element_addr;
3462 u16 response_offset;
3463 u16 prdt_offset;
3464 int cmd_desc_size;
3465 int i;
3466
3467 utrdlp = hba->utrdl_base_addr;
7a3e97b0
SY		 3468
3469 response_offset =
3470 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3471 prdt_offset =
3472 offsetof(struct utp_transfer_cmd_desc, prd_table);
3473
3474 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3475 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3476
3477 for (i = 0; i < hba->nutrs; i++) {
3478 /* Configure UTRD with command descriptor base address */
3479 cmd_desc_element_addr =
3480 (cmd_desc_dma_addr + (cmd_desc_size * i));
3481 utrdlp[i].command_desc_base_addr_lo =
3482 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3483 utrdlp[i].command_desc_base_addr_hi =
3484 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3485
3486 /* Response upiu and prdt offset should be in double words */
49200199
CH		 3487 utrdlp[i].response_upiu_offset =
3488 cpu_to_le16(response_offset >> 2);
3489 utrdlp[i].prd_table_offset = cpu_to_le16(prdt_offset >> 2);
3490 utrdlp[i].response_upiu_length =
3491 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
7a3e97b0 3492
4d2b8d40 3493 ufshcd_init_lrb(hba, &hba->lrb[i], i);
7a3e97b0
SY		 3494 }
3495}
3496
3497/**
3498 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3499 * @hba: per adapter instance
3500 *
3501 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3502 * in order to initialize the Unipro link startup procedure.
3503 * Once the Unipro links are up, the device connected to the controller
3504 * is detected.
3505 *
3506 * Returns 0 on success, non-zero value on failure
3507 */
3508static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3509{
6ccf44fe
SJ		 3510 struct uic_command uic_cmd = {0};
3511 int ret;
7a3e97b0 3512
6ccf44fe 3513 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
7a3e97b0 3514
6ccf44fe
SJ		 3515 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3516 if (ret)
ff8e20c6 3517 dev_dbg(hba->dev,
6ccf44fe
SJ		 3518 "dme-link-startup: error code %d\n", ret);
3519 return ret;
7a3e97b0
SY		 3520}
3521
cad2e03d
YG		 3522static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3523{
3524 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3525 unsigned long min_sleep_time_us;
3526
3527 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3528 return;
3529
3530 /*
3531 * last_dme_cmd_tstamp will be 0 only for 1st call to
3532 * this function
3533 */
3534 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3535 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3536 } else {
3537 unsigned long delta =
3538 (unsigned long) ktime_to_us(
3539 ktime_sub(ktime_get(),
3540 hba->last_dme_cmd_tstamp));
3541
3542 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3543 min_sleep_time_us =
3544 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3545 else
3546 return; /* no more delay required */
3547 }
3548
3549 /* allow sleep for extra 50us if needed */
3550 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3551}
3552
12b4fdb4
SJ		 3553/**
3554 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3555 * @hba: per adapter instance
3556 * @attr_sel: uic command argument1
3557 * @attr_set: attribute set type as uic command argument2
3558 * @mib_val: setting value as uic command argument3
3559 * @peer: indicate whether peer or local
3560 *
3561 * Returns 0 on success, non-zero value on failure
3562 */
3563int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3564 u8 attr_set, u32 mib_val, u8 peer)
3565{
3566 struct uic_command uic_cmd = {0};
3567 static const char *const action[] = {
3568 "dme-set",
3569 "dme-peer-set"
3570 };
3571 const char *set = action[!!peer];
3572 int ret;
64238fbd 3573 int retries = UFS_UIC_COMMAND_RETRIES;
12b4fdb4
SJ		 3574
3575 uic_cmd.command = peer ?
3576 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3577 uic_cmd.argument1 = attr_sel;
3578 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3579 uic_cmd.argument3 = mib_val;
3580
64238fbd
YG		 3581 do {
3582 /* for peer attributes we retry upon failure */
3583 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3584 if (ret)
3585 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3586 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3587 } while (ret && peer && --retries);
3588
f37e9f8c 3589 if (ret)
64238fbd 3590 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
f37e9f8c
YG		 3591 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3592 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4
SJ		 3593
3594 return ret;
3595}
3596EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3597
3598/**
3599 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3600 * @hba: per adapter instance
3601 * @attr_sel: uic command argument1
3602 * @mib_val: the value of the attribute as returned by the UIC command
3603 * @peer: indicate whether peer or local
3604 *
3605 * Returns 0 on success, non-zero value on failure
3606 */
3607int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3608 u32 *mib_val, u8 peer)
3609{
3610 struct uic_command uic_cmd = {0};
3611 static const char *const action[] = {
3612 "dme-get",
3613 "dme-peer-get"
3614 };
3615 const char *get = action[!!peer];
3616 int ret;
64238fbd 3617 int retries = UFS_UIC_COMMAND_RETRIES;
874237f7
YG		 3618 struct ufs_pa_layer_attr orig_pwr_info;
3619 struct ufs_pa_layer_attr temp_pwr_info;
3620 bool pwr_mode_change = false;
3621
3622 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3623 orig_pwr_info = hba->pwr_info;
3624 temp_pwr_info = orig_pwr_info;
3625
3626 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3627 orig_pwr_info.pwr_rx == FAST_MODE) {
3628 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3629 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3630 pwr_mode_change = true;
3631 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3632 orig_pwr_info.pwr_rx == SLOW_MODE) {
3633 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3634 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3635 pwr_mode_change = true;
3636 }
3637 if (pwr_mode_change) {
3638 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3639 if (ret)
3640 goto out;
3641 }
3642 }
12b4fdb4
SJ		 3643
3644 uic_cmd.command = peer ?
3645 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3646 uic_cmd.argument1 = attr_sel;
3647
64238fbd
YG		 3648 do {
3649 /* for peer attributes we retry upon failure */
3650 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3651 if (ret)
3652 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3653 get, UIC_GET_ATTR_ID(attr_sel), ret);
3654 } while (ret && peer && --retries);
3655
f37e9f8c 3656 if (ret)
64238fbd 3657 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
f37e9f8c
YG		 3658 get, UIC_GET_ATTR_ID(attr_sel),
3659 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4 3660
64238fbd 3661 if (mib_val && !ret)
12b4fdb4 3662 *mib_val = uic_cmd.argument3;
874237f7
YG		 3663
3664 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3665 && pwr_mode_change)
3666 ufshcd_change_power_mode(hba, &orig_pwr_info);
12b4fdb4
SJ		 3667out:
3668 return ret;
3669}
3670EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3671
53b3d9c3 3672/**
57d104c1
SJ		 3673 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3674 * state) and waits for it to take effect.
3675 *
53b3d9c3 3676 * @hba: per adapter instance
57d104c1
SJ		 3677 * @cmd: UIC command to execute
3678 *
3679 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3680 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3681 * and device UniPro link and hence it's final completion would be indicated by
3682 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3683 * addition to normal UIC command completion Status (UCCS). This function only
3684 * returns after the relevant status bits indicate the completion.
53b3d9c3
SJ		 3685 *
3686 * Returns 0 on success, non-zero value on failure
3687 */
57d104c1 3688static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
53b3d9c3 3689{
57d104c1 3690 struct completion uic_async_done;
53b3d9c3
SJ		 3691 unsigned long flags;
3692 u8 status;
3693 int ret;
d75f7fe4 3694 bool reenable_intr = false;
53b3d9c3 3695
53b3d9c3 3696 mutex_lock(&hba->uic_cmd_mutex);
57d104c1 3697 init_completion(&uic_async_done);
cad2e03d 3698 ufshcd_add_delay_before_dme_cmd(hba);
53b3d9c3
SJ		 3699
3700 spin_lock_irqsave(hba->host->host_lock, flags);
57d104c1 3701 hba->uic_async_done = &uic_async_done;
d75f7fe4
YG		 3702 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3703 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3704 /*
3705 * Make sure UIC command completion interrupt is disabled before
3706 * issuing UIC command.
3707 */
3708 wmb();
3709 reenable_intr = true;
57d104c1 3710 }
d75f7fe4
YG		 3711 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3712 spin_unlock_irqrestore(hba->host->host_lock, flags);
57d104c1
SJ		 3713 if (ret) {
3714 dev_err(hba->dev,
3715 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3716 cmd->command, cmd->argument3, ret);
53b3d9c3
SJ		 3717 goto out;
3718 }
3719
57d104c1 3720 if (!wait_for_completion_timeout(hba->uic_async_done,
53b3d9c3
SJ		 3721 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3722 dev_err(hba->dev,
57d104c1
SJ		 3723 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3724 cmd->command, cmd->argument3);
53b3d9c3
SJ		 3725 ret = -ETIMEDOUT;
3726 goto out;
3727 }
3728
3729 status = ufshcd_get_upmcrs(hba);
3730 if (status != PWR_LOCAL) {
3731 dev_err(hba->dev,
479da360 3732 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
57d104c1 3733 cmd->command, status);
53b3d9c3
SJ		 3734 ret = (status != PWR_OK) ? status : -1;
3735 }
3736out:
7942f7b5
VG		 3737 if (ret) {
3738 ufshcd_print_host_state(hba);
3739 ufshcd_print_pwr_info(hba);
3740 ufshcd_print_host_regs(hba);
3741 }
3742
53b3d9c3 3743 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 3744 hba->active_uic_cmd = NULL;
57d104c1 3745 hba->uic_async_done = NULL;
d75f7fe4
YG		 3746 if (reenable_intr)
3747 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
53b3d9c3
SJ		 3748 spin_unlock_irqrestore(hba->host->host_lock, flags);
3749 mutex_unlock(&hba->uic_cmd_mutex);
1ab27c9c 3750
53b3d9c3
SJ		 3751 return ret;
3752}
3753
57d104c1
SJ		 3754/**
3755 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3756 * using DME_SET primitives.
3757 * @hba: per adapter instance
3758 * @mode: powr mode value
3759 *
3760 * Returns 0 on success, non-zero value on failure
3761 */
3762static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
3763{
3764 struct uic_command uic_cmd = {0};
1ab27c9c 3765 int ret;
57d104c1 3766
c3a2f9ee
YG		 3767 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
3768 ret = ufshcd_dme_set(hba,
3769 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
3770 if (ret) {
3771 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
3772 __func__, ret);
3773 goto out;
3774 }
3775 }
3776
57d104c1
SJ		 3777 uic_cmd.command = UIC_CMD_DME_SET;
3778 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
3779 uic_cmd.argument3 = mode;
1ab27c9c
ST		 3780 ufshcd_hold(hba, false);
3781 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3782 ufshcd_release(hba);
57d104c1 3783
c3a2f9ee 3784out:
1ab27c9c 3785 return ret;
57d104c1
SJ		 3786}
3787
53c12d0e
YG		 3788static int ufshcd_link_recovery(struct ufs_hba *hba)
3789{
3790 int ret;
3791 unsigned long flags;
3792
3793 spin_lock_irqsave(hba->host->host_lock, flags);
3794 hba->ufshcd_state = UFSHCD_STATE_RESET;
3795 ufshcd_set_eh_in_progress(hba);
3796 spin_unlock_irqrestore(hba->host->host_lock, flags);
3797
ebdd1dfd
CG		 3798 /* Reset the attached device */
3799 ufshcd_vops_device_reset(hba);
3800
53c12d0e
YG		 3801 ret = ufshcd_host_reset_and_restore(hba);
3802
3803 spin_lock_irqsave(hba->host->host_lock, flags);
3804 if (ret)
3805 hba->ufshcd_state = UFSHCD_STATE_ERROR;
3806 ufshcd_clear_eh_in_progress(hba);
3807 spin_unlock_irqrestore(hba->host->host_lock, flags);
3808
3809 if (ret)
3810 dev_err(hba->dev, "%s: link recovery failed, err %d",
3811 __func__, ret);
3812
3813 return ret;
3814}
3815
87d0b4a6 3816static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
57d104c1 3817{
87d0b4a6 3818 int ret;
57d104c1 3819 struct uic_command uic_cmd = {0};
911a0771 3820 ktime_t start = ktime_get();
57d104c1 3821
ee32c909
KK		 3822 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
3823
57d104c1 3824 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
87d0b4a6 3825 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771
SJ		 3826 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
3827 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
87d0b4a6 3828
53c12d0e 3829 if (ret) {
6d303e4b
SJ		 3830 int err;
3831
87d0b4a6
YG		 3832 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
3833 __func__, ret);
3834
53c12d0e 3835 /*
6d303e4b
SJ		 3836 * If link recovery fails then return error code returned from
3837 * ufshcd_link_recovery().
3838 * If link recovery succeeds then return -EAGAIN to attempt
3839 * hibern8 enter retry again.
53c12d0e 3840 */
6d303e4b
SJ		 3841 err = ufshcd_link_recovery(hba);
3842 if (err) {
3843 dev_err(hba->dev, "%s: link recovery failed", __func__);
3844 ret = err;
3845 } else {
3846 ret = -EAGAIN;
3847 }
ee32c909
KK		 3848 } else
3849 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
3850 POST_CHANGE);
53c12d0e 3851
87d0b4a6
YG		 3852 return ret;
3853}
3854
3855static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
3856{
3857 int ret = 0, retries;
57d104c1 3858
87d0b4a6
YG		 3859 for (retries = UIC_HIBERN8_ENTER_RETRIES; retries > 0; retries--) {
3860 ret = __ufshcd_uic_hibern8_enter(hba);
6d303e4b 3861 if (!ret)
87d0b4a6
YG		 3862 goto out;
3863 }
3864out:
3865 return ret;
57d104c1
SJ		 3866}
3867
9d19bf7a 3868int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
57d104c1
SJ		 3869{
3870 struct uic_command uic_cmd = {0};
3871 int ret;
911a0771 3872 ktime_t start = ktime_get();
57d104c1 3873
ee32c909
KK		 3874 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
3875
57d104c1
SJ		 3876 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
3877 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771
SJ		 3878 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
3879 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3880
57d104c1 3881 if (ret) {
53c12d0e
YG		 3882 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
3883 __func__, ret);
3884 ret = ufshcd_link_recovery(hba);
ff8e20c6 3885 } else {
ee32c909
KK		 3886 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
3887 POST_CHANGE);
ff8e20c6
DR		 3888 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
3889 hba->ufs_stats.hibern8_exit_cnt++;
3890 }
57d104c1
SJ		 3891
3892 return ret;
3893}
9d19bf7a 3894EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
57d104c1 3895
ba7af5ec
SC		 3896void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
3897{
3898 unsigned long flags;
3899
3900 if (!(hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT))
3901 return;
3902
3903 spin_lock_irqsave(hba->host->host_lock, flags);
3904 if (hba->ahit == ahit)
3905 goto out_unlock;
3906 hba->ahit = ahit;
3907 if (!pm_runtime_suspended(hba->dev))
3908 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
3909out_unlock:
3910 spin_unlock_irqrestore(hba->host->host_lock, flags);
3911}
3912EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
3913
71d848b8 3914void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
ad448378
AH		 3915{
3916 unsigned long flags;
3917
ee5f1042 3918 if (!ufshcd_is_auto_hibern8_supported(hba) || !hba->ahit)
ad448378
AH		 3919 return;
3920
3921 spin_lock_irqsave(hba->host->host_lock, flags);
3922 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
3923 spin_unlock_irqrestore(hba->host->host_lock, flags);
3924}
3925
5064636c
YG		 3926 /**
3927 * ufshcd_init_pwr_info - setting the POR (power on reset)
3928 * values in hba power info
3929 * @hba: per-adapter instance
3930 */
3931static void ufshcd_init_pwr_info(struct ufs_hba *hba)
3932{
3933 hba->pwr_info.gear_rx = UFS_PWM_G1;
3934 hba->pwr_info.gear_tx = UFS_PWM_G1;
3935 hba->pwr_info.lane_rx = 1;
3936 hba->pwr_info.lane_tx = 1;
3937 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
3938 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
3939 hba->pwr_info.hs_rate = 0;
3940}
3941
d3e89bac 3942/**
7eb584db
DR		 3943 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3944 * @hba: per-adapter instance
d3e89bac 3945 */
7eb584db 3946static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
d3e89bac 3947{
7eb584db
DR		 3948 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
3949
3950 if (hba->max_pwr_info.is_valid)
3951 return 0;
3952
2349b533
SJ		 3953 pwr_info->pwr_tx = FAST_MODE;
3954 pwr_info->pwr_rx = FAST_MODE;
7eb584db 3955 pwr_info->hs_rate = PA_HS_MODE_B;
d3e89bac
SJ		 3956
3957 /* Get the connected lane count */
7eb584db
DR		 3958 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
3959 &pwr_info->lane_rx);
3960 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
3961 &pwr_info->lane_tx);
3962
3963 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
3964 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
3965 __func__,
3966 pwr_info->lane_rx,
3967 pwr_info->lane_tx);
3968 return -EINVAL;
3969 }
d3e89bac
SJ		 3970
3971 /*
3972 * First, get the maximum gears of HS speed.
3973 * If a zero value, it means there is no HSGEAR capability.
3974 * Then, get the maximum gears of PWM speed.
3975 */
7eb584db
DR		 3976 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
3977 if (!pwr_info->gear_rx) {
3978 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
3979 &pwr_info->gear_rx);
3980 if (!pwr_info->gear_rx) {
3981 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
3982 __func__, pwr_info->gear_rx);
3983 return -EINVAL;
3984 }
2349b533 3985 pwr_info->pwr_rx = SLOW_MODE;
d3e89bac
SJ		 3986 }
3987
7eb584db
DR		 3988 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
3989 &pwr_info->gear_tx);
3990 if (!pwr_info->gear_tx) {
d3e89bac 3991 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
7eb584db
DR		 3992 &pwr_info->gear_tx);
3993 if (!pwr_info->gear_tx) {
3994 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
3995 __func__, pwr_info->gear_tx);
3996 return -EINVAL;
3997 }
2349b533 3998 pwr_info->pwr_tx = SLOW_MODE;
7eb584db
DR		 3999 }
4000
4001 hba->max_pwr_info.is_valid = true;
4002 return 0;
4003}
4004
4005static int ufshcd_change_power_mode(struct ufs_hba *hba,
4006 struct ufs_pa_layer_attr *pwr_mode)
4007{
4008 int ret;
4009
4010 /* if already configured to the requested pwr_mode */
4011 if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4012 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4013 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4014 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4015 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4016 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4017 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4018 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4019 return 0;
d3e89bac
SJ		 4020 }
4021
4022 /*
4023 * Configure attributes for power mode change with below.
4024 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4025 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4026 * - PA_HSSERIES
4027 */
7eb584db
DR		 4028 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4029 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4030 pwr_mode->lane_rx);
4031 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4032 pwr_mode->pwr_rx == FAST_MODE)
d3e89bac 4033 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
7eb584db
DR		 4034 else
4035 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
d3e89bac 4036
7eb584db
DR		 4037 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4038 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4039 pwr_mode->lane_tx);
4040 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4041 pwr_mode->pwr_tx == FAST_MODE)
d3e89bac 4042 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
7eb584db
DR		 4043 else
4044 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
d3e89bac 4045
7eb584db
DR		 4046 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4047 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4048 pwr_mode->pwr_rx == FAST_MODE ||
4049 pwr_mode->pwr_tx == FAST_MODE)
4050 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4051 pwr_mode->hs_rate);
d3e89bac 4052
08342537
CG		 4053 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4054 DL_FC0ProtectionTimeOutVal_Default);
4055 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4056 DL_TC0ReplayTimeOutVal_Default);
4057 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4058 DL_AFC0ReqTimeOutVal_Default);
4059 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4060 DL_FC1ProtectionTimeOutVal_Default);
4061 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4062 DL_TC1ReplayTimeOutVal_Default);
4063 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4064 DL_AFC1ReqTimeOutVal_Default);
4065
4066 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4067 DL_FC0ProtectionTimeOutVal_Default);
4068 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4069 DL_TC0ReplayTimeOutVal_Default);
4070 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4071 DL_AFC0ReqTimeOutVal_Default);
4072
7eb584db
DR		 4073 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4074 | pwr_mode->pwr_tx);
4075
4076 if (ret) {
d3e89bac 4077 dev_err(hba->dev,
7eb584db
DR		 4078 "%s: power mode change failed %d\n", __func__, ret);
4079 } else {
0263bcd0
YG		 4080 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4081 pwr_mode);
7eb584db
DR		 4082
4083 memcpy(&hba->pwr_info, pwr_mode,
4084 sizeof(struct ufs_pa_layer_attr));
4085 }
4086
4087 return ret;
4088}
4089
4090/**
4091 * ufshcd_config_pwr_mode - configure a new power mode
4092 * @hba: per-adapter instance
4093 * @desired_pwr_mode: desired power configuration
4094 */
0d846e70 4095int ufshcd_config_pwr_mode(struct ufs_hba *hba,
7eb584db
DR		 4096 struct ufs_pa_layer_attr *desired_pwr_mode)
4097{
4098 struct ufs_pa_layer_attr final_params = { 0 };
4099 int ret;
4100
0263bcd0
YG		 4101 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4102 desired_pwr_mode, &final_params);
4103
4104 if (ret)
7eb584db
DR		 4105 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4106
4107 ret = ufshcd_change_power_mode(hba, &final_params);
a3cd5ec5
SJ		 4108 if (!ret)
4109 ufshcd_print_pwr_info(hba);
d3e89bac
SJ		 4110
4111 return ret;
4112}
0d846e70 4113EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
d3e89bac 4114
68078d5c
DR		 4115/**
4116 * ufshcd_complete_dev_init() - checks device readiness
8aa29f19 4117 * @hba: per-adapter instance
68078d5c
DR		 4118 *
4119 * Set fDeviceInit flag and poll until device toggles it.
4120 */
4121static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4122{
dc3c8d3a
YG		 4123 int i;
4124 int err;
68078d5c
DR		 4125 bool flag_res = 1;
4126
dc3c8d3a
YG		 4127 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4128 QUERY_FLAG_IDN_FDEVICEINIT, NULL);
68078d5c
DR		 4129 if (err) {
4130 dev_err(hba->dev,
4131 "%s setting fDeviceInit flag failed with error %d\n",
4132 __func__, err);
4133 goto out;
4134 }
4135
dc3c8d3a
YG		 4136 /* poll for max. 1000 iterations for fDeviceInit flag to clear */
4137 for (i = 0; i < 1000 && !err && flag_res; i++)
4138 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4139 QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
4140
68078d5c
DR		 4141 if (err)
4142 dev_err(hba->dev,
4143 "%s reading fDeviceInit flag failed with error %d\n",
4144 __func__, err);
4145 else if (flag_res)
4146 dev_err(hba->dev,
4147 "%s fDeviceInit was not cleared by the device\n",
4148 __func__);
4149
4150out:
4151 return err;
4152}
4153
7a3e97b0
SY		 4154/**
4155 * ufshcd_make_hba_operational - Make UFS controller operational
4156 * @hba: per adapter instance
4157 *
4158 * To bring UFS host controller to operational state,
5c0c28a8
SRT		 4159 * 1. Enable required interrupts
4160 * 2. Configure interrupt aggregation
897efe62 4161 * 3. Program UTRL and UTMRL base address
5c0c28a8 4162 * 4. Configure run-stop-registers
7a3e97b0
SY		 4163 *
4164 * Returns 0 on success, non-zero value on failure
4165 */
9d19bf7a 4166int ufshcd_make_hba_operational(struct ufs_hba *hba)
7a3e97b0
SY		 4167{
4168 int err = 0;
4169 u32 reg;
4170
6ccf44fe
SJ		 4171 /* Enable required interrupts */
4172 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4173
4174 /* Configure interrupt aggregation */
b852190e
YG		 4175 if (ufshcd_is_intr_aggr_allowed(hba))
4176 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4177 else
4178 ufshcd_disable_intr_aggr(hba);
6ccf44fe
SJ		 4179
4180 /* Configure UTRL and UTMRL base address registers */
4181 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4182 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4183 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4184 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4185 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4186 REG_UTP_TASK_REQ_LIST_BASE_L);
4187 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4188 REG_UTP_TASK_REQ_LIST_BASE_H);
4189
897efe62
YG		 4190 /*
4191 * Make sure base address and interrupt setup are updated before
4192 * enabling the run/stop registers below.
4193 */
4194 wmb();
4195
7a3e97b0
SY		 4196 /*
4197 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
7a3e97b0 4198 */
5c0c28a8 4199 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
7a3e97b0
SY		 4200 if (!(ufshcd_get_lists_status(reg))) {
4201 ufshcd_enable_run_stop_reg(hba);
4202 } else {
3b1d0580 4203 dev_err(hba->dev,
7a3e97b0
SY		 4204 "Host controller not ready to process requests");
4205 err = -EIO;
4206 goto out;
4207 }
4208
7a3e97b0
SY		 4209out:
4210 return err;
4211}
9d19bf7a 4212EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
7a3e97b0 4213
596585a2
YG		 4214/**
4215 * ufshcd_hba_stop - Send controller to reset state
4216 * @hba: per adapter instance
4217 * @can_sleep: perform sleep or just spin
4218 */
4219static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
4220{
4221 int err;
4222
4223 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
4224 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4225 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4226 10, 1, can_sleep);
4227 if (err)
4228 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4229}
4230
7a3e97b0 4231/**
49200199 4232 * ufshcd_hba_enable - initialize the controller
7a3e97b0
SY		 4233 * @hba: per adapter instance
4234 *
4235 * The controller resets itself and controller firmware initialization
4236 * sequence kicks off. When controller is ready it will set
4237 * the Host Controller Enable bit to 1.
4238 *
4239 * Returns 0 on success, non-zero value on failure
4240 */
49200199 4241int ufshcd_hba_enable(struct ufs_hba *hba)
7a3e97b0
SY		 4242{
4243 int retry;
4244
596585a2 4245 if (!ufshcd_is_hba_active(hba))
7a3e97b0 4246 /* change controller state to "reset state" */
596585a2 4247 ufshcd_hba_stop(hba, true);
7a3e97b0 4248
57d104c1
SJ		 4249 /* UniPro link is disabled at this point */
4250 ufshcd_set_link_off(hba);
4251
0263bcd0 4252 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
5c0c28a8 4253
7a3e97b0
SY		 4254 /* start controller initialization sequence */
4255 ufshcd_hba_start(hba);
4256
4257 /*
4258 * To initialize a UFS host controller HCE bit must be set to 1.
4259 * During initialization the HCE bit value changes from 1->0->1.
4260 * When the host controller completes initialization sequence
4261 * it sets the value of HCE bit to 1. The same HCE bit is read back
4262 * to check if the controller has completed initialization sequence.
4263 * So without this delay the value HCE = 1, set in the previous
4264 * instruction might be read back.
4265 * This delay can be changed based on the controller.
4266 */
b9dc8aca 4267 ufshcd_delay_us(hba->hba_enable_delay_us, 100);
7a3e97b0
SY		 4268
4269 /* wait for the host controller to complete initialization */
9fc305ef 4270 retry = 50;
7a3e97b0
SY		 4271 while (ufshcd_is_hba_active(hba)) {
4272 if (retry) {
4273 retry--;
4274 } else {
3b1d0580 4275 dev_err(hba->dev,
7a3e97b0
SY		 4276 "Controller enable failed\n");
4277 return -EIO;
4278 }
9fc305ef 4279 usleep_range(1000, 1100);
7a3e97b0 4280 }
5c0c28a8 4281
1d337ec2 4282 /* enable UIC related interrupts */
57d104c1 4283 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
1d337ec2 4284
0263bcd0 4285 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
5c0c28a8 4286
7a3e97b0
SY		 4287 return 0;
4288}
9d19bf7a
SC		 4289EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4290
7ca38cf3
YG		 4291static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4292{
ba0320fb 4293 int tx_lanes = 0, i, err = 0;
7ca38cf3
YG		 4294
4295 if (!peer)
4296 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4297 &tx_lanes);
4298 else
4299 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4300 &tx_lanes);
4301 for (i = 0; i < tx_lanes; i++) {
4302 if (!peer)
4303 err = ufshcd_dme_set(hba,
4304 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4305 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4306 0);
4307 else
4308 err = ufshcd_dme_peer_set(hba,
4309 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4310 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4311 0);
4312 if (err) {
4313 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4314 __func__, peer, i, err);
4315 break;
4316 }
4317 }
4318
4319 return err;
4320}
4321
4322static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4323{
4324 return ufshcd_disable_tx_lcc(hba, true);
4325}
4326
a5fe372d
SC		 4327void ufshcd_update_reg_hist(struct ufs_err_reg_hist *reg_hist,
4328 u32 reg)
8808b4e9
SC		 4329{
4330 reg_hist->reg[reg_hist->pos] = reg;
4331 reg_hist->tstamp[reg_hist->pos] = ktime_get();
4332 reg_hist->pos = (reg_hist->pos + 1) % UFS_ERR_REG_HIST_LENGTH;
4333}
a5fe372d 4334EXPORT_SYMBOL_GPL(ufshcd_update_reg_hist);
8808b4e9 4335
7a3e97b0 4336/**
6ccf44fe 4337 * ufshcd_link_startup - Initialize unipro link startup
7a3e97b0
SY		 4338 * @hba: per adapter instance
4339 *
6ccf44fe 4340 * Returns 0 for success, non-zero in case of failure
7a3e97b0 4341 */
6ccf44fe 4342static int ufshcd_link_startup(struct ufs_hba *hba)
7a3e97b0 4343{
6ccf44fe 4344 int ret;
1d337ec2 4345 int retries = DME_LINKSTARTUP_RETRIES;
7caf489b 4346 bool link_startup_again = false;
7a3e97b0 4347
7caf489b
SJ		 4348 /*
4349 * If UFS device isn't active then we will have to issue link startup
4350 * 2 times to make sure the device state move to active.
4351 */
4352 if (!ufshcd_is_ufs_dev_active(hba))
4353 link_startup_again = true;
7a3e97b0 4354
7caf489b 4355link_startup:
1d337ec2 4356 do {
0263bcd0 4357 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
6ccf44fe 4358
1d337ec2 4359 ret = ufshcd_dme_link_startup(hba);
5c0c28a8 4360
1d337ec2
SRT		 4361 /* check if device is detected by inter-connect layer */
4362 if (!ret && !ufshcd_is_device_present(hba)) {
8808b4e9
SC		 4363 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4364 0);
1d337ec2
SRT		 4365 dev_err(hba->dev, "%s: Device not present\n", __func__);
4366 ret = -ENXIO;
4367 goto out;
4368 }
6ccf44fe 4369
1d337ec2
SRT		 4370 /*
4371 * DME link lost indication is only received when link is up,
4372 * but we can't be sure if the link is up until link startup
4373 * succeeds. So reset the local Uni-Pro and try again.
4374 */
8808b4e9
SC		 4375 if (ret && ufshcd_hba_enable(hba)) {
4376 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4377 (u32)ret);
1d337ec2 4378 goto out;
8808b4e9 4379 }
1d337ec2
SRT		 4380 } while (ret && retries--);
4381
8808b4e9 4382 if (ret) {
1d337ec2 4383 /* failed to get the link up... retire */
8808b4e9
SC		 4384 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4385 (u32)ret);
5c0c28a8 4386 goto out;
8808b4e9 4387 }
5c0c28a8 4388
7caf489b
SJ		 4389 if (link_startup_again) {
4390 link_startup_again = false;
4391 retries = DME_LINKSTARTUP_RETRIES;
4392 goto link_startup;
4393 }
4394
d2aebb9b
SJ		 4395 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4396 ufshcd_init_pwr_info(hba);
4397 ufshcd_print_pwr_info(hba);
4398
7ca38cf3
YG		 4399 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4400 ret = ufshcd_disable_device_tx_lcc(hba);
4401 if (ret)
4402 goto out;
4403 }
4404
5c0c28a8 4405 /* Include any host controller configuration via UIC commands */
0263bcd0
YG		 4406 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4407 if (ret)
4408 goto out;
7a3e97b0 4409
5c0c28a8 4410 ret = ufshcd_make_hba_operational(hba);
6ccf44fe 4411out:
7942f7b5 4412 if (ret) {
6ccf44fe 4413 dev_err(hba->dev, "link startup failed %d\n", ret);
7942f7b5
VG		 4414 ufshcd_print_host_state(hba);
4415 ufshcd_print_pwr_info(hba);
4416 ufshcd_print_host_regs(hba);
4417 }
6ccf44fe 4418 return ret;
7a3e97b0
SY		 4419}
4420
5a0b0cb9
SRT		 4421/**
4422 * ufshcd_verify_dev_init() - Verify device initialization
4423 * @hba: per-adapter instance
4424 *
4425 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4426 * device Transport Protocol (UTP) layer is ready after a reset.
4427 * If the UTP layer at the device side is not initialized, it may
4428 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4429 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4430 */
4431static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4432{
4433 int err = 0;
4434 int retries;
4435
1ab27c9c 4436 ufshcd_hold(hba, false);
5a0b0cb9
SRT		 4437 mutex_lock(&hba->dev_cmd.lock);
4438 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4439 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4440 NOP_OUT_TIMEOUT);
4441
4442 if (!err || err == -ETIMEDOUT)
4443 break;
4444
4445 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4446 }
4447 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 4448 ufshcd_release(hba);
5a0b0cb9
SRT		 4449
4450 if (err)
4451 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4452 return err;
4453}
4454
0ce147d4
SJ		 4455/**
4456 * ufshcd_set_queue_depth - set lun queue depth
4457 * @sdev: pointer to SCSI device
4458 *
4459 * Read bLUQueueDepth value and activate scsi tagged command
4460 * queueing. For WLUN, queue depth is set to 1. For best-effort
4461 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4462 * value that host can queue.
4463 */
4464static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4465{
4466 int ret = 0;
4467 u8 lun_qdepth;
4468 struct ufs_hba *hba;
4469
4470 hba = shost_priv(sdev->host);
4471
4472 lun_qdepth = hba->nutrs;
dbd34a61
SM		 4473 ret = ufshcd_read_unit_desc_param(hba,
4474 ufshcd_scsi_to_upiu_lun(sdev->lun),
4475 UNIT_DESC_PARAM_LU_Q_DEPTH,
4476 &lun_qdepth,
4477 sizeof(lun_qdepth));
0ce147d4
SJ		 4478
4479 /* Some WLUN doesn't support unit descriptor */
4480 if (ret == -EOPNOTSUPP)
4481 lun_qdepth = 1;
4482 else if (!lun_qdepth)
4483 /* eventually, we can figure out the real queue depth */
4484 lun_qdepth = hba->nutrs;
4485 else
4486 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4487
4488 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4489 __func__, lun_qdepth);
db5ed4df 4490 scsi_change_queue_depth(sdev, lun_qdepth);
0ce147d4
SJ		 4491}
4492
57d104c1
SJ		 4493/*
4494 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4495 * @hba: per-adapter instance
4496 * @lun: UFS device lun id
4497 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4498 *
4499 * Returns 0 in case of success and b_lu_write_protect status would be returned
4500 * @b_lu_write_protect parameter.
4501 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4502 * Returns -EINVAL in case of invalid parameters passed to this function.
4503 */
4504static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4505 u8 lun,
4506 u8 *b_lu_write_protect)
4507{
4508 int ret;
4509
4510 if (!b_lu_write_protect)
4511 ret = -EINVAL;
4512 /*
4513 * According to UFS device spec, RPMB LU can't be write
4514 * protected so skip reading bLUWriteProtect parameter for
4515 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4516 */
1baa8011 4517 else if (lun >= hba->dev_info.max_lu_supported)
57d104c1
SJ		 4518 ret = -ENOTSUPP;
4519 else
4520 ret = ufshcd_read_unit_desc_param(hba,
4521 lun,
4522 UNIT_DESC_PARAM_LU_WR_PROTECT,
4523 b_lu_write_protect,
4524 sizeof(*b_lu_write_protect));
4525 return ret;
4526}
4527
4528/**
4529 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4530 * status
4531 * @hba: per-adapter instance
4532 * @sdev: pointer to SCSI device
4533 *
4534 */
4535static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4536 struct scsi_device *sdev)
4537{
4538 if (hba->dev_info.f_power_on_wp_en &&
4539 !hba->dev_info.is_lu_power_on_wp) {
4540 u8 b_lu_write_protect;
4541
4542 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4543 &b_lu_write_protect) &&
4544 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4545 hba->dev_info.is_lu_power_on_wp = true;
4546 }
4547}
4548
7a3e97b0
SY		 4549/**
4550 * ufshcd_slave_alloc - handle initial SCSI device configurations
4551 * @sdev: pointer to SCSI device
4552 *
4553 * Returns success
4554 */
4555static int ufshcd_slave_alloc(struct scsi_device *sdev)
4556{
4557 struct ufs_hba *hba;
4558
4559 hba = shost_priv(sdev->host);
7a3e97b0
SY		 4560
4561 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4562 sdev->use_10_for_ms = 1;
a3a76391
CG		 4563
4564 /* DBD field should be set to 1 in mode sense(10) */
4565 sdev->set_dbd_for_ms = 1;
7a3e97b0 4566
e8e7f271
SRT		 4567 /* allow SCSI layer to restart the device in case of errors */
4568 sdev->allow_restart = 1;
4264fd61 4569
b2a6c522
SRT		 4570 /* REPORT SUPPORTED OPERATION CODES is not supported */
4571 sdev->no_report_opcodes = 1;
4572
84af7e8b
SRT		 4573 /* WRITE_SAME command is not supported */
4574 sdev->no_write_same = 1;
e8e7f271 4575
0ce147d4 4576 ufshcd_set_queue_depth(sdev);
4264fd61 4577
57d104c1
SJ		 4578 ufshcd_get_lu_power_on_wp_status(hba, sdev);
4579
7a3e97b0
SY		 4580 return 0;
4581}
4582
4264fd61
SRT		 4583/**
4584 * ufshcd_change_queue_depth - change queue depth
4585 * @sdev: pointer to SCSI device
4586 * @depth: required depth to set
4264fd61 4587 *
db5ed4df 4588 * Change queue depth and make sure the max. limits are not crossed.
4264fd61 4589 */
db5ed4df 4590static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4264fd61
SRT		 4591{
4592 struct ufs_hba *hba = shost_priv(sdev->host);
4593
4594 if (depth > hba->nutrs)
4595 depth = hba->nutrs;
db5ed4df 4596 return scsi_change_queue_depth(sdev, depth);
4264fd61
SRT		 4597}
4598
eeda4749
AM		 4599/**
4600 * ufshcd_slave_configure - adjust SCSI device configurations
4601 * @sdev: pointer to SCSI device
4602 */
4603static int ufshcd_slave_configure(struct scsi_device *sdev)
4604{
49615ba1 4605 struct ufs_hba *hba = shost_priv(sdev->host);
eeda4749
AM		 4606 struct request_queue *q = sdev->request_queue;
4607
4608 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
49615ba1
SC		 4609
4610 if (ufshcd_is_rpm_autosuspend_allowed(hba))
4611 sdev->rpm_autosuspend = 1;
4612
eeda4749
AM		 4613 return 0;
4614}
4615
7a3e97b0
SY		 4616/**
4617 * ufshcd_slave_destroy - remove SCSI device configurations
4618 * @sdev: pointer to SCSI device
4619 */
4620static void ufshcd_slave_destroy(struct scsi_device *sdev)
4621{
4622 struct ufs_hba *hba;
4623
4624 hba = shost_priv(sdev->host);
0ce147d4 4625 /* Drop the reference as it won't be needed anymore */
7c48bfd0
AM		 4626 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4627 unsigned long flags;
4628
4629 spin_lock_irqsave(hba->host->host_lock, flags);
0ce147d4 4630 hba->sdev_ufs_device = NULL;
7c48bfd0
AM		 4631 spin_unlock_irqrestore(hba->host->host_lock, flags);
4632 }
7a3e97b0
SY		 4633}
4634
7a3e97b0
SY		 4635/**
4636 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
8aa29f19 4637 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4638 * @scsi_status: SCSI command status
4639 *
4640 * Returns value base on SCSI command status
4641 */
4642static inline int
4643ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4644{
4645 int result = 0;
4646
4647 switch (scsi_status) {
7a3e97b0 4648 case SAM_STAT_CHECK_CONDITION:
1c2623c5 4649 ufshcd_copy_sense_data(lrbp);
30eb2e4c 4650 /* fallthrough */
1c2623c5 4651 case SAM_STAT_GOOD:
7a3e97b0
SY		 4652 result |= DID_OK << 16 |
4653 COMMAND_COMPLETE << 8 |
1c2623c5 4654 scsi_status;
7a3e97b0
SY		 4655 break;
4656 case SAM_STAT_TASK_SET_FULL:
1c2623c5 4657 case SAM_STAT_BUSY:
7a3e97b0 4658 case SAM_STAT_TASK_ABORTED:
1c2623c5
SJ		 4659 ufshcd_copy_sense_data(lrbp);
4660 result |= scsi_status;
7a3e97b0
SY		 4661 break;
4662 default:
4663 result |= DID_ERROR << 16;
4664 break;
4665 } /* end of switch */
4666
4667 return result;
4668}
4669
4670/**
4671 * ufshcd_transfer_rsp_status - Get overall status of the response
4672 * @hba: per adapter instance
8aa29f19 4673 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4674 *
4675 * Returns result of the command to notify SCSI midlayer
4676 */
4677static inline int
4678ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
4679{
4680 int result = 0;
4681 int scsi_status;
4682 int ocs;
4683
4684 /* overall command status of utrd */
4685 ocs = ufshcd_get_tr_ocs(lrbp);
4686
4687 switch (ocs) {
4688 case OCS_SUCCESS:
5a0b0cb9 4689 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
ff8e20c6 4690 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 4691 switch (result) {
4692 case UPIU_TRANSACTION_RESPONSE:
4693 /*
4694 * get the response UPIU result to extract
4695 * the SCSI command status
4696 */
4697 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
4698
4699 /*
4700 * get the result based on SCSI status response
4701 * to notify the SCSI midlayer of the command status
4702 */
4703 scsi_status = result & MASK_SCSI_STATUS;
4704 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
66ec6d59 4705
f05ac2e5
YG		 4706 /*
4707 * Currently we are only supporting BKOPs exception
4708 * events hence we can ignore BKOPs exception event
4709 * during power management callbacks. BKOPs exception
4710 * event is not expected to be raised in runtime suspend
4711 * callback as it allows the urgent bkops.
4712 * During system suspend, we are anyway forcefully
4713 * disabling the bkops and if urgent bkops is needed
4714 * it will be enabled on system resume. Long term
4715 * solution could be to abort the system suspend if
4716 * UFS device needs urgent BKOPs.
4717 */
4718 if (!hba->pm_op_in_progress &&
2824ec9f
SL		 4719 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr) &&
4720 schedule_work(&hba->eeh_work)) {
4721 /*
4722 * Prevent suspend once eeh_work is scheduled
4723 * to avoid deadlock between ufshcd_suspend
4724 * and exception event handler.
4725 */
4726 pm_runtime_get_noresume(hba->dev);
4727 }
5a0b0cb9
SRT		 4728 break;
4729 case UPIU_TRANSACTION_REJECT_UPIU:
4730 /* TODO: handle Reject UPIU Response */
4731 result = DID_ERROR << 16;
3b1d0580 4732 dev_err(hba->dev,
5a0b0cb9
SRT		 4733 "Reject UPIU not fully implemented\n");
4734 break;
4735 default:
5a0b0cb9
SRT		 4736 dev_err(hba->dev,
4737 "Unexpected request response code = %x\n",
4738 result);
e0347d89 4739 result = DID_ERROR << 16;
7a3e97b0
SY		 4740 break;
4741 }
7a3e97b0
SY		 4742 break;
4743 case OCS_ABORTED:
4744 result |= DID_ABORT << 16;
4745 break;
e8e7f271
SRT		 4746 case OCS_INVALID_COMMAND_STATUS:
4747 result |= DID_REQUEUE << 16;
4748 break;
7a3e97b0
SY		 4749 case OCS_INVALID_CMD_TABLE_ATTR:
4750 case OCS_INVALID_PRDT_ATTR:
4751 case OCS_MISMATCH_DATA_BUF_SIZE:
4752 case OCS_MISMATCH_RESP_UPIU_SIZE:
4753 case OCS_PEER_COMM_FAILURE:
4754 case OCS_FATAL_ERROR:
4755 default:
4756 result |= DID_ERROR << 16;
3b1d0580 4757 dev_err(hba->dev,
ff8e20c6
DR		 4758 "OCS error from controller = %x for tag %d\n",
4759 ocs, lrbp->task_tag);
4760 ufshcd_print_host_regs(hba);
6ba65588 4761 ufshcd_print_host_state(hba);
7a3e97b0
SY		 4762 break;
4763 } /* end of switch */
4764
2df74b69 4765 if ((host_byte(result) != DID_OK) && !hba->silence_err_logs)
66cc820f 4766 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
7a3e97b0
SY		 4767 return result;
4768}
4769
6ccf44fe
SJ		 4770/**
4771 * ufshcd_uic_cmd_compl - handle completion of uic command
4772 * @hba: per adapter instance
53b3d9c3 4773 * @intr_status: interrupt status generated by the controller
9333d775
VG		 4774 *
4775 * Returns
4776 * IRQ_HANDLED - If interrupt is valid
4777 * IRQ_NONE - If invalid interrupt
6ccf44fe 4778 */
9333d775 4779static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
6ccf44fe 4780{
9333d775
VG		 4781 irqreturn_t retval = IRQ_NONE;
4782
53b3d9c3 4783 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
6ccf44fe
SJ		 4784 hba->active_uic_cmd->argument2 |=
4785 ufshcd_get_uic_cmd_result(hba);
12b4fdb4
SJ		 4786 hba->active_uic_cmd->argument3 =
4787 ufshcd_get_dme_attr_val(hba);
6ccf44fe 4788 complete(&hba->active_uic_cmd->done);
9333d775 4789 retval = IRQ_HANDLED;
6ccf44fe 4790 }
53b3d9c3 4791
9333d775 4792 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
57d104c1 4793 complete(hba->uic_async_done);
9333d775
VG		 4794 retval = IRQ_HANDLED;
4795 }
4796 return retval;
6ccf44fe
SJ		 4797}
4798
7a3e97b0 4799/**
9a47ec7c 4800 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
7a3e97b0 4801 * @hba: per adapter instance
9a47ec7c 4802 * @completed_reqs: requests to complete
7a3e97b0 4803 */
9a47ec7c
YG		 4804static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
4805 unsigned long completed_reqs)
7a3e97b0 4806{
5a0b0cb9
SRT		 4807 struct ufshcd_lrb *lrbp;
4808 struct scsi_cmnd *cmd;
7a3e97b0
SY		 4809 int result;
4810 int index;
e9d501b1 4811
e9d501b1
DR		 4812 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
4813 lrbp = &hba->lrb[index];
4814 cmd = lrbp->cmd;
4815 if (cmd) {
1a07f2d9 4816 ufshcd_add_command_trace(hba, index, "complete");
e9d501b1
DR		 4817 result = ufshcd_transfer_rsp_status(hba, lrbp);
4818 scsi_dma_unmap(cmd);
4819 cmd->result = result;
4820 /* Mark completed command as NULL in LRB */
4821 lrbp->cmd = NULL;
74a527a2 4822 lrbp->compl_time_stamp = ktime_get();
e9d501b1
DR		 4823 /* Do not touch lrbp after scsi done */
4824 cmd->scsi_done(cmd);
1ab27c9c 4825 __ufshcd_release(hba);
300bb13f
JP		 4826 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
4827 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
74a527a2 4828 lrbp->compl_time_stamp = ktime_get();
1a07f2d9
LS		 4829 if (hba->dev_cmd.complete) {
4830 ufshcd_add_command_trace(hba, index,
4831 "dev_complete");
e9d501b1 4832 complete(hba->dev_cmd.complete);
1a07f2d9 4833 }
e9d501b1 4834 }
401f1e44
SJ		 4835 if (ufshcd_is_clkscaling_supported(hba))
4836 hba->clk_scaling.active_reqs--;
e9d501b1 4837 }
7a3e97b0
SY		 4838
4839 /* clear corresponding bits of completed commands */
4840 hba->outstanding_reqs ^= completed_reqs;
4841
856b3483 4842 ufshcd_clk_scaling_update_busy(hba);
7a3e97b0
SY		 4843}
4844
9a47ec7c
YG		 4845/**
4846 * ufshcd_transfer_req_compl - handle SCSI and query command completion
4847 * @hba: per adapter instance
9333d775
VG		 4848 *
4849 * Returns
4850 * IRQ_HANDLED - If interrupt is valid
4851 * IRQ_NONE - If invalid interrupt
9a47ec7c 4852 */
9333d775 4853static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
9a47ec7c
YG		 4854{
4855 unsigned long completed_reqs;
4856 u32 tr_doorbell;
4857
4858 /* Resetting interrupt aggregation counters first and reading the
4859 * DOOR_BELL afterward allows us to handle all the completed requests.
4860 * In order to prevent other interrupts starvation the DB is read once
4861 * after reset. The down side of this solution is the possibility of
4862 * false interrupt if device completes another request after resetting
4863 * aggregation and before reading the DB.
4864 */
49200199 4865 if (ufshcd_is_intr_aggr_allowed(hba))
9a47ec7c
YG		 4866 ufshcd_reset_intr_aggr(hba);
4867
4868 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4869 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
4870
9333d775
VG		 4871 if (completed_reqs) {
4872 __ufshcd_transfer_req_compl(hba, completed_reqs);
4873 return IRQ_HANDLED;
4874 } else {
4875 return IRQ_NONE;
4876 }
9a47ec7c
YG		 4877}
4878
66ec6d59
SRT		 4879/**
4880 * ufshcd_disable_ee - disable exception event
4881 * @hba: per-adapter instance
4882 * @mask: exception event to disable
4883 *
4884 * Disables exception event in the device so that the EVENT_ALERT
4885 * bit is not set.
4886 *
4887 * Returns zero on success, non-zero error value on failure.
4888 */
4889static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
4890{
4891 int err = 0;
4892 u32 val;
4893
4894 if (!(hba->ee_ctrl_mask & mask))
4895 goto out;
4896
4897 val = hba->ee_ctrl_mask & ~mask;
d7e2ddd5 4898 val &= MASK_EE_STATUS;
5e86ae44 4899 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 4900 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4901 if (!err)
4902 hba->ee_ctrl_mask &= ~mask;
4903out:
4904 return err;
4905}
4906
4907/**
4908 * ufshcd_enable_ee - enable exception event
4909 * @hba: per-adapter instance
4910 * @mask: exception event to enable
4911 *
4912 * Enable corresponding exception event in the device to allow
4913 * device to alert host in critical scenarios.
4914 *
4915 * Returns zero on success, non-zero error value on failure.
4916 */
4917static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
4918{
4919 int err = 0;
4920 u32 val;
4921
4922 if (hba->ee_ctrl_mask & mask)
4923 goto out;
4924
4925 val = hba->ee_ctrl_mask | mask;
d7e2ddd5 4926 val &= MASK_EE_STATUS;
5e86ae44 4927 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 4928 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4929 if (!err)
4930 hba->ee_ctrl_mask |= mask;
4931out:
4932 return err;
4933}
4934
4935/**
4936 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4937 * @hba: per-adapter instance
4938 *
4939 * Allow device to manage background operations on its own. Enabling
4940 * this might lead to inconsistent latencies during normal data transfers
4941 * as the device is allowed to manage its own way of handling background
4942 * operations.
4943 *
4944 * Returns zero on success, non-zero on failure.
4945 */
4946static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
4947{
4948 int err = 0;
4949
4950 if (hba->auto_bkops_enabled)
4951 goto out;
4952
dc3c8d3a 4953 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
66ec6d59
SRT		 4954 QUERY_FLAG_IDN_BKOPS_EN, NULL);
4955 if (err) {
4956 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
4957 __func__, err);
4958 goto out;
4959 }
4960
4961 hba->auto_bkops_enabled = true;
7ff5ab47 4962 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
66ec6d59
SRT		 4963
4964 /* No need of URGENT_BKOPS exception from the device */
4965 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
4966 if (err)
4967 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
4968 __func__, err);
4969out:
4970 return err;
4971}
4972
4973/**
4974 * ufshcd_disable_auto_bkops - block device in doing background operations
4975 * @hba: per-adapter instance
4976 *
4977 * Disabling background operations improves command response latency but
4978 * has drawback of device moving into critical state where the device is
4979 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
4980 * host is idle so that BKOPS are managed effectively without any negative
4981 * impacts.
4982 *
4983 * Returns zero on success, non-zero on failure.
4984 */
4985static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
4986{
4987 int err = 0;
4988
4989 if (!hba->auto_bkops_enabled)
4990 goto out;
4991
4992 /*
4993 * If host assisted BKOPs is to be enabled, make sure
4994 * urgent bkops exception is allowed.
4995 */
4996 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
4997 if (err) {
4998 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
4999 __func__, err);
5000 goto out;
5001 }
5002
dc3c8d3a 5003 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
66ec6d59
SRT		 5004 QUERY_FLAG_IDN_BKOPS_EN, NULL);
5005 if (err) {
5006 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5007 __func__, err);
5008 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5009 goto out;
5010 }
5011
5012 hba->auto_bkops_enabled = false;
7ff5ab47 5013 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
24366c2a 5014 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5015out:
5016 return err;
5017}
5018
5019/**
4e768e76 5020 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
66ec6d59
SRT		 5021 * @hba: per adapter instance
5022 *
5023 * After a device reset the device may toggle the BKOPS_EN flag
5024 * to default value. The s/w tracking variables should be updated
4e768e76
SJ		 5025 * as well. This function would change the auto-bkops state based on
5026 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
66ec6d59 5027 */
4e768e76 5028static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
66ec6d59 5029{
4e768e76
SJ		 5030 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5031 hba->auto_bkops_enabled = false;
5032 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5033 ufshcd_enable_auto_bkops(hba);
5034 } else {
5035 hba->auto_bkops_enabled = true;
5036 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5037 ufshcd_disable_auto_bkops(hba);
5038 }
24366c2a 5039 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5040}
5041
5042static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5043{
5e86ae44 5044 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5045 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5046}
5047
5048/**
57d104c1 5049 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
66ec6d59 5050 * @hba: per-adapter instance
57d104c1 5051 * @status: bkops_status value
66ec6d59 5052 *
57d104c1
SJ		 5053 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5054 * flag in the device to permit background operations if the device
5055 * bkops_status is greater than or equal to "status" argument passed to
5056 * this function, disable otherwise.
5057 *
5058 * Returns 0 for success, non-zero in case of failure.
5059 *
5060 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5061 * to know whether auto bkops is enabled or disabled after this function
5062 * returns control to it.
66ec6d59 5063 */
57d104c1
SJ		 5064static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5065 enum bkops_status status)
66ec6d59
SRT		 5066{
5067 int err;
57d104c1 5068 u32 curr_status = 0;
66ec6d59 5069
57d104c1 5070 err = ufshcd_get_bkops_status(hba, &curr_status);
66ec6d59
SRT		 5071 if (err) {
5072 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5073 __func__, err);
5074 goto out;
57d104c1
SJ		 5075 } else if (curr_status > BKOPS_STATUS_MAX) {
5076 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5077 __func__, curr_status);
5078 err = -EINVAL;
5079 goto out;
66ec6d59
SRT		 5080 }
5081
57d104c1 5082 if (curr_status >= status)
66ec6d59 5083 err = ufshcd_enable_auto_bkops(hba);
57d104c1
SJ		 5084 else
5085 err = ufshcd_disable_auto_bkops(hba);
24366c2a 5086 hba->urgent_bkops_lvl = curr_status;
66ec6d59
SRT		 5087out:
5088 return err;
5089}
5090
57d104c1
SJ		 5091/**
5092 * ufshcd_urgent_bkops - handle urgent bkops exception event
5093 * @hba: per-adapter instance
5094 *
5095 * Enable fBackgroundOpsEn flag in the device to permit background
5096 * operations.
5097 *
5098 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5099 * and negative error value for any other failure.
5100 */
5101static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5102{
afdfff59 5103 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
57d104c1
SJ		 5104}
5105
66ec6d59
SRT		 5106static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5107{
5e86ae44 5108 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5109 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5110}
5111
afdfff59
YG		 5112static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5113{
5114 int err;
5115 u32 curr_status = 0;
5116
5117 if (hba->is_urgent_bkops_lvl_checked)
5118 goto enable_auto_bkops;
5119
5120 err = ufshcd_get_bkops_status(hba, &curr_status);
5121 if (err) {
5122 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5123 __func__, err);
5124 goto out;
5125 }
5126
5127 /*
5128 * We are seeing that some devices are raising the urgent bkops
5129 * exception events even when BKOPS status doesn't indicate performace
5130 * impacted or critical. Handle these device by determining their urgent
5131 * bkops status at runtime.
5132 */
5133 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5134 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5135 __func__, curr_status);
5136 /* update the current status as the urgent bkops level */
5137 hba->urgent_bkops_lvl = curr_status;
5138 hba->is_urgent_bkops_lvl_checked = true;
5139 }
5140
5141enable_auto_bkops:
5142 err = ufshcd_enable_auto_bkops(hba);
5143out:
5144 if (err < 0)
5145 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5146 __func__, err);
5147}
5148
66ec6d59
SRT		 5149/**
5150 * ufshcd_exception_event_handler - handle exceptions raised by device
5151 * @work: pointer to work data
5152 *
5153 * Read bExceptionEventStatus attribute from the device and handle the
5154 * exception event accordingly.
5155 */
5156static void ufshcd_exception_event_handler(struct work_struct *work)
5157{
5158 struct ufs_hba *hba;
5159 int err;
5160 u32 status = 0;
5161 hba = container_of(work, struct ufs_hba, eeh_work);
5162
62694735 5163 pm_runtime_get_sync(hba->dev);
03e1d28e 5164 ufshcd_scsi_block_requests(hba);
66ec6d59
SRT		 5165 err = ufshcd_get_ee_status(hba, &status);
5166 if (err) {
5167 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5168 __func__, err);
5169 goto out;
5170 }
5171
5172 status &= hba->ee_ctrl_mask;
afdfff59
YG		 5173
5174 if (status & MASK_EE_URGENT_BKOPS)
5175 ufshcd_bkops_exception_event_handler(hba);
5176
66ec6d59 5177out:
03e1d28e 5178 ufshcd_scsi_unblock_requests(hba);
2824ec9f
SL		 5179 /*
5180 * pm_runtime_get_noresume is called while scheduling
5181 * eeh_work to avoid suspend racing with exception work.
5182 * Hence decrement usage counter using pm_runtime_put_noidle
5183 * to allow suspend on completion of exception event handler.
5184 */
5185 pm_runtime_put_noidle(hba->dev);
5186 pm_runtime_put(hba->dev);
66ec6d59
SRT		 5187 return;
5188}
5189
9a47ec7c
YG		 5190/* Complete requests that have door-bell cleared */
5191static void ufshcd_complete_requests(struct ufs_hba *hba)
5192{
5193 ufshcd_transfer_req_compl(hba);
5194 ufshcd_tmc_handler(hba);
5195}
5196
583fa62d
YG		 5197/**
5198 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5199 * to recover from the DL NAC errors or not.
5200 * @hba: per-adapter instance
5201 *
5202 * Returns true if error handling is required, false otherwise
5203 */
5204static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5205{
5206 unsigned long flags;
5207 bool err_handling = true;
5208
5209 spin_lock_irqsave(hba->host->host_lock, flags);
5210 /*
5211 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5212 * device fatal error and/or DL NAC & REPLAY timeout errors.
5213 */
5214 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5215 goto out;
5216
5217 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5218 ((hba->saved_err & UIC_ERROR) &&
5219 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5220 goto out;
5221
5222 if ((hba->saved_err & UIC_ERROR) &&
5223 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5224 int err;
5225 /*
5226 * wait for 50ms to see if we can get any other errors or not.
5227 */
5228 spin_unlock_irqrestore(hba->host->host_lock, flags);
5229 msleep(50);
5230 spin_lock_irqsave(hba->host->host_lock, flags);
5231
5232 /*
5233 * now check if we have got any other severe errors other than
5234 * DL NAC error?
5235 */
5236 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5237 ((hba->saved_err & UIC_ERROR) &&
5238 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5239 goto out;
5240
5241 /*
5242 * As DL NAC is the only error received so far, send out NOP
5243 * command to confirm if link is still active or not.
5244 * - If we don't get any response then do error recovery.
5245 * - If we get response then clear the DL NAC error bit.
5246 */
5247
5248 spin_unlock_irqrestore(hba->host->host_lock, flags);
5249 err = ufshcd_verify_dev_init(hba);
5250 spin_lock_irqsave(hba->host->host_lock, flags);
5251
5252 if (err)
5253 goto out;
5254
5255 /* Link seems to be alive hence ignore the DL NAC errors */
5256 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5257 hba->saved_err &= ~UIC_ERROR;
5258 /* clear NAC error */
5259 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5260 if (!hba->saved_uic_err) {
5261 err_handling = false;
5262 goto out;
5263 }
5264 }
5265out:
5266 spin_unlock_irqrestore(hba->host->host_lock, flags);
5267 return err_handling;
5268}
5269
7a3e97b0 5270/**
e8e7f271
SRT		 5271 * ufshcd_err_handler - handle UFS errors that require s/w attention
5272 * @work: pointer to work structure
7a3e97b0 5273 */
e8e7f271 5274static void ufshcd_err_handler(struct work_struct *work)
7a3e97b0
SY		 5275{
5276 struct ufs_hba *hba;
e8e7f271
SRT		 5277 unsigned long flags;
5278 u32 err_xfer = 0;
5279 u32 err_tm = 0;
5280 int err = 0;
5281 int tag;
9a47ec7c 5282 bool needs_reset = false;
e8e7f271
SRT		 5283
5284 hba = container_of(work, struct ufs_hba, eh_work);
7a3e97b0 5285
62694735 5286 pm_runtime_get_sync(hba->dev);
1ab27c9c 5287 ufshcd_hold(hba, false);
e8e7f271
SRT		 5288
5289 spin_lock_irqsave(hba->host->host_lock, flags);
9a47ec7c 5290 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
e8e7f271 5291 goto out;
e8e7f271
SRT		 5292
5293 hba->ufshcd_state = UFSHCD_STATE_RESET;
5294 ufshcd_set_eh_in_progress(hba);
5295
5296 /* Complete requests that have door-bell cleared by h/w */
9a47ec7c 5297 ufshcd_complete_requests(hba);
583fa62d
YG		 5298
5299 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5300 bool ret;
5301
5302 spin_unlock_irqrestore(hba->host->host_lock, flags);
5303 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5304 ret = ufshcd_quirk_dl_nac_errors(hba);
5305 spin_lock_irqsave(hba->host->host_lock, flags);
5306 if (!ret)
5307 goto skip_err_handling;
5308 }
9a47ec7c 5309 if ((hba->saved_err & INT_FATAL_ERRORS) ||
82174440 5310 (hba->saved_err & UFSHCD_UIC_HIBERN8_MASK) ||
9a47ec7c
YG		 5311 ((hba->saved_err & UIC_ERROR) &&
5312 (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
5313 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5314 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5315 needs_reset = true;
e8e7f271 5316
9a47ec7c
YG		 5317 /*
5318 * if host reset is required then skip clearing the pending
2df74b69
CG		 5319 * transfers forcefully because they will get cleared during
5320 * host reset and restore
9a47ec7c
YG		 5321 */
5322 if (needs_reset)
5323 goto skip_pending_xfer_clear;
5324
5325 /* release lock as clear command might sleep */
5326 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5327 /* Clear pending transfer requests */
9a47ec7c
YG		 5328 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
5329 if (ufshcd_clear_cmd(hba, tag)) {
5330 err_xfer = true;
5331 goto lock_skip_pending_xfer_clear;
5332 }
5333 }
e8e7f271
SRT		 5334
5335 /* Clear pending task management requests */
9a47ec7c
YG		 5336 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
5337 if (ufshcd_clear_tm_cmd(hba, tag)) {
5338 err_tm = true;
5339 goto lock_skip_pending_xfer_clear;
5340 }
5341 }
e8e7f271 5342
9a47ec7c 5343lock_skip_pending_xfer_clear:
e8e7f271 5344 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 5345
9a47ec7c
YG		 5346 /* Complete the requests that are cleared by s/w */
5347 ufshcd_complete_requests(hba);
5348
5349 if (err_xfer || err_tm)
5350 needs_reset = true;
5351
5352skip_pending_xfer_clear:
e8e7f271 5353 /* Fatal errors need reset */
9a47ec7c
YG		 5354 if (needs_reset) {
5355 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
5356
5357 /*
5358 * ufshcd_reset_and_restore() does the link reinitialization
5359 * which will need atleast one empty doorbell slot to send the
5360 * device management commands (NOP and query commands).
5361 * If there is no slot empty at this moment then free up last
5362 * slot forcefully.
5363 */
5364 if (hba->outstanding_reqs == max_doorbells)
5365 __ufshcd_transfer_req_compl(hba,
5366 (1UL << (hba->nutrs - 1)));
5367
5368 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5369 err = ufshcd_reset_and_restore(hba);
9a47ec7c 5370 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271
SRT		 5371 if (err) {
5372 dev_err(hba->dev, "%s: reset and restore failed\n",
5373 __func__);
5374 hba->ufshcd_state = UFSHCD_STATE_ERROR;
5375 }
5376 /*
5377 * Inform scsi mid-layer that we did reset and allow to handle
5378 * Unit Attention properly.
5379 */
5380 scsi_report_bus_reset(hba->host, 0);
5381 hba->saved_err = 0;
5382 hba->saved_uic_err = 0;
5383 }
9a47ec7c 5384
583fa62d 5385skip_err_handling:
9a47ec7c
YG		 5386 if (!needs_reset) {
5387 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5388 if (hba->saved_err || hba->saved_uic_err)
5389 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
5390 __func__, hba->saved_err, hba->saved_uic_err);
5391 }
5392
e8e7f271
SRT		 5393 ufshcd_clear_eh_in_progress(hba);
5394
5395out:
9a47ec7c 5396 spin_unlock_irqrestore(hba->host->host_lock, flags);
38135535 5397 ufshcd_scsi_unblock_requests(hba);
1ab27c9c 5398 ufshcd_release(hba);
62694735 5399 pm_runtime_put_sync(hba->dev);
7a3e97b0
SY		 5400}
5401
5402/**
e8e7f271
SRT		 5403 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5404 * @hba: per-adapter instance
9333d775
VG		 5405 *
5406 * Returns
5407 * IRQ_HANDLED - If interrupt is valid
5408 * IRQ_NONE - If invalid interrupt
7a3e97b0 5409 */
9333d775 5410static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
7a3e97b0
SY		 5411{
5412 u32 reg;
9333d775 5413 irqreturn_t retval = IRQ_NONE;
7a3e97b0 5414
fb7b45f0
DR		 5415 /* PHY layer lane error */
5416 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5417 /* Ignore LINERESET indication, as this is not an error */
5418 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
9333d775 5419 (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)) {
fb7b45f0
DR		 5420 /*
5421 * To know whether this error is fatal or not, DB timeout
5422 * must be checked but this error is handled separately.
5423 */
5424 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", __func__);
48d5b973 5425 ufshcd_update_reg_hist(&hba->ufs_stats.pa_err, reg);
9333d775 5426 retval |= IRQ_HANDLED;
ff8e20c6 5427 }
fb7b45f0 5428
e8e7f271
SRT		 5429 /* PA_INIT_ERROR is fatal and needs UIC reset */
5430 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
9333d775
VG		 5431 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
5432 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
48d5b973 5433 ufshcd_update_reg_hist(&hba->ufs_stats.dl_err, reg);
ff8e20c6 5434
9333d775
VG		 5435 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
5436 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
5437 else if (hba->dev_quirks &
5438 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5439 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
5440 hba->uic_error |=
5441 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5442 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
5443 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
5444 }
5445 retval |= IRQ_HANDLED;
583fa62d 5446 }
e8e7f271
SRT		 5447
5448 /* UIC NL/TL/DME errors needs software retry */
5449 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
9333d775
VG		 5450 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
5451 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
48d5b973 5452 ufshcd_update_reg_hist(&hba->ufs_stats.nl_err, reg);
e8e7f271 5453 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
9333d775 5454 retval |= IRQ_HANDLED;
ff8e20c6 5455 }
e8e7f271
SRT		 5456
5457 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
9333d775
VG		 5458 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
5459 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
48d5b973 5460 ufshcd_update_reg_hist(&hba->ufs_stats.tl_err, reg);
e8e7f271 5461 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
9333d775 5462 retval |= IRQ_HANDLED;
ff8e20c6 5463 }
e8e7f271
SRT		 5464
5465 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
9333d775
VG		 5466 if ((reg & UIC_DME_ERROR) &&
5467 (reg & UIC_DME_ERROR_CODE_MASK)) {
48d5b973 5468 ufshcd_update_reg_hist(&hba->ufs_stats.dme_err, reg);
e8e7f271 5469 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
9333d775 5470 retval |= IRQ_HANDLED;
ff8e20c6 5471 }
e8e7f271
SRT		 5472
5473 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
5474 __func__, hba->uic_error);
9333d775 5475 return retval;
e8e7f271
SRT		 5476}
5477
82174440
SC		 5478static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5479 u32 intr_mask)
5480{
5a244e0e
SC		 5481 if (!ufshcd_is_auto_hibern8_supported(hba) ||
5482 !ufshcd_is_auto_hibern8_enabled(hba))
82174440
SC		 5483 return false;
5484
5485 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5486 return false;
5487
5488 if (hba->active_uic_cmd &&
5489 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5490 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5491 return false;
5492
5493 return true;
5494}
5495
e8e7f271
SRT		 5496/**
5497 * ufshcd_check_errors - Check for errors that need s/w attention
5498 * @hba: per-adapter instance
9333d775
VG		 5499 *
5500 * Returns
5501 * IRQ_HANDLED - If interrupt is valid
5502 * IRQ_NONE - If invalid interrupt
e8e7f271 5503 */
9333d775 5504static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba)
e8e7f271
SRT		 5505{
5506 bool queue_eh_work = false;
9333d775 5507 irqreturn_t retval = IRQ_NONE;
e8e7f271 5508
d3c615bf
SC		 5509 if (hba->errors & INT_FATAL_ERRORS) {
5510 ufshcd_update_reg_hist(&hba->ufs_stats.fatal_err, hba->errors);
e8e7f271 5511 queue_eh_work = true;
d3c615bf 5512 }
7a3e97b0
SY		 5513
5514 if (hba->errors & UIC_ERROR) {
e8e7f271 5515 hba->uic_error = 0;
9333d775 5516 retval = ufshcd_update_uic_error(hba);
e8e7f271
SRT		 5517 if (hba->uic_error)
5518 queue_eh_work = true;
7a3e97b0 5519 }
e8e7f271 5520
82174440
SC		 5521 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
5522 dev_err(hba->dev,
5523 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
5524 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
5525 "Enter" : "Exit",
5526 hba->errors, ufshcd_get_upmcrs(hba));
d3c615bf
SC		 5527 ufshcd_update_reg_hist(&hba->ufs_stats.auto_hibern8_err,
5528 hba->errors);
82174440
SC		 5529 queue_eh_work = true;
5530 }
5531
e8e7f271 5532 if (queue_eh_work) {
9a47ec7c
YG		 5533 /*
5534 * update the transfer error masks to sticky bits, let's do this
5535 * irrespective of current ufshcd_state.
5536 */
5537 hba->saved_err |= hba->errors;
5538 hba->saved_uic_err |= hba->uic_error;
5539
e8e7f271
SRT		 5540 /* handle fatal errors only when link is functional */
5541 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
5542 /* block commands from scsi mid-layer */
38135535 5543 ufshcd_scsi_block_requests(hba);
e8e7f271 5544
141f8165 5545 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
66cc820f
DR		 5546
5547 /* dump controller state before resetting */
5548 if (hba->saved_err & (INT_FATAL_ERRORS | UIC_ERROR)) {
5549 bool pr_prdt = !!(hba->saved_err &
5550 SYSTEM_BUS_FATAL_ERROR);
5551
5552 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
5553 __func__, hba->saved_err,
5554 hba->saved_uic_err);
5555
5556 ufshcd_print_host_regs(hba);
5557 ufshcd_print_pwr_info(hba);
5558 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
5559 ufshcd_print_trs(hba, hba->outstanding_reqs,
5560 pr_prdt);
5561 }
e8e7f271
SRT		 5562 schedule_work(&hba->eh_work);
5563 }
9333d775 5564 retval |= IRQ_HANDLED;
3441da7d 5565 }
e8e7f271
SRT		 5566 /*
5567 * if (!queue_eh_work) -
5568 * Other errors are either non-fatal where host recovers
5569 * itself without s/w intervention or errors that will be
5570 * handled by the SCSI core layer.
5571 */
9333d775 5572 return retval;
7a3e97b0
SY		 5573}
5574
69a6c269
BVA		 5575struct ctm_info {
5576 struct ufs_hba *hba;
5577 unsigned long pending;
5578 unsigned int ncpl;
5579};
5580
5581static bool ufshcd_compl_tm(struct request *req, void *priv, bool reserved)
5582{
5583 struct ctm_info *const ci = priv;
5584 struct completion *c;
5585
5586 WARN_ON_ONCE(reserved);
5587 if (test_bit(req->tag, &ci->pending))
5588 return true;
5589 ci->ncpl++;
5590 c = req->end_io_data;
5591 if (c)
5592 complete(c);
5593 return true;
5594}
5595
7a3e97b0
SY		 5596/**
5597 * ufshcd_tmc_handler - handle task management function completion
5598 * @hba: per adapter instance
9333d775
VG		 5599 *
5600 * Returns
5601 * IRQ_HANDLED - If interrupt is valid
5602 * IRQ_NONE - If invalid interrupt
7a3e97b0 5603 */
9333d775 5604static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
7a3e97b0 5605{
69a6c269
BVA		 5606 struct request_queue *q = hba->tmf_queue;
5607 struct ctm_info ci = {
5608 .hba = hba,
5609 .pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL),
5610 };
7a3e97b0 5611
69a6c269
BVA		 5612 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_compl_tm, &ci);
5613 return ci.ncpl ? IRQ_HANDLED : IRQ_NONE;
7a3e97b0
SY		 5614}
5615
5616/**
5617 * ufshcd_sl_intr - Interrupt service routine
5618 * @hba: per adapter instance
5619 * @intr_status: contains interrupts generated by the controller
9333d775
VG		 5620 *
5621 * Returns
5622 * IRQ_HANDLED - If interrupt is valid
5623 * IRQ_NONE - If invalid interrupt
7a3e97b0 5624 */
9333d775 5625static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
7a3e97b0 5626{
9333d775
VG		 5627 irqreturn_t retval = IRQ_NONE;
5628
7a3e97b0 5629 hba->errors = UFSHCD_ERROR_MASK & intr_status;
82174440
SC		 5630
5631 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5632 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5633
7a3e97b0 5634 if (hba->errors)
9333d775 5635 retval |= ufshcd_check_errors(hba);
7a3e97b0 5636
53b3d9c3 5637 if (intr_status & UFSHCD_UIC_MASK)
9333d775 5638 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
7a3e97b0
SY		 5639
5640 if (intr_status & UTP_TASK_REQ_COMPL)
9333d775 5641 retval |= ufshcd_tmc_handler(hba);
7a3e97b0
SY		 5642
5643 if (intr_status & UTP_TRANSFER_REQ_COMPL)
9333d775
VG		 5644 retval |= ufshcd_transfer_req_compl(hba);
5645
5646 return retval;
7a3e97b0
SY		 5647}
5648
5649/**
5650 * ufshcd_intr - Main interrupt service routine
5651 * @irq: irq number
5652 * @__hba: pointer to adapter instance
5653 *
9333d775
VG		 5654 * Returns
5655 * IRQ_HANDLED - If interrupt is valid
5656 * IRQ_NONE - If invalid interrupt
7a3e97b0
SY		 5657 */
5658static irqreturn_t ufshcd_intr(int irq, void *__hba)
5659{
d75f7fe4 5660 u32 intr_status, enabled_intr_status;
7a3e97b0
SY		 5661 irqreturn_t retval = IRQ_NONE;
5662 struct ufs_hba *hba = __hba;
7f6ba4f1 5663 int retries = hba->nutrs;
7a3e97b0
SY		 5664
5665 spin_lock(hba->host->host_lock);
b873a275 5666 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
7a3e97b0 5667
7f6ba4f1
VG		 5668 /*
5669 * There could be max of hba->nutrs reqs in flight and in worst case
5670 * if the reqs get finished 1 by 1 after the interrupt status is
5671 * read, make sure we handle them by checking the interrupt status
5672 * again in a loop until we process all of the reqs before returning.
5673 */
5674 do {
5675 enabled_intr_status =
5676 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
5677 if (intr_status)
5678 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
9333d775
VG		 5679 if (enabled_intr_status)
5680 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
7f6ba4f1
VG		 5681
5682 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
5683 } while (intr_status && --retries);
d75f7fe4 5684
9333d775
VG		 5685 if (retval == IRQ_NONE) {
5686 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
5687 __func__, intr_status);
5688 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
5689 }
5690
7a3e97b0
SY		 5691 spin_unlock(hba->host->host_lock);
5692 return retval;
5693}
5694
e2933132
SRT		 5695static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
5696{
5697 int err = 0;
5698 u32 mask = 1 << tag;
5699 unsigned long flags;
5700
5701 if (!test_bit(tag, &hba->outstanding_tasks))
5702 goto out;
5703
5704 spin_lock_irqsave(hba->host->host_lock, flags);
1399c5b0 5705 ufshcd_utmrl_clear(hba, tag);
e2933132
SRT		 5706 spin_unlock_irqrestore(hba->host->host_lock, flags);
5707
5708 /* poll for max. 1 sec to clear door bell register by h/w */
5709 err = ufshcd_wait_for_register(hba,
5710 REG_UTP_TASK_REQ_DOOR_BELL,
596585a2 5711 mask, 0, 1000, 1000, true);
e2933132
SRT		 5712out:
5713 return err;
5714}
5715
c6049cd9
CH		 5716static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
5717 struct utp_task_req_desc *treq, u8 tm_function)
7a3e97b0 5718{
69a6c269 5719 struct request_queue *q = hba->tmf_queue;
c6049cd9 5720 struct Scsi_Host *host = hba->host;
69a6c269
BVA		 5721 DECLARE_COMPLETION_ONSTACK(wait);
5722 struct request *req;
7a3e97b0 5723 unsigned long flags;
c6049cd9 5724 int free_slot, task_tag, err;
7a3e97b0 5725
e2933132
SRT		 5726 /*
5727 * Get free slot, sleep if slots are unavailable.
5728 * Even though we use wait_event() which sleeps indefinitely,
5729 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5730 */
69a6c269
BVA		 5731 req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
5732 req->end_io_data = &wait;
5733 free_slot = req->tag;
5734 WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);
1ab27c9c 5735 ufshcd_hold(hba, false);
7a3e97b0 5736
e2933132 5737 spin_lock_irqsave(host->host_lock, flags);
e2933132 5738 task_tag = hba->nutrs + free_slot;
7a3e97b0 5739
c6049cd9
CH		 5740 treq->req_header.dword_0 |= cpu_to_be32(task_tag);
5741
5742 memcpy(hba->utmrdl_base_addr + free_slot, treq, sizeof(*treq));
d2877be4
KK		 5743 ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
5744
7a3e97b0
SY		 5745 /* send command to the controller */
5746 __set_bit(free_slot, &hba->outstanding_tasks);
897efe62
YG		 5747
5748 /* Make sure descriptors are ready before ringing the task doorbell */
5749 wmb();
5750
b873a275 5751 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
ad1a1b9c
GB		 5752 /* Make sure that doorbell is committed immediately */
5753 wmb();
7a3e97b0
SY		 5754
5755 spin_unlock_irqrestore(host->host_lock, flags);
5756
6667e6d9
OS		 5757 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_send");
5758
7a3e97b0 5759 /* wait until the task management command is completed */
69a6c269 5760 err = wait_for_completion_io_timeout(&wait,
e2933132 5761 msecs_to_jiffies(TM_CMD_TIMEOUT));
7a3e97b0 5762 if (!err) {
69a6c269
BVA		 5763 /*
5764 * Make sure that ufshcd_compl_tm() does not trigger a
5765 * use-after-free.
5766 */
5767 req->end_io_data = NULL;
6667e6d9 5768 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete_err");
e2933132
SRT		 5769 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
5770 __func__, tm_function);
5771 if (ufshcd_clear_tm_cmd(hba, free_slot))
5772 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
5773 __func__, free_slot);
5774 err = -ETIMEDOUT;
5775 } else {
c6049cd9
CH		 5776 err = 0;
5777 memcpy(treq, hba->utmrdl_base_addr + free_slot, sizeof(*treq));
5778
6667e6d9 5779 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
7a3e97b0 5780 }
e2933132 5781
b557217c
SC		 5782 spin_lock_irqsave(hba->host->host_lock, flags);
5783 __clear_bit(free_slot, &hba->outstanding_tasks);
5784 spin_unlock_irqrestore(hba->host->host_lock, flags);
5785
69a6c269 5786 blk_put_request(req);
e2933132 5787
1ab27c9c 5788 ufshcd_release(hba);
7a3e97b0
SY		 5789 return err;
5790}
5791
c6049cd9
CH		 5792/**
5793 * ufshcd_issue_tm_cmd - issues task management commands to controller
5794 * @hba: per adapter instance
5795 * @lun_id: LUN ID to which TM command is sent
5796 * @task_id: task ID to which the TM command is applicable
5797 * @tm_function: task management function opcode
5798 * @tm_response: task management service response return value
5799 *
5800 * Returns non-zero value on error, zero on success.
5801 */
5802static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
5803 u8 tm_function, u8 *tm_response)
5804{
5805 struct utp_task_req_desc treq = { { 0 }, };
5806 int ocs_value, err;
5807
5808 /* Configure task request descriptor */
5809 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
5810 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
5811
5812 /* Configure task request UPIU */
5813 treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
5814 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
5815 treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
5816
5817 /*
5818 * The host shall provide the same value for LUN field in the basic
5819 * header and for Input Parameter.
5820 */
5821 treq.input_param1 = cpu_to_be32(lun_id);
5822 treq.input_param2 = cpu_to_be32(task_id);
5823
5824 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
5825 if (err == -ETIMEDOUT)
5826 return err;
5827
5828 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
5829 if (ocs_value != OCS_SUCCESS)
5830 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
5831 __func__, ocs_value);
5832 else if (tm_response)
5833 *tm_response = be32_to_cpu(treq.output_param1) &
5834 MASK_TM_SERVICE_RESP;
5835 return err;
5836}
5837
5e0a86ee
AA		 5838/**
5839 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
5840 * @hba: per-adapter instance
5841 * @req_upiu: upiu request
5842 * @rsp_upiu: upiu reply
5e0a86ee
AA		 5843 * @desc_buff: pointer to descriptor buffer, NULL if NA
5844 * @buff_len: descriptor size, 0 if NA
d0e9760d 5845 * @cmd_type: specifies the type (NOP, Query...)
5e0a86ee
AA		 5846 * @desc_op: descriptor operation
5847 *
5848 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
5849 * Therefore, it "rides" the device management infrastructure: uses its tag and
5850 * tasks work queues.
5851 *
5852 * Since there is only one available tag for device management commands,
5853 * the caller is expected to hold the hba->dev_cmd.lock mutex.
5854 */
5855static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
5856 struct utp_upiu_req *req_upiu,
5857 struct utp_upiu_req *rsp_upiu,
5858 u8 *desc_buff, int *buff_len,
7f674c38 5859 enum dev_cmd_type cmd_type,
5e0a86ee
AA		 5860 enum query_opcode desc_op)
5861{
7252a360
BVA		 5862 struct request_queue *q = hba->cmd_queue;
5863 struct request *req;
5e0a86ee
AA		 5864 struct ufshcd_lrb *lrbp;
5865 int err = 0;
5866 int tag;
5867 struct completion wait;
5868 unsigned long flags;
5869 u32 upiu_flags;
5870
5871 down_read(&hba->clk_scaling_lock);
5872
7252a360 5873 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
bb14dd15
DC		 5874 if (IS_ERR(req)) {
5875 err = PTR_ERR(req);
5876 goto out_unlock;
5877 }
7252a360
BVA		 5878 tag = req->tag;
5879 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5e0a86ee
AA		 5880
5881 init_completion(&wait);
5882 lrbp = &hba->lrb[tag];
5883 WARN_ON(lrbp->cmd);
5884
5885 lrbp->cmd = NULL;
5886 lrbp->sense_bufflen = 0;
5887 lrbp->sense_buffer = NULL;
5888 lrbp->task_tag = tag;
5889 lrbp->lun = 0;
5890 lrbp->intr_cmd = true;
5891 hba->dev_cmd.type = cmd_type;
5892
5893 switch (hba->ufs_version) {
5894 case UFSHCI_VERSION_10:
5895 case UFSHCI_VERSION_11:
5896 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
5897 break;
5898 default:
5899 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
5900 break;
5901 }
5902
5903 /* update the task tag in the request upiu */
5904 req_upiu->header.dword_0 |= cpu_to_be32(tag);
5905
5906 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
5907
5908 /* just copy the upiu request as it is */
5909 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
5910 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
5911 /* The Data Segment Area is optional depending upon the query
5912 * function value. for WRITE DESCRIPTOR, the data segment
5913 * follows right after the tsf.
5914 */
5915 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
5916 *buff_len = 0;
5917 }
5918
5919 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5920
5921 hba->dev_cmd.complete = &wait;
5922
5923 /* Make sure descriptors are ready before ringing the doorbell */
5924 wmb();
5925 spin_lock_irqsave(hba->host->host_lock, flags);
5926 ufshcd_send_command(hba, tag);
5927 spin_unlock_irqrestore(hba->host->host_lock, flags);
5928
5929 /*
5930 * ignore the returning value here - ufshcd_check_query_response is
5931 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
5932 * read the response directly ignoring all errors.
5933 */
5934 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
5935
5936 /* just copy the upiu response as it is */
5937 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
4bbbe242
AA		 5938 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
5939 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
5940 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
5941 MASK_QUERY_DATA_SEG_LEN;
5942
5943 if (*buff_len >= resp_len) {
5944 memcpy(desc_buff, descp, resp_len);
5945 *buff_len = resp_len;
5946 } else {
3d4881d1
BH		 5947 dev_warn(hba->dev,
5948 "%s: rsp size %d is bigger than buffer size %d",
5949 __func__, resp_len, *buff_len);
4bbbe242
AA		 5950 *buff_len = 0;
5951 err = -EINVAL;
5952 }
5953 }
5e0a86ee 5954
7252a360 5955 blk_put_request(req);
bb14dd15 5956out_unlock:
5e0a86ee
AA		 5957 up_read(&hba->clk_scaling_lock);
5958 return err;
5959}
5960
5961/**
5962 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
5963 * @hba: per-adapter instance
5964 * @req_upiu: upiu request
5965 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
5966 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
5967 * @desc_buff: pointer to descriptor buffer, NULL if NA
5968 * @buff_len: descriptor size, 0 if NA
5969 * @desc_op: descriptor operation
5970 *
5971 * Supports UTP Transfer requests (nop and query), and UTP Task
5972 * Management requests.
5973 * It is up to the caller to fill the upiu conent properly, as it will
5974 * be copied without any further input validations.
5975 */
5976int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
5977 struct utp_upiu_req *req_upiu,
5978 struct utp_upiu_req *rsp_upiu,
5979 int msgcode,
5980 u8 *desc_buff, int *buff_len,
5981 enum query_opcode desc_op)
5982{
5983 int err;
7f674c38 5984 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
5e0a86ee
AA		 5985 struct utp_task_req_desc treq = { { 0 }, };
5986 int ocs_value;
5987 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
5988
5e0a86ee
AA		 5989 switch (msgcode) {
5990 case UPIU_TRANSACTION_NOP_OUT:
5991 cmd_type = DEV_CMD_TYPE_NOP;
5992 /* fall through */
5993 case UPIU_TRANSACTION_QUERY_REQ:
5994 ufshcd_hold(hba, false);
5995 mutex_lock(&hba->dev_cmd.lock);
5996 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
5997 desc_buff, buff_len,
5998 cmd_type, desc_op);
5999 mutex_unlock(&hba->dev_cmd.lock);
6000 ufshcd_release(hba);
6001
6002 break;
6003 case UPIU_TRANSACTION_TASK_REQ:
6004 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6005 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6006
6007 memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
6008
6009 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
6010 if (err == -ETIMEDOUT)
6011 break;
6012
6013 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6014 if (ocs_value != OCS_SUCCESS) {
6015 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
6016 ocs_value);
6017 break;
6018 }
6019
6020 memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
6021
6022 break;
6023 default:
6024 err = -EINVAL;
6025
6026 break;
6027 }
6028
5e0a86ee
AA		 6029 return err;
6030}
6031
7a3e97b0 6032/**
3441da7d
SRT		 6033 * ufshcd_eh_device_reset_handler - device reset handler registered to
6034 * scsi layer.
7a3e97b0
SY		 6035 * @cmd: SCSI command pointer
6036 *
6037 * Returns SUCCESS/FAILED
6038 */
3441da7d 6039static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7a3e97b0
SY		 6040{
6041 struct Scsi_Host *host;
6042 struct ufs_hba *hba;
6043 unsigned int tag;
6044 u32 pos;
6045 int err;
e2933132
SRT		 6046 u8 resp = 0xF;
6047 struct ufshcd_lrb *lrbp;
3441da7d 6048 unsigned long flags;
7a3e97b0
SY		 6049
6050 host = cmd->device->host;
6051 hba = shost_priv(host);
6052 tag = cmd->request->tag;
6053
e2933132
SRT		 6054 lrbp = &hba->lrb[tag];
6055 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
6056 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
3441da7d
SRT		 6057 if (!err)
6058 err = resp;
7a3e97b0 6059 goto out;
e2933132 6060 }
7a3e97b0 6061
3441da7d
SRT		 6062 /* clear the commands that were pending for corresponding LUN */
6063 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
6064 if (hba->lrb[pos].lun == lrbp->lun) {
6065 err = ufshcd_clear_cmd(hba, pos);
6066 if (err)
6067 break;
7a3e97b0 6068 }
3441da7d
SRT		 6069 }
6070 spin_lock_irqsave(host->host_lock, flags);
6071 ufshcd_transfer_req_compl(hba);
6072 spin_unlock_irqrestore(host->host_lock, flags);
7fabb77b 6073
7a3e97b0 6074out:
7fabb77b 6075 hba->req_abort_count = 0;
8808b4e9 6076 ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, (u32)err);
3441da7d
SRT		 6077 if (!err) {
6078 err = SUCCESS;
6079 } else {
6080 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6081 err = FAILED;
6082 }
7a3e97b0
SY		 6083 return err;
6084}
6085
e0b299e3
GB		 6086static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
6087{
6088 struct ufshcd_lrb *lrbp;
6089 int tag;
6090
6091 for_each_set_bit(tag, &bitmap, hba->nutrs) {
6092 lrbp = &hba->lrb[tag];
6093 lrbp->req_abort_skip = true;
6094 }
6095}
6096
7a3e97b0
SY		 6097/**
6098 * ufshcd_abort - abort a specific command
6099 * @cmd: SCSI command pointer
6100 *
f20810d8
SRT		 6101 * Abort the pending command in device by sending UFS_ABORT_TASK task management
6102 * command, and in host controller by clearing the door-bell register. There can
6103 * be race between controller sending the command to the device while abort is
6104 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6105 * really issued and then try to abort it.
6106 *
7a3e97b0
SY		 6107 * Returns SUCCESS/FAILED
6108 */
6109static int ufshcd_abort(struct scsi_cmnd *cmd)
6110{
6111 struct Scsi_Host *host;
6112 struct ufs_hba *hba;
6113 unsigned long flags;
6114 unsigned int tag;
f20810d8
SRT		 6115 int err = 0;
6116 int poll_cnt;
e2933132
SRT		 6117 u8 resp = 0xF;
6118 struct ufshcd_lrb *lrbp;
e9d501b1 6119 u32 reg;
7a3e97b0
SY		 6120
6121 host = cmd->device->host;
6122 hba = shost_priv(host);
6123 tag = cmd->request->tag;
e7d38257 6124 lrbp = &hba->lrb[tag];
14497328
YG		 6125 if (!ufshcd_valid_tag(hba, tag)) {
6126 dev_err(hba->dev,
6127 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
6128 __func__, tag, cmd, cmd->request);
6129 BUG();
6130 }
7a3e97b0 6131
e7d38257
DR		 6132 /*
6133 * Task abort to the device W-LUN is illegal. When this command
6134 * will fail, due to spec violation, scsi err handling next step
6135 * will be to send LU reset which, again, is a spec violation.
6136 * To avoid these unnecessary/illegal step we skip to the last error
6137 * handling stage: reset and restore.
6138 */
6139 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN)
6140 return ufshcd_eh_host_reset_handler(cmd);
6141
1ab27c9c 6142 ufshcd_hold(hba, false);
14497328 6143 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
f20810d8 6144 /* If command is already aborted/completed, return SUCCESS */
14497328
YG		 6145 if (!(test_bit(tag, &hba->outstanding_reqs))) {
6146 dev_err(hba->dev,
6147 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
6148 __func__, tag, hba->outstanding_reqs, reg);
f20810d8 6149 goto out;
14497328 6150 }
7a3e97b0 6151
e9d501b1
DR		 6152 if (!(reg & (1 << tag))) {
6153 dev_err(hba->dev,
6154 "%s: cmd was completed, but without a notifying intr, tag = %d",
6155 __func__, tag);
6156 }
6157
66cc820f
DR		 6158 /* Print Transfer Request of aborted task */
6159 dev_err(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
66cc820f 6160
7fabb77b
GB		 6161 /*
6162 * Print detailed info about aborted request.
6163 * As more than one request might get aborted at the same time,
6164 * print full information only for the first aborted request in order
6165 * to reduce repeated printouts. For other aborted requests only print
6166 * basic details.
6167 */
6168 scsi_print_command(hba->lrb[tag].cmd);
6169 if (!hba->req_abort_count) {
8808b4e9 6170 ufshcd_update_reg_hist(&hba->ufs_stats.task_abort, 0);
7fabb77b 6171 ufshcd_print_host_regs(hba);
6ba65588 6172 ufshcd_print_host_state(hba);
7fabb77b
GB		 6173 ufshcd_print_pwr_info(hba);
6174 ufshcd_print_trs(hba, 1 << tag, true);
6175 } else {
6176 ufshcd_print_trs(hba, 1 << tag, false);
6177 }
6178 hba->req_abort_count++;
e0b299e3
GB		 6179
6180 /* Skip task abort in case previous aborts failed and report failure */
6181 if (lrbp->req_abort_skip) {
6182 err = -EIO;
6183 goto out;
6184 }
6185
f20810d8
SRT		 6186 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6187 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6188 UFS_QUERY_TASK, &resp);
6189 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6190 /* cmd pending in the device */
ff8e20c6
DR		 6191 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6192 __func__, tag);
f20810d8
SRT		 6193 break;
6194 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
f20810d8
SRT		 6195 /*
6196 * cmd not pending in the device, check if it is
6197 * in transition.
6198 */
ff8e20c6
DR		 6199 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6200 __func__, tag);
f20810d8
SRT		 6201 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6202 if (reg & (1 << tag)) {
6203 /* sleep for max. 200us to stabilize */
6204 usleep_range(100, 200);
6205 continue;
6206 }
6207 /* command completed already */
ff8e20c6
DR		 6208 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6209 __func__, tag);
f20810d8
SRT		 6210 goto out;
6211 } else {
ff8e20c6
DR		 6212 dev_err(hba->dev,
6213 "%s: no response from device. tag = %d, err %d\n",
6214 __func__, tag, err);
f20810d8
SRT		 6215 if (!err)
6216 err = resp; /* service response error */
6217 goto out;
6218 }
6219 }
6220
6221 if (!poll_cnt) {
6222 err = -EBUSY;
7a3e97b0
SY		 6223 goto out;
6224 }
7a3e97b0 6225
e2933132
SRT		 6226 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6227 UFS_ABORT_TASK, &resp);
6228 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
ff8e20c6 6229 if (!err) {
f20810d8 6230 err = resp; /* service response error */
ff8e20c6
DR		 6231 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6232 __func__, tag, err);
6233 }
7a3e97b0 6234 goto out;
e2933132 6235 }
7a3e97b0 6236
f20810d8 6237 err = ufshcd_clear_cmd(hba, tag);
ff8e20c6
DR		 6238 if (err) {
6239 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6240 __func__, tag, err);
f20810d8 6241 goto out;
ff8e20c6 6242 }
f20810d8 6243
7a3e97b0
SY		 6244 scsi_dma_unmap(cmd);
6245
6246 spin_lock_irqsave(host->host_lock, flags);
a48353f6 6247 ufshcd_outstanding_req_clear(hba, tag);
7a3e97b0
SY		 6248 hba->lrb[tag].cmd = NULL;
6249 spin_unlock_irqrestore(host->host_lock, flags);
5a0b0cb9 6250
7a3e97b0 6251out:
f20810d8
SRT		 6252 if (!err) {
6253 err = SUCCESS;
6254 } else {
6255 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
e0b299e3 6256 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
f20810d8
SRT		 6257 err = FAILED;
6258 }
6259
1ab27c9c
ST		 6260 /*
6261 * This ufshcd_release() corresponds to the original scsi cmd that got
6262 * aborted here (as we won't get any IRQ for it).
6263 */
6264 ufshcd_release(hba);
7a3e97b0
SY		 6265 return err;
6266}
6267
3441da7d
SRT		 6268/**
6269 * ufshcd_host_reset_and_restore - reset and restore host controller
6270 * @hba: per-adapter instance
6271 *
6272 * Note that host controller reset may issue DME_RESET to
6273 * local and remote (device) Uni-Pro stack and the attributes
6274 * are reset to default state.
6275 *
6276 * Returns zero on success, non-zero on failure
6277 */
6278static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
6279{
6280 int err;
3441da7d
SRT		 6281 unsigned long flags;
6282
2df74b69
CG		 6283 /*
6284 * Stop the host controller and complete the requests
6285 * cleared by h/w
6286 */
3441da7d 6287 spin_lock_irqsave(hba->host->host_lock, flags);
596585a2 6288 ufshcd_hba_stop(hba, false);
2df74b69
CG		 6289 hba->silence_err_logs = true;
6290 ufshcd_complete_requests(hba);
6291 hba->silence_err_logs = false;
3441da7d
SRT		 6292 spin_unlock_irqrestore(hba->host->host_lock, flags);
6293
a3cd5ec5
SJ		 6294 /* scale up clocks to max frequency before full reinitialization */
6295 ufshcd_scale_clks(hba, true);
6296
3441da7d
SRT		 6297 err = ufshcd_hba_enable(hba);
6298 if (err)
6299 goto out;
6300
6301 /* Establish the link again and restore the device */
1b9e2141 6302 err = ufshcd_probe_hba(hba, false);
1d337ec2
SRT		 6303
6304 if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL))
3441da7d
SRT		 6305 err = -EIO;
6306out:
6307 if (err)
6308 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
8808b4e9 6309 ufshcd_update_reg_hist(&hba->ufs_stats.host_reset, (u32)err);
3441da7d
SRT		 6310 return err;
6311}
6312
6313/**
6314 * ufshcd_reset_and_restore - reset and re-initialize host/device
6315 * @hba: per-adapter instance
6316 *
6317 * Reset and recover device, host and re-establish link. This
6318 * is helpful to recover the communication in fatal error conditions.
6319 *
6320 * Returns zero on success, non-zero on failure
6321 */
6322static int ufshcd_reset_and_restore(struct ufs_hba *hba)
6323{
6324 int err = 0;
1d337ec2 6325 int retries = MAX_HOST_RESET_RETRIES;
3441da7d 6326
1d337ec2 6327 do {
d8d9f793
BA		 6328 /* Reset the attached device */
6329 ufshcd_vops_device_reset(hba);
6330
1d337ec2
SRT		 6331 err = ufshcd_host_reset_and_restore(hba);
6332 } while (err && --retries);
3441da7d 6333
3441da7d
SRT		 6334 return err;
6335}
6336
6337/**
6338 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
8aa29f19 6339 * @cmd: SCSI command pointer
3441da7d
SRT		 6340 *
6341 * Returns SUCCESS/FAILED
6342 */
6343static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
6344{
6345 int err;
6346 unsigned long flags;
6347 struct ufs_hba *hba;
6348
6349 hba = shost_priv(cmd->device->host);
6350
1ab27c9c 6351 ufshcd_hold(hba, false);
3441da7d
SRT		 6352 /*
6353 * Check if there is any race with fatal error handling.
6354 * If so, wait for it to complete. Even though fatal error
6355 * handling does reset and restore in some cases, don't assume
6356 * anything out of it. We are just avoiding race here.
6357 */
6358 do {
6359 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 6360 if (!(work_pending(&hba->eh_work) ||
8dc0da79
ZL		 6361 hba->ufshcd_state == UFSHCD_STATE_RESET ||
6362 hba->ufshcd_state == UFSHCD_STATE_EH_SCHEDULED))
3441da7d
SRT		 6363 break;
6364 spin_unlock_irqrestore(hba->host->host_lock, flags);
6365 dev_dbg(hba->dev, "%s: reset in progress\n", __func__);
e8e7f271 6366 flush_work(&hba->eh_work);
3441da7d
SRT		 6367 } while (1);
6368
6369 hba->ufshcd_state = UFSHCD_STATE_RESET;
6370 ufshcd_set_eh_in_progress(hba);
6371 spin_unlock_irqrestore(hba->host->host_lock, flags);
6372
6373 err = ufshcd_reset_and_restore(hba);
6374
6375 spin_lock_irqsave(hba->host->host_lock, flags);
6376 if (!err) {
6377 err = SUCCESS;
6378 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6379 } else {
6380 err = FAILED;
6381 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6382 }
6383 ufshcd_clear_eh_in_progress(hba);
6384 spin_unlock_irqrestore(hba->host->host_lock, flags);
6385
1ab27c9c 6386 ufshcd_release(hba);
3441da7d
SRT		 6387 return err;
6388}
6389
3a4bf06d
YG		 6390/**
6391 * ufshcd_get_max_icc_level - calculate the ICC level
6392 * @sup_curr_uA: max. current supported by the regulator
6393 * @start_scan: row at the desc table to start scan from
6394 * @buff: power descriptor buffer
6395 *
6396 * Returns calculated max ICC level for specific regulator
6397 */
6398static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
6399{
6400 int i;
6401 int curr_uA;
6402 u16 data;
6403 u16 unit;
6404
6405 for (i = start_scan; i >= 0; i--) {
d79713f9 6406 data = be16_to_cpup((__be16 *)&buff[2 * i]);
3a4bf06d
YG		 6407 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
6408 ATTR_ICC_LVL_UNIT_OFFSET;
6409 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
6410 switch (unit) {
6411 case UFSHCD_NANO_AMP:
6412 curr_uA = curr_uA / 1000;
6413 break;
6414 case UFSHCD_MILI_AMP:
6415 curr_uA = curr_uA * 1000;
6416 break;
6417 case UFSHCD_AMP:
6418 curr_uA = curr_uA * 1000 * 1000;
6419 break;
6420 case UFSHCD_MICRO_AMP:
6421 default:
6422 break;
6423 }
6424 if (sup_curr_uA >= curr_uA)
6425 break;
6426 }
6427 if (i < 0) {
6428 i = 0;
6429 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
6430 }
6431
6432 return (u32)i;
6433}
6434
6435/**
6436 * ufshcd_calc_icc_level - calculate the max ICC level
6437 * In case regulators are not initialized we'll return 0
6438 * @hba: per-adapter instance
6439 * @desc_buf: power descriptor buffer to extract ICC levels from.
6440 * @len: length of desc_buff
6441 *
6442 * Returns calculated ICC level
6443 */
6444static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
6445 u8 *desc_buf, int len)
6446{
6447 u32 icc_level = 0;
6448
6449 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
6450 !hba->vreg_info.vccq2) {
6451 dev_err(hba->dev,
6452 "%s: Regulator capability was not set, actvIccLevel=%d",
6453 __func__, icc_level);
6454 goto out;
6455 }
6456
0487fff7 6457 if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
3a4bf06d
YG		 6458 icc_level = ufshcd_get_max_icc_level(
6459 hba->vreg_info.vcc->max_uA,
6460 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
6461 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
6462
0487fff7 6463 if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
3a4bf06d
YG		 6464 icc_level = ufshcd_get_max_icc_level(
6465 hba->vreg_info.vccq->max_uA,
6466 icc_level,
6467 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
6468
0487fff7 6469 if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
3a4bf06d
YG		 6470 icc_level = ufshcd_get_max_icc_level(
6471 hba->vreg_info.vccq2->max_uA,
6472 icc_level,
6473 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
6474out:
6475 return icc_level;
6476}
6477
e89860f1 6478static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
3a4bf06d
YG		 6479{
6480 int ret;
a4b0e8a4 6481 int buff_len = hba->desc_size.pwr_desc;
bbe21d7a 6482 u8 *desc_buf;
e89860f1 6483 u32 icc_level;
bbe21d7a
KC		 6484
6485 desc_buf = kmalloc(buff_len, GFP_KERNEL);
6486 if (!desc_buf)
6487 return;
3a4bf06d 6488
8c9a51b0
BH		 6489 ret = ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0,
6490 desc_buf, buff_len);
3a4bf06d
YG		 6491 if (ret) {
6492 dev_err(hba->dev,
6493 "%s: Failed reading power descriptor.len = %d ret = %d",
6494 __func__, buff_len, ret);
bbe21d7a 6495 goto out;
3a4bf06d
YG		 6496 }
6497
e89860f1
CG		 6498 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf,
6499 buff_len);
6500 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
3a4bf06d 6501
dbd34a61 6502 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
e89860f1 6503 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
3a4bf06d
YG		 6504
6505 if (ret)
6506 dev_err(hba->dev,
6507 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
e89860f1 6508 __func__, icc_level, ret);
3a4bf06d 6509
bbe21d7a
KC		 6510out:
6511 kfree(desc_buf);
3a4bf06d
YG		 6512}
6513
fb276f77
CG		 6514static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
6515{
6516 scsi_autopm_get_device(sdev);
6517 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
6518 if (sdev->rpm_autosuspend)
6519 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
6520 RPM_AUTOSUSPEND_DELAY_MS);
6521 scsi_autopm_put_device(sdev);
6522}
6523
2a8fa600
SJ		 6524/**
6525 * ufshcd_scsi_add_wlus - Adds required W-LUs
6526 * @hba: per-adapter instance
6527 *
6528 * UFS device specification requires the UFS devices to support 4 well known
6529 * logical units:
6530 * "REPORT_LUNS" (address: 01h)
6531 * "UFS Device" (address: 50h)
6532 * "RPMB" (address: 44h)
6533 * "BOOT" (address: 30h)
6534 * UFS device's power management needs to be controlled by "POWER CONDITION"
6535 * field of SSU (START STOP UNIT) command. But this "power condition" field
6536 * will take effect only when its sent to "UFS device" well known logical unit
6537 * hence we require the scsi_device instance to represent this logical unit in
6538 * order for the UFS host driver to send the SSU command for power management.
8aa29f19 6539 *
2a8fa600
SJ		 6540 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
6541 * Block) LU so user space process can control this LU. User space may also
6542 * want to have access to BOOT LU.
8aa29f19 6543 *
2a8fa600
SJ		 6544 * This function adds scsi device instances for each of all well known LUs
6545 * (except "REPORT LUNS" LU).
6546 *
6547 * Returns zero on success (all required W-LUs are added successfully),
6548 * non-zero error value on failure (if failed to add any of the required W-LU).
6549 */
6550static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
6551{
6552 int ret = 0;
7c48bfd0
AM		 6553 struct scsi_device *sdev_rpmb;
6554 struct scsi_device *sdev_boot;
2a8fa600
SJ		 6555
6556 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
6557 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
6558 if (IS_ERR(hba->sdev_ufs_device)) {
6559 ret = PTR_ERR(hba->sdev_ufs_device);
6560 hba->sdev_ufs_device = NULL;
6561 goto out;
6562 }
fb276f77 6563 ufshcd_blk_pm_runtime_init(hba->sdev_ufs_device);
7c48bfd0 6564 scsi_device_put(hba->sdev_ufs_device);
2a8fa600 6565
7c48bfd0 6566 sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
2a8fa600 6567 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7c48bfd0
AM		 6568 if (IS_ERR(sdev_rpmb)) {
6569 ret = PTR_ERR(sdev_rpmb);
3d21fbde 6570 goto remove_sdev_ufs_device;
2a8fa600 6571 }
fb276f77 6572 ufshcd_blk_pm_runtime_init(sdev_rpmb);
7c48bfd0 6573 scsi_device_put(sdev_rpmb);
3d21fbde
HK		 6574
6575 sdev_boot = __scsi_add_device(hba->host, 0, 0,
6576 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
fb276f77 6577 if (IS_ERR(sdev_boot)) {
3d21fbde 6578 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
fb276f77
CG		 6579 } else {
6580 ufshcd_blk_pm_runtime_init(sdev_boot);
3d21fbde 6581 scsi_device_put(sdev_boot);
fb276f77 6582 }
2a8fa600
SJ		 6583 goto out;
6584
2a8fa600
SJ		 6585remove_sdev_ufs_device:
6586 scsi_remove_device(hba->sdev_ufs_device);
6587out:
6588 return ret;
6589}
6590
09750066 6591static int ufs_get_device_desc(struct ufs_hba *hba)
c58ab7aa
YG		 6592{
6593 int err;
bbe21d7a 6594 size_t buff_len;
c58ab7aa 6595 u8 model_index;
bbe21d7a 6596 u8 *desc_buf;
09750066 6597 struct ufs_dev_info *dev_info = &hba->dev_info;
4b828fe1 6598
bbe21d7a
KC		 6599 buff_len = max_t(size_t, hba->desc_size.dev_desc,
6600 QUERY_DESC_MAX_SIZE + 1);
6601 desc_buf = kmalloc(buff_len, GFP_KERNEL);
6602 if (!desc_buf) {
6603 err = -ENOMEM;
6604 goto out;
6605 }
c58ab7aa 6606
8c9a51b0
BH		 6607 err = ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, desc_buf,
6608 hba->desc_size.dev_desc);
c58ab7aa
YG		 6609 if (err) {
6610 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
6611 __func__, err);
6612 goto out;
6613 }
6614
6615 /*
6616 * getting vendor (manufacturerID) and Bank Index in big endian
6617 * format
6618 */
09750066 6619 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
c58ab7aa
YG		 6620 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
6621
09f17791
CG		 6622 /* getting Specification Version in big endian format */
6623 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
6624 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
6625
c58ab7aa 6626 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
4b828fe1 6627 err = ufshcd_read_string_desc(hba, model_index,
09750066 6628 &dev_info->model, SD_ASCII_STD);
4b828fe1 6629 if (err < 0) {
c58ab7aa
YG		 6630 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
6631 __func__, err);
6632 goto out;
6633 }
6634
4b828fe1
TW		 6635 /*
6636 * ufshcd_read_string_desc returns size of the string
6637 * reset the error value
6638 */
6639 err = 0;
c58ab7aa
YG		 6640
6641out:
bbe21d7a 6642 kfree(desc_buf);
c58ab7aa
YG		 6643 return err;
6644}
6645
09750066 6646static void ufs_put_device_desc(struct ufs_hba *hba)
4b828fe1 6647{
09750066
BH		 6648 struct ufs_dev_info *dev_info = &hba->dev_info;
6649
6650 kfree(dev_info->model);
6651 dev_info->model = NULL;
4b828fe1
TW		 6652}
6653
09750066 6654static void ufs_fixup_device_setup(struct ufs_hba *hba)
c58ab7aa 6655{
c58ab7aa 6656 struct ufs_dev_fix *f;
09750066 6657 struct ufs_dev_info *dev_info = &hba->dev_info;
c58ab7aa
YG		 6658
6659 for (f = ufs_fixups; f->quirk; f++) {
09750066
BH		 6660 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
6661 f->wmanufacturerid == UFS_ANY_VENDOR) &&
6662 ((dev_info->model &&
6663 STR_PRFX_EQUAL(f->model, dev_info->model)) ||
6664 !strcmp(f->model, UFS_ANY_MODEL)))
c58ab7aa
YG		 6665 hba->dev_quirks |= f->quirk;
6666 }
6667}
6668
37113106
YG		 6669/**
6670 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6671 * @hba: per-adapter instance
6672 *
6673 * PA_TActivate parameter can be tuned manually if UniPro version is less than
6674 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6675 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6676 * the hibern8 exit latency.
6677 *
6678 * Returns zero on success, non-zero error value on failure.
6679 */
6680static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
6681{
6682 int ret = 0;
6683 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
6684
6685 ret = ufshcd_dme_peer_get(hba,
6686 UIC_ARG_MIB_SEL(
6687 RX_MIN_ACTIVATETIME_CAPABILITY,
6688 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6689 &peer_rx_min_activatetime);
6690 if (ret)
6691 goto out;
6692
6693 /* make sure proper unit conversion is applied */
6694 tuned_pa_tactivate =
6695 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
6696 / PA_TACTIVATE_TIME_UNIT_US);
6697 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6698 tuned_pa_tactivate);
6699
6700out:
6701 return ret;
6702}
6703
6704/**
6705 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6706 * @hba: per-adapter instance
6707 *
6708 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6709 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6710 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6711 * This optimal value can help reduce the hibern8 exit latency.
6712 *
6713 * Returns zero on success, non-zero error value on failure.
6714 */
6715static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
6716{
6717 int ret = 0;
6718 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
6719 u32 max_hibern8_time, tuned_pa_hibern8time;
6720
6721 ret = ufshcd_dme_get(hba,
6722 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
6723 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
6724 &local_tx_hibern8_time_cap);
6725 if (ret)
6726 goto out;
6727
6728 ret = ufshcd_dme_peer_get(hba,
6729 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
6730 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6731 &peer_rx_hibern8_time_cap);
6732 if (ret)
6733 goto out;
6734
6735 max_hibern8_time = max(local_tx_hibern8_time_cap,
6736 peer_rx_hibern8_time_cap);
6737 /* make sure proper unit conversion is applied */
6738 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
6739 / PA_HIBERN8_TIME_UNIT_US);
6740 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
6741 tuned_pa_hibern8time);
6742out:
6743 return ret;
6744}
6745
c6a6db43
SJ		 6746/**
6747 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6748 * less than device PA_TACTIVATE time.
6749 * @hba: per-adapter instance
6750 *
6751 * Some UFS devices require host PA_TACTIVATE to be lower than device
6752 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6753 * for such devices.
6754 *
6755 * Returns zero on success, non-zero error value on failure.
6756 */
6757static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
6758{
6759 int ret = 0;
6760 u32 granularity, peer_granularity;
6761 u32 pa_tactivate, peer_pa_tactivate;
6762 u32 pa_tactivate_us, peer_pa_tactivate_us;
6763 u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
6764
6765 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6766 &granularity);
6767 if (ret)
6768 goto out;
6769
6770 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6771 &peer_granularity);
6772 if (ret)
6773 goto out;
6774
6775 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
6776 (granularity > PA_GRANULARITY_MAX_VAL)) {
6777 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
6778 __func__, granularity);
6779 return -EINVAL;
6780 }
6781
6782 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
6783 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
6784 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
6785 __func__, peer_granularity);
6786 return -EINVAL;
6787 }
6788
6789 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
6790 if (ret)
6791 goto out;
6792
6793 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
6794 &peer_pa_tactivate);
6795 if (ret)
6796 goto out;
6797
6798 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
6799 peer_pa_tactivate_us = peer_pa_tactivate *
6800 gran_to_us_table[peer_granularity - 1];
6801
6802 if (pa_tactivate_us > peer_pa_tactivate_us) {
6803 u32 new_peer_pa_tactivate;
6804
6805 new_peer_pa_tactivate = pa_tactivate_us /
6806 gran_to_us_table[peer_granularity - 1];
6807 new_peer_pa_tactivate++;
6808 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6809 new_peer_pa_tactivate);
6810 }
6811
6812out:
6813 return ret;
6814}
6815
09750066 6816static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
37113106
YG		 6817{
6818 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
6819 ufshcd_tune_pa_tactivate(hba);
6820 ufshcd_tune_pa_hibern8time(hba);
6821 }
6822
e91ed9e0
CG		 6823 ufshcd_vops_apply_dev_quirks(hba);
6824
37113106
YG		 6825 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
6826 /* set 1ms timeout for PA_TACTIVATE */
6827 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
c6a6db43
SJ		 6828
6829 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
6830 ufshcd_quirk_tune_host_pa_tactivate(hba);
37113106
YG		 6831}
6832
ff8e20c6
DR		 6833static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
6834{
ff8e20c6
DR		 6835 hba->ufs_stats.hibern8_exit_cnt = 0;
6836 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7fabb77b 6837 hba->req_abort_count = 0;
ff8e20c6
DR		 6838}
6839
a4b0e8a4
PM		 6840static void ufshcd_init_desc_sizes(struct ufs_hba *hba)
6841{
6842 int err;
6843
6844 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0,
6845 &hba->desc_size.dev_desc);
6846 if (err)
6847 hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
6848
6849 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0,
6850 &hba->desc_size.pwr_desc);
6851 if (err)
6852 hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
6853
6854 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0,
6855 &hba->desc_size.interc_desc);
6856 if (err)
6857 hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
6858
6859 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0,
6860 &hba->desc_size.conf_desc);
6861 if (err)
6862 hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
6863
6864 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0,
6865 &hba->desc_size.unit_desc);
6866 if (err)
6867 hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
6868
6869 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0,
6870 &hba->desc_size.geom_desc);
6871 if (err)
6872 hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
059efd84 6873
c648c2d2
SN		 6874 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0,
6875 &hba->desc_size.hlth_desc);
6876 if (err)
6877 hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
a4b0e8a4
PM		 6878}
6879
731f0621
BH		 6880static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
6881{
6882 int err;
6883 size_t buff_len;
6884 u8 *desc_buf;
6885
6886 buff_len = hba->desc_size.geom_desc;
6887 desc_buf = kmalloc(buff_len, GFP_KERNEL);
6888 if (!desc_buf) {
6889 err = -ENOMEM;
6890 goto out;
6891 }
6892
6893 err = ufshcd_read_desc(hba, QUERY_DESC_IDN_GEOMETRY, 0,
6894 desc_buf, buff_len);
6895 if (err) {
6896 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
6897 __func__, err);
6898 goto out;
6899 }
6900
6901 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
6902 hba->dev_info.max_lu_supported = 32;
6903 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
6904 hba->dev_info.max_lu_supported = 8;
6905
6906out:
6907 kfree(desc_buf);
6908 return err;
6909}
6910
9e1e8a75
SJ		 6911static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
6912 {19200000, REF_CLK_FREQ_19_2_MHZ},
6913 {26000000, REF_CLK_FREQ_26_MHZ},
6914 {38400000, REF_CLK_FREQ_38_4_MHZ},
6915 {52000000, REF_CLK_FREQ_52_MHZ},
6916 {0, REF_CLK_FREQ_INVAL},
6917};
6918
6919static enum ufs_ref_clk_freq
6920ufs_get_bref_clk_from_hz(unsigned long freq)
6921{
6922 int i;
6923
6924 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
6925 if (ufs_ref_clk_freqs[i].freq_hz == freq)
6926 return ufs_ref_clk_freqs[i].val;
6927
6928 return REF_CLK_FREQ_INVAL;
6929}
6930
6931void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
6932{
6933 unsigned long freq;
6934
6935 freq = clk_get_rate(refclk);
6936
6937 hba->dev_ref_clk_freq =
6938 ufs_get_bref_clk_from_hz(freq);
6939
6940 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
6941 dev_err(hba->dev,
6942 "invalid ref_clk setting = %ld\n", freq);
6943}
6944
6945static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
6946{
6947 int err;
6948 u32 ref_clk;
6949 u32 freq = hba->dev_ref_clk_freq;
6950
6951 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6952 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
6953
6954 if (err) {
6955 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
6956 err);
6957 goto out;
6958 }
6959
6960 if (ref_clk == freq)
6961 goto out; /* nothing to update */
6962
6963 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
6964 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
6965
6966 if (err) {
6967 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
6968 ufs_ref_clk_freqs[freq].freq_hz);
6969 goto out;
6970 }
6971
6972 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
6973 ufs_ref_clk_freqs[freq].freq_hz);
6974
6975out:
6976 return err;
6977}
6978
1b9e2141
BH		 6979static int ufshcd_device_params_init(struct ufs_hba *hba)
6980{
6981 bool flag;
6982 int ret;
6983
731f0621
BH		 6984 /* Clear any previous UFS device information */
6985 memset(&hba->dev_info, 0, sizeof(hba->dev_info));
6986
1b9e2141
BH		 6987 /* Init check for device descriptor sizes */
6988 ufshcd_init_desc_sizes(hba);
6989
731f0621
BH		 6990 /* Init UFS geometry descriptor related parameters */
6991 ret = ufshcd_device_geo_params_init(hba);
6992 if (ret)
6993 goto out;
6994
1b9e2141
BH		 6995 /* Check and apply UFS device quirks */
6996 ret = ufs_get_device_desc(hba);
6997 if (ret) {
6998 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
6999 __func__, ret);
7000 goto out;
7001 }
7002
09f17791
CG		 7003 ufshcd_get_ref_clk_gating_wait(hba);
7004
1b9e2141
BH		 7005 ufs_fixup_device_setup(hba);
7006
1b9e2141
BH		 7007 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
7008 QUERY_FLAG_IDN_PWR_ON_WPE, &flag))
7009 hba->dev_info.f_power_on_wp_en = flag;
7010
2b35b2ad
BH		 7011 /* Probe maximum power mode co-supported by both UFS host and device */
7012 if (ufshcd_get_max_pwr_mode(hba))
7013 dev_err(hba->dev,
7014 "%s: Failed getting max supported power mode\n",
7015 __func__);
1b9e2141
BH		 7016out:
7017 return ret;
7018}
7019
7020/**
7021 * ufshcd_add_lus - probe and add UFS logical units
7022 * @hba: per-adapter instance
7023 */
7024static int ufshcd_add_lus(struct ufs_hba *hba)
7025{
7026 int ret;
7027
1b9e2141
BH		 7028 /* Add required well known logical units to scsi mid layer */
7029 ret = ufshcd_scsi_add_wlus(hba);
7030 if (ret)
7031 goto out;
7032
7033 /* Initialize devfreq after UFS device is detected */
7034 if (ufshcd_is_clkscaling_supported(hba)) {
7035 memcpy(&hba->clk_scaling.saved_pwr_info.info,
7036 &hba->pwr_info,
7037 sizeof(struct ufs_pa_layer_attr));
7038 hba->clk_scaling.saved_pwr_info.is_valid = true;
7039 if (!hba->devfreq) {
7040 ret = ufshcd_devfreq_init(hba);
7041 if (ret)
7042 goto out;
7043 }
7044
7045 hba->clk_scaling.is_allowed = true;
7046 }
7047
7048 ufs_bsg_probe(hba);
7049 scsi_scan_host(hba->host);
7050 pm_runtime_put_sync(hba->dev);
7051
1b9e2141
BH		 7052out:
7053 return ret;
7054}
7055
6ccf44fe 7056/**
1d337ec2
SRT		 7057 * ufshcd_probe_hba - probe hba to detect device and initialize
7058 * @hba: per-adapter instance
1b9e2141 7059 * @async: asynchronous execution or not
1d337ec2
SRT		 7060 *
7061 * Execute link-startup and verify device initialization
6ccf44fe 7062 */
1b9e2141 7063static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
6ccf44fe 7064{
6ccf44fe 7065 int ret;
7ff5ab47 7066 ktime_t start = ktime_get();
6ccf44fe
SJ		 7067
7068 ret = ufshcd_link_startup(hba);
5a0b0cb9
SRT		 7069 if (ret)
7070 goto out;
7071
afdfff59
YG		 7072 /* set the default level for urgent bkops */
7073 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
7074 hba->is_urgent_bkops_lvl_checked = false;
7075
ff8e20c6
DR		 7076 /* Debug counters initialization */
7077 ufshcd_clear_dbg_ufs_stats(hba);
7078
57d104c1
SJ		 7079 /* UniPro link is active now */
7080 ufshcd_set_link_active(hba);
d3e89bac 7081
1b9e2141 7082 /* Verify device initialization by sending NOP OUT UPIU */
5a0b0cb9
SRT		 7083 ret = ufshcd_verify_dev_init(hba);
7084 if (ret)
7085 goto out;
68078d5c 7086
1b9e2141 7087 /* Initiate UFS initialization, and waiting until completion */
68078d5c
DR		 7088 ret = ufshcd_complete_dev_init(hba);
7089 if (ret)
7090 goto out;
5a0b0cb9 7091
1b9e2141
BH		 7092 /*
7093 * Initialize UFS device parameters used by driver, these
7094 * parameters are associated with UFS descriptors.
7095 */
7096 if (async) {
7097 ret = ufshcd_device_params_init(hba);
7098 if (ret)
7099 goto out;
93fdd5ac
TW		 7100 }
7101
09750066 7102 ufshcd_tune_unipro_params(hba);
4b828fe1 7103
57d104c1
SJ		 7104 /* UFS device is also active now */
7105 ufshcd_set_ufs_dev_active(hba);
66ec6d59 7106 ufshcd_force_reset_auto_bkops(hba);
57d104c1
SJ		 7107 hba->wlun_dev_clr_ua = true;
7108
2b35b2ad
BH		 7109 /* Gear up to HS gear if supported */
7110 if (hba->max_pwr_info.is_valid) {
9e1e8a75
SJ		 7111 /*
7112 * Set the right value to bRefClkFreq before attempting to
7113 * switch to HS gears.
7114 */
7115 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
7116 ufshcd_set_dev_ref_clk(hba);
7eb584db 7117 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8643ae66 7118 if (ret) {
7eb584db
DR		 7119 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
7120 __func__, ret);
8643ae66
DL		 7121 goto out;
7122 }
7eb584db 7123 }
57d104c1 7124
e89860f1
CG		 7125 /*
7126 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
7127 * and for removable UFS card as well, hence always set the parameter.
7128 * Note: Error handler may issue the device reset hence resetting
7129 * bActiveICCLevel as well so it is always safe to set this here.
7130 */
7131 ufshcd_set_active_icc_lvl(hba);
7132
53c12d0e
YG		 7133 /* set the state as operational after switching to desired gear */
7134 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
a4b0e8a4 7135
71d848b8
CG		 7136 /* Enable Auto-Hibernate if configured */
7137 ufshcd_auto_hibern8_enable(hba);
7138
5a0b0cb9 7139out:
1d337ec2 7140
7ff5ab47
SJ		 7141 trace_ufshcd_init(dev_name(hba->dev), ret,
7142 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 7143 hba->curr_dev_pwr_mode, hba->uic_link_state);
1d337ec2
SRT		 7144 return ret;
7145}
7146
7147/**
7148 * ufshcd_async_scan - asynchronous execution for probing hba
7149 * @data: data pointer to pass to this function
7150 * @cookie: cookie data
7151 */
7152static void ufshcd_async_scan(void *data, async_cookie_t cookie)
7153{
7154 struct ufs_hba *hba = (struct ufs_hba *)data;
1b9e2141 7155 int ret;
1d337ec2 7156
1b9e2141
BH		 7157 /* Initialize hba, detect and initialize UFS device */
7158 ret = ufshcd_probe_hba(hba, true);
7159 if (ret)
7160 goto out;
7161
7162 /* Probe and add UFS logical units */
7163 ret = ufshcd_add_lus(hba);
7164out:
7165 /*
7166 * If we failed to initialize the device or the device is not
7167 * present, turn off the power/clocks etc.
7168 */
7169 if (ret) {
7170 pm_runtime_put_sync(hba->dev);
7171 ufshcd_exit_clk_scaling(hba);
7172 ufshcd_hba_exit(hba);
7173 }
6ccf44fe
SJ		 7174}
7175
d829fc8a
SN		 7176static const struct attribute_group *ufshcd_driver_groups[] = {
7177 &ufs_sysfs_unit_descriptor_group,
ec92b59c 7178 &ufs_sysfs_lun_attributes_group,
d829fc8a
SN		 7179 NULL,
7180};
7181
7a3e97b0
SY		 7182static struct scsi_host_template ufshcd_driver_template = {
7183 .module = THIS_MODULE,
7184 .name = UFSHCD,
7185 .proc_name = UFSHCD,
7186 .queuecommand = ufshcd_queuecommand,
7187 .slave_alloc = ufshcd_slave_alloc,
eeda4749 7188 .slave_configure = ufshcd_slave_configure,
7a3e97b0 7189 .slave_destroy = ufshcd_slave_destroy,
4264fd61 7190 .change_queue_depth = ufshcd_change_queue_depth,
7a3e97b0 7191 .eh_abort_handler = ufshcd_abort,
3441da7d
SRT		 7192 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
7193 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
7a3e97b0
SY		 7194 .this_id = -1,
7195 .sg_tablesize = SG_ALL,
7196 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
7197 .can_queue = UFSHCD_CAN_QUEUE,
552a990c 7198 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
1ab27c9c 7199 .max_host_blocked = 1,
c40ecc12 7200 .track_queue_depth = 1,
d829fc8a 7201 .sdev_groups = ufshcd_driver_groups,
4af14d11 7202 .dma_boundary = PAGE_SIZE - 1,
49615ba1 7203 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS,
7a3e97b0
SY		 7204};
7205
57d104c1
SJ		 7206static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
7207 int ua)
7208{
7b16a07c 7209 int ret;
57d104c1 7210
7b16a07c
BA		 7211 if (!vreg)
7212 return 0;
57d104c1 7213
0487fff7
SC		 7214 /*
7215 * "set_load" operation shall be required on those regulators
7216 * which specifically configured current limitation. Otherwise
7217 * zero max_uA may cause unexpected behavior when regulator is
7218 * enabled or set as high power mode.
7219 */
7220 if (!vreg->max_uA)
7221 return 0;
7222
7b16a07c
BA		 7223 ret = regulator_set_load(vreg->reg, ua);
7224 if (ret < 0) {
7225 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
7226 __func__, vreg->name, ua, ret);
57d104c1
SJ		 7227 }
7228
7229 return ret;
7230}
7231
7232static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
7233 struct ufs_vreg *vreg)
7234{
73067981 7235 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
57d104c1
SJ		 7236}
7237
7238static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
7239 struct ufs_vreg *vreg)
7240{
7c7cfdcf
AH		 7241 if (!vreg)
7242 return 0;
7243
73067981 7244 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
57d104c1
SJ		 7245}
7246
aa497613
SRT		 7247static int ufshcd_config_vreg(struct device *dev,
7248 struct ufs_vreg *vreg, bool on)
7249{
7250 int ret = 0;
72753590
GS		 7251 struct regulator *reg;
7252 const char *name;
aa497613
SRT		 7253 int min_uV, uA_load;
7254
7255 BUG_ON(!vreg);
7256
72753590
GS		 7257 reg = vreg->reg;
7258 name = vreg->name;
7259
aa497613 7260 if (regulator_count_voltages(reg) > 0) {
90d88f47
AD		 7261 uA_load = on ? vreg->max_uA : 0;
7262 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
7263 if (ret)
7264 goto out;
7265
3b141e8c
SC		 7266 if (vreg->min_uV && vreg->max_uV) {
7267 min_uV = on ? vreg->min_uV : 0;
7268 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
7269 if (ret) {
7270 dev_err(dev,
7271 "%s: %s set voltage failed, err=%d\n",
aa497613 7272 __func__, name, ret);
3b141e8c
SC		 7273 goto out;
7274 }
aa497613 7275 }
aa497613
SRT		 7276 }
7277out:
7278 return ret;
7279}
7280
7281static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
7282{
7283 int ret = 0;
7284
73067981 7285 if (!vreg || vreg->enabled)
aa497613
SRT		 7286 goto out;
7287
7288 ret = ufshcd_config_vreg(dev, vreg, true);
7289 if (!ret)
7290 ret = regulator_enable(vreg->reg);
7291
7292 if (!ret)
7293 vreg->enabled = true;
7294 else
7295 dev_err(dev, "%s: %s enable failed, err=%d\n",
7296 __func__, vreg->name, ret);
7297out:
7298 return ret;
7299}
7300
7301static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
7302{
7303 int ret = 0;
7304
73067981 7305 if (!vreg || !vreg->enabled)
aa497613
SRT		 7306 goto out;
7307
7308 ret = regulator_disable(vreg->reg);
7309
7310 if (!ret) {
7311 /* ignore errors on applying disable config */
7312 ufshcd_config_vreg(dev, vreg, false);
7313 vreg->enabled = false;
7314 } else {
7315 dev_err(dev, "%s: %s disable failed, err=%d\n",
7316 __func__, vreg->name, ret);
7317 }
7318out:
7319 return ret;
7320}
7321
7322static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
7323{
7324 int ret = 0;
7325 struct device *dev = hba->dev;
7326 struct ufs_vreg_info *info = &hba->vreg_info;
7327
aa497613
SRT		 7328 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
7329 if (ret)
7330 goto out;
7331
7332 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
7333 if (ret)
7334 goto out;
7335
7336 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
7337 if (ret)
7338 goto out;
7339
7340out:
7341 if (ret) {
7342 ufshcd_toggle_vreg(dev, info->vccq2, false);
7343 ufshcd_toggle_vreg(dev, info->vccq, false);
7344 ufshcd_toggle_vreg(dev, info->vcc, false);
7345 }
7346 return ret;
7347}
7348
6a771a65
RS		 7349static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
7350{
7351 struct ufs_vreg_info *info = &hba->vreg_info;
7352
60b7b823 7353 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
6a771a65
RS		 7354}
7355
aa497613
SRT		 7356static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
7357{
7358 int ret = 0;
7359
7360 if (!vreg)
7361 goto out;
7362
7363 vreg->reg = devm_regulator_get(dev, vreg->name);
7364 if (IS_ERR(vreg->reg)) {
7365 ret = PTR_ERR(vreg->reg);
7366 dev_err(dev, "%s: %s get failed, err=%d\n",
7367 __func__, vreg->name, ret);
7368 }
7369out:
7370 return ret;
7371}
7372
7373static int ufshcd_init_vreg(struct ufs_hba *hba)
7374{
7375 int ret = 0;
7376 struct device *dev = hba->dev;
7377 struct ufs_vreg_info *info = &hba->vreg_info;
7378
aa497613
SRT		 7379 ret = ufshcd_get_vreg(dev, info->vcc);
7380 if (ret)
7381 goto out;
7382
7383 ret = ufshcd_get_vreg(dev, info->vccq);
7384 if (ret)
7385 goto out;
7386
7387 ret = ufshcd_get_vreg(dev, info->vccq2);
7388out:
7389 return ret;
7390}
7391
6a771a65
RS		 7392static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
7393{
7394 struct ufs_vreg_info *info = &hba->vreg_info;
7395
7396 if (info)
7397 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
7398
7399 return 0;
7400}
7401
57d104c1
SJ		 7402static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
7403 bool skip_ref_clk)
c6e79dac
SRT		 7404{
7405 int ret = 0;
7406 struct ufs_clk_info *clki;
7407 struct list_head *head = &hba->clk_list_head;
1ab27c9c 7408 unsigned long flags;
911a0771
SJ		 7409 ktime_t start = ktime_get();
7410 bool clk_state_changed = false;
c6e79dac 7411
566ec9ad 7412 if (list_empty(head))
c6e79dac
SRT		 7413 goto out;
7414
38f3242e
CG		 7415 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
7416 if (ret)
7417 return ret;
1e879e8f 7418
c6e79dac
SRT		 7419 list_for_each_entry(clki, head, list) {
7420 if (!IS_ERR_OR_NULL(clki->clk)) {
57d104c1
SJ		 7421 if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
7422 continue;
7423
911a0771 7424 clk_state_changed = on ^ clki->enabled;
c6e79dac
SRT		 7425 if (on && !clki->enabled) {
7426 ret = clk_prepare_enable(clki->clk);
7427 if (ret) {
7428 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
7429 __func__, clki->name, ret);
7430 goto out;
7431 }
7432 } else if (!on && clki->enabled) {
7433 clk_disable_unprepare(clki->clk);
7434 }
7435 clki->enabled = on;
7436 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
7437 clki->name, on ? "en" : "dis");
7438 }
7439 }
1ab27c9c 7440
38f3242e
CG		 7441 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
7442 if (ret)
7443 return ret;
1e879e8f 7444
c6e79dac
SRT		 7445out:
7446 if (ret) {
7447 list_for_each_entry(clki, head, list) {
7448 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
7449 clk_disable_unprepare(clki->clk);
7450 }
7ff5ab47 7451 } else if (!ret && on) {
1ab27c9c
ST		 7452 spin_lock_irqsave(hba->host->host_lock, flags);
7453 hba->clk_gating.state = CLKS_ON;
7ff5ab47
SJ		 7454 trace_ufshcd_clk_gating(dev_name(hba->dev),
7455 hba->clk_gating.state);
1ab27c9c 7456 spin_unlock_irqrestore(hba->host->host_lock, flags);
c6e79dac 7457 }
7ff5ab47 7458
911a0771
SJ		 7459 if (clk_state_changed)
7460 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
7461 (on ? "on" : "off"),
7462 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
c6e79dac
SRT		 7463 return ret;
7464}
7465
57d104c1
SJ		 7466static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
7467{
7468 return __ufshcd_setup_clocks(hba, on, false);
7469}
7470
c6e79dac
SRT		 7471static int ufshcd_init_clocks(struct ufs_hba *hba)
7472{
7473 int ret = 0;
7474 struct ufs_clk_info *clki;
7475 struct device *dev = hba->dev;
7476 struct list_head *head = &hba->clk_list_head;
7477
566ec9ad 7478 if (list_empty(head))
c6e79dac
SRT		 7479 goto out;
7480
7481 list_for_each_entry(clki, head, list) {
7482 if (!clki->name)
7483 continue;
7484
7485 clki->clk = devm_clk_get(dev, clki->name);
7486 if (IS_ERR(clki->clk)) {
7487 ret = PTR_ERR(clki->clk);
7488 dev_err(dev, "%s: %s clk get failed, %d\n",
7489 __func__, clki->name, ret);
7490 goto out;
7491 }
7492
9e1e8a75
SJ		 7493 /*
7494 * Parse device ref clk freq as per device tree "ref_clk".
7495 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
7496 * in ufshcd_alloc_host().
7497 */
7498 if (!strcmp(clki->name, "ref_clk"))
7499 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
7500
c6e79dac
SRT		 7501 if (clki->max_freq) {
7502 ret = clk_set_rate(clki->clk, clki->max_freq);
7503 if (ret) {
7504 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
7505 __func__, clki->name,
7506 clki->max_freq, ret);
7507 goto out;
7508 }
856b3483 7509 clki->curr_freq = clki->max_freq;
c6e79dac
SRT		 7510 }
7511 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
7512 clki->name, clk_get_rate(clki->clk));
7513 }
7514out:
7515 return ret;
7516}
7517
5c0c28a8
SRT		 7518static int ufshcd_variant_hba_init(struct ufs_hba *hba)
7519{
7520 int err = 0;
7521
7522 if (!hba->vops)
7523 goto out;
7524
0263bcd0
YG		 7525 err = ufshcd_vops_init(hba);
7526 if (err)
7527 goto out;
5c0c28a8 7528
0263bcd0
YG		 7529 err = ufshcd_vops_setup_regulators(hba, true);
7530 if (err)
7531 goto out_exit;
5c0c28a8
SRT		 7532
7533 goto out;
7534
5c0c28a8 7535out_exit:
0263bcd0 7536 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 7537out:
7538 if (err)
7539 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
0263bcd0 7540 __func__, ufshcd_get_var_name(hba), err);
5c0c28a8
SRT		 7541 return err;
7542}
7543
7544static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
7545{
7546 if (!hba->vops)
7547 return;
7548
0263bcd0 7549 ufshcd_vops_setup_regulators(hba, false);
5c0c28a8 7550
0263bcd0 7551 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 7552}
7553
aa497613
SRT		 7554static int ufshcd_hba_init(struct ufs_hba *hba)
7555{
7556 int err;
7557
6a771a65
RS		 7558 /*
7559 * Handle host controller power separately from the UFS device power
7560 * rails as it will help controlling the UFS host controller power
7561 * collapse easily which is different than UFS device power collapse.
7562 * Also, enable the host controller power before we go ahead with rest
7563 * of the initialization here.
7564 */
7565 err = ufshcd_init_hba_vreg(hba);
aa497613
SRT		 7566 if (err)
7567 goto out;
7568
6a771a65 7569 err = ufshcd_setup_hba_vreg(hba, true);
aa497613
SRT		 7570 if (err)
7571 goto out;
7572
6a771a65
RS		 7573 err = ufshcd_init_clocks(hba);
7574 if (err)
7575 goto out_disable_hba_vreg;
7576
7577 err = ufshcd_setup_clocks(hba, true);
7578 if (err)
7579 goto out_disable_hba_vreg;
7580
c6e79dac
SRT		 7581 err = ufshcd_init_vreg(hba);
7582 if (err)
7583 goto out_disable_clks;
7584
7585 err = ufshcd_setup_vreg(hba, true);
7586 if (err)
7587 goto out_disable_clks;
7588
aa497613
SRT		 7589 err = ufshcd_variant_hba_init(hba);
7590 if (err)
7591 goto out_disable_vreg;
7592
1d337ec2 7593 hba->is_powered = true;
aa497613
SRT		 7594 goto out;
7595
7596out_disable_vreg:
7597 ufshcd_setup_vreg(hba, false);
c6e79dac
SRT		 7598out_disable_clks:
7599 ufshcd_setup_clocks(hba, false);
6a771a65
RS		 7600out_disable_hba_vreg:
7601 ufshcd_setup_hba_vreg(hba, false);
aa497613
SRT		 7602out:
7603 return err;
7604}
7605
7606static void ufshcd_hba_exit(struct ufs_hba *hba)
7607{
1d337ec2
SRT		 7608 if (hba->is_powered) {
7609 ufshcd_variant_hba_exit(hba);
7610 ufshcd_setup_vreg(hba, false);
a508253d 7611 ufshcd_suspend_clkscaling(hba);
eebcc196 7612 if (ufshcd_is_clkscaling_supported(hba))
0701e49d
SJ		 7613 if (hba->devfreq)
7614 ufshcd_suspend_clkscaling(hba);
1d337ec2
SRT		 7615 ufshcd_setup_clocks(hba, false);
7616 ufshcd_setup_hba_vreg(hba, false);
7617 hba->is_powered = false;
09750066 7618 ufs_put_device_desc(hba);
1d337ec2 7619 }
aa497613
SRT		 7620}
7621
57d104c1
SJ		 7622static int
7623ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
7624{
7625 unsigned char cmd[6] = {REQUEST_SENSE,
7626 0,
7627 0,
7628 0,
09a5a24f 7629 UFS_SENSE_SIZE,
57d104c1
SJ		 7630 0};
7631 char *buffer;
7632 int ret;
7633
09a5a24f 7634 buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
57d104c1
SJ		 7635 if (!buffer) {
7636 ret = -ENOMEM;
7637 goto out;
7638 }
7639
fcbfffe2 7640 ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
09a5a24f 7641 UFS_SENSE_SIZE, NULL, NULL,
fcbfffe2 7642 msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
57d104c1
SJ		 7643 if (ret)
7644 pr_err("%s: failed with err %d\n", __func__, ret);
7645
7646 kfree(buffer);
7647out:
7648 return ret;
7649}
7650
7651/**
7652 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
7653 * power mode
7654 * @hba: per adapter instance
7655 * @pwr_mode: device power mode to set
7656 *
7657 * Returns 0 if requested power mode is set successfully
7658 * Returns non-zero if failed to set the requested power mode
7659 */
7660static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
7661 enum ufs_dev_pwr_mode pwr_mode)
7662{
7663 unsigned char cmd[6] = { START_STOP };
7664 struct scsi_sense_hdr sshdr;
7c48bfd0
AM		 7665 struct scsi_device *sdp;
7666 unsigned long flags;
57d104c1
SJ		 7667 int ret;
7668
7c48bfd0
AM		 7669 spin_lock_irqsave(hba->host->host_lock, flags);
7670 sdp = hba->sdev_ufs_device;
7671 if (sdp) {
7672 ret = scsi_device_get(sdp);
7673 if (!ret && !scsi_device_online(sdp)) {
7674 ret = -ENODEV;
7675 scsi_device_put(sdp);
7676 }
7677 } else {
7678 ret = -ENODEV;
7679 }
7680 spin_unlock_irqrestore(hba->host->host_lock, flags);
7681
7682 if (ret)
7683 return ret;
57d104c1
SJ		 7684
7685 /*
7686 * If scsi commands fail, the scsi mid-layer schedules scsi error-
7687 * handling, which would wait for host to be resumed. Since we know
7688 * we are functional while we are here, skip host resume in error
7689 * handling context.
7690 */
7691 hba->host->eh_noresume = 1;
7692 if (hba->wlun_dev_clr_ua) {
7693 ret = ufshcd_send_request_sense(hba, sdp);
7694 if (ret)
7695 goto out;
7696 /* Unit attention condition is cleared now */
7697 hba->wlun_dev_clr_ua = false;
7698 }
7699
7700 cmd[4] = pwr_mode << 4;
7701
7702 /*
7703 * Current function would be generally called from the power management
e8064021 7704 * callbacks hence set the RQF_PM flag so that it doesn't resume the
57d104c1
SJ		 7705 * already suspended childs.
7706 */
fcbfffe2
CH		 7707 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
7708 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
57d104c1
SJ		 7709 if (ret) {
7710 sdev_printk(KERN_WARNING, sdp,
ef61329d
HR		 7711 "START_STOP failed for power mode: %d, result %x\n",
7712 pwr_mode, ret);
c65be1a6 7713 if (driver_byte(ret) == DRIVER_SENSE)
21045519 7714 scsi_print_sense_hdr(sdp, NULL, &sshdr);
57d104c1
SJ		 7715 }
7716
7717 if (!ret)
7718 hba->curr_dev_pwr_mode = pwr_mode;
7719out:
7c48bfd0 7720 scsi_device_put(sdp);
57d104c1
SJ		 7721 hba->host->eh_noresume = 0;
7722 return ret;
7723}
7724
7725static int ufshcd_link_state_transition(struct ufs_hba *hba,
7726 enum uic_link_state req_link_state,
7727 int check_for_bkops)
7728{
7729 int ret = 0;
7730
7731 if (req_link_state == hba->uic_link_state)
7732 return 0;
7733
7734 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
7735 ret = ufshcd_uic_hibern8_enter(hba);
7736 if (!ret)
7737 ufshcd_set_link_hibern8(hba);
7738 else
7739 goto out;
7740 }
7741 /*
7742 * If autobkops is enabled, link can't be turned off because
7743 * turning off the link would also turn off the device.
7744 */
7745 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
dc30c9e6 7746 (!check_for_bkops || !hba->auto_bkops_enabled)) {
f3099fbd
YG		 7747 /*
7748 * Let's make sure that link is in low power mode, we are doing
7749 * this currently by putting the link in Hibern8. Otherway to
7750 * put the link in low power mode is to send the DME end point
7751 * to device and then send the DME reset command to local
7752 * unipro. But putting the link in hibern8 is much faster.
7753 */
7754 ret = ufshcd_uic_hibern8_enter(hba);
7755 if (ret)
7756 goto out;
57d104c1
SJ		 7757 /*
7758 * Change controller state to "reset state" which
7759 * should also put the link in off/reset state
7760 */
596585a2 7761 ufshcd_hba_stop(hba, true);
57d104c1
SJ		 7762 /*
7763 * TODO: Check if we need any delay to make sure that
7764 * controller is reset
7765 */
7766 ufshcd_set_link_off(hba);
7767 }
7768
7769out:
7770 return ret;
7771}
7772
7773static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
7774{
b799fdf7
YG		 7775 /*
7776 * It seems some UFS devices may keep drawing more than sleep current
7777 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7778 * To avoid this situation, add 2ms delay before putting these UFS
7779 * rails in LPM mode.
7780 */
7781 if (!ufshcd_is_link_active(hba) &&
7782 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
7783 usleep_range(2000, 2100);
7784
57d104c1
SJ		 7785 /*
7786 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7787 * power.
7788 *
7789 * If UFS device and link is in OFF state, all power supplies (VCC,
7790 * VCCQ, VCCQ2) can be turned off if power on write protect is not
7791 * required. If UFS link is inactive (Hibern8 or OFF state) and device
7792 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7793 *
7794 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7795 * in low power state which would save some power.
7796 */
7797 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7798 !hba->dev_info.is_lu_power_on_wp) {
7799 ufshcd_setup_vreg(hba, false);
7800 } else if (!ufshcd_is_ufs_dev_active(hba)) {
7801 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7802 if (!ufshcd_is_link_active(hba)) {
7803 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7804 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
7805 }
7806 }
7807}
7808
7809static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
7810{
7811 int ret = 0;
7812
7813 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7814 !hba->dev_info.is_lu_power_on_wp) {
7815 ret = ufshcd_setup_vreg(hba, true);
7816 } else if (!ufshcd_is_ufs_dev_active(hba)) {
57d104c1
SJ		 7817 if (!ret && !ufshcd_is_link_active(hba)) {
7818 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
7819 if (ret)
7820 goto vcc_disable;
7821 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
7822 if (ret)
7823 goto vccq_lpm;
7824 }
69d72ac8 7825 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
57d104c1
SJ		 7826 }
7827 goto out;
7828
7829vccq_lpm:
7830 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7831vcc_disable:
7832 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7833out:
7834 return ret;
7835}
7836
7837static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
7838{
7839 if (ufshcd_is_link_off(hba))
7840 ufshcd_setup_hba_vreg(hba, false);
7841}
7842
7843static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
7844{
7845 if (ufshcd_is_link_off(hba))
7846 ufshcd_setup_hba_vreg(hba, true);
7847}
7848
7a3e97b0 7849/**
57d104c1 7850 * ufshcd_suspend - helper function for suspend operations
3b1d0580 7851 * @hba: per adapter instance
57d104c1
SJ		 7852 * @pm_op: desired low power operation type
7853 *
7854 * This function will try to put the UFS device and link into low power
7855 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7856 * (System PM level).
7857 *
7858 * If this function is called during shutdown, it will make sure that
7859 * both UFS device and UFS link is powered off.
7a3e97b0 7860 *
57d104c1
SJ		 7861 * NOTE: UFS device & link must be active before we enter in this function.
7862 *
7863 * Returns 0 for success and non-zero for failure
7a3e97b0 7864 */
57d104c1 7865static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 7866{
57d104c1
SJ		 7867 int ret = 0;
7868 enum ufs_pm_level pm_lvl;
7869 enum ufs_dev_pwr_mode req_dev_pwr_mode;
7870 enum uic_link_state req_link_state;
7871
7872 hba->pm_op_in_progress = 1;
7873 if (!ufshcd_is_shutdown_pm(pm_op)) {
7874 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
7875 hba->rpm_lvl : hba->spm_lvl;
7876 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
7877 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
7878 } else {
7879 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
7880 req_link_state = UIC_LINK_OFF_STATE;
7881 }
7882
7a3e97b0 7883 /*
57d104c1
SJ		 7884 * If we can't transition into any of the low power modes
7885 * just gate the clocks.
7a3e97b0 7886 */
1ab27c9c
ST		 7887 ufshcd_hold(hba, false);
7888 hba->clk_gating.is_suspended = true;
7889
401f1e44
SJ		 7890 if (hba->clk_scaling.is_allowed) {
7891 cancel_work_sync(&hba->clk_scaling.suspend_work);
7892 cancel_work_sync(&hba->clk_scaling.resume_work);
7893 ufshcd_suspend_clkscaling(hba);
7894 }
d6fcf81a 7895
57d104c1
SJ		 7896 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
7897 req_link_state == UIC_LINK_ACTIVE_STATE) {
7898 goto disable_clks;
7899 }
7a3e97b0 7900
57d104c1
SJ		 7901 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
7902 (req_link_state == hba->uic_link_state))
d6fcf81a 7903 goto enable_gating;
57d104c1
SJ		 7904
7905 /* UFS device & link must be active before we enter in this function */
7906 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
7907 ret = -EINVAL;
d6fcf81a 7908 goto enable_gating;
57d104c1
SJ		 7909 }
7910
7911 if (ufshcd_is_runtime_pm(pm_op)) {
374a246e
SJ		 7912 if (ufshcd_can_autobkops_during_suspend(hba)) {
7913 /*
7914 * The device is idle with no requests in the queue,
7915 * allow background operations if bkops status shows
7916 * that performance might be impacted.
7917 */
7918 ret = ufshcd_urgent_bkops(hba);
7919 if (ret)
7920 goto enable_gating;
7921 } else {
7922 /* make sure that auto bkops is disabled */
7923 ufshcd_disable_auto_bkops(hba);
7924 }
57d104c1
SJ		 7925 }
7926
7927 if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) &&
7928 ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
7929 !ufshcd_is_runtime_pm(pm_op))) {
7930 /* ensure that bkops is disabled */
7931 ufshcd_disable_auto_bkops(hba);
7932 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
7933 if (ret)
1ab27c9c 7934 goto enable_gating;
57d104c1
SJ		 7935 }
7936
2824ec9f 7937 flush_work(&hba->eeh_work);
57d104c1
SJ		 7938 ret = ufshcd_link_state_transition(hba, req_link_state, 1);
7939 if (ret)
7940 goto set_dev_active;
7941
7942 ufshcd_vreg_set_lpm(hba);
7943
7944disable_clks:
7945 /*
7946 * Call vendor specific suspend callback. As these callbacks may access
7947 * vendor specific host controller register space call them before the
7948 * host clocks are ON.
7949 */
0263bcd0
YG		 7950 ret = ufshcd_vops_suspend(hba, pm_op);
7951 if (ret)
7952 goto set_link_active;
dcb6cec5
SC		 7953 /*
7954 * Disable the host irq as host controller as there won't be any
7955 * host controller transaction expected till resume.
7956 */
7957 ufshcd_disable_irq(hba);
57d104c1 7958
57d104c1
SJ		 7959 if (!ufshcd_is_link_active(hba))
7960 ufshcd_setup_clocks(hba, false);
7961 else
7962 /* If link is active, device ref_clk can't be switched off */
7963 __ufshcd_setup_clocks(hba, false, true);
7964
1ab27c9c 7965 hba->clk_gating.state = CLKS_OFF;
7ff5ab47 7966 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
dcb6cec5 7967
57d104c1
SJ		 7968 /* Put the host controller in low power mode if possible */
7969 ufshcd_hba_vreg_set_lpm(hba);
7970 goto out;
7971
57d104c1 7972set_link_active:
401f1e44
SJ		 7973 if (hba->clk_scaling.is_allowed)
7974 ufshcd_resume_clkscaling(hba);
57d104c1
SJ		 7975 ufshcd_vreg_set_hpm(hba);
7976 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
7977 ufshcd_set_link_active(hba);
7978 else if (ufshcd_is_link_off(hba))
7979 ufshcd_host_reset_and_restore(hba);
7980set_dev_active:
7981 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
7982 ufshcd_disable_auto_bkops(hba);
1ab27c9c 7983enable_gating:
401f1e44
SJ		 7984 if (hba->clk_scaling.is_allowed)
7985 ufshcd_resume_clkscaling(hba);
1ab27c9c
ST		 7986 hba->clk_gating.is_suspended = false;
7987 ufshcd_release(hba);
57d104c1
SJ		 7988out:
7989 hba->pm_op_in_progress = 0;
8808b4e9
SC		 7990 if (ret)
7991 ufshcd_update_reg_hist(&hba->ufs_stats.suspend_err, (u32)ret);
57d104c1 7992 return ret;
7a3e97b0
SY		 7993}
7994
7995/**
57d104c1 7996 * ufshcd_resume - helper function for resume operations
3b1d0580 7997 * @hba: per adapter instance
57d104c1 7998 * @pm_op: runtime PM or system PM
7a3e97b0 7999 *
57d104c1
SJ		 8000 * This function basically brings the UFS device, UniPro link and controller
8001 * to active state.
8002 *
8003 * Returns 0 for success and non-zero for failure
7a3e97b0 8004 */
57d104c1 8005static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 8006{
57d104c1
SJ		 8007 int ret;
8008 enum uic_link_state old_link_state;
8009
8010 hba->pm_op_in_progress = 1;
8011 old_link_state = hba->uic_link_state;
8012
8013 ufshcd_hba_vreg_set_hpm(hba);
8014 /* Make sure clocks are enabled before accessing controller */
8015 ret = ufshcd_setup_clocks(hba, true);
8016 if (ret)
8017 goto out;
8018
57d104c1 8019 /* enable the host irq as host controller would be active soon */
5231d38c 8020 ufshcd_enable_irq(hba);
57d104c1
SJ		 8021
8022 ret = ufshcd_vreg_set_hpm(hba);
8023 if (ret)
8024 goto disable_irq_and_vops_clks;
8025
7a3e97b0 8026 /*
57d104c1
SJ		 8027 * Call vendor specific resume callback. As these callbacks may access
8028 * vendor specific host controller register space call them when the
8029 * host clocks are ON.
7a3e97b0 8030 */
0263bcd0
YG		 8031 ret = ufshcd_vops_resume(hba, pm_op);
8032 if (ret)
8033 goto disable_vreg;
57d104c1
SJ		 8034
8035 if (ufshcd_is_link_hibern8(hba)) {
8036 ret = ufshcd_uic_hibern8_exit(hba);
8037 if (!ret)
8038 ufshcd_set_link_active(hba);
8039 else
8040 goto vendor_suspend;
8041 } else if (ufshcd_is_link_off(hba)) {
8042 ret = ufshcd_host_reset_and_restore(hba);
8043 /*
8044 * ufshcd_host_reset_and_restore() should have already
8045 * set the link state as active
8046 */
8047 if (ret || !ufshcd_is_link_active(hba))
8048 goto vendor_suspend;
8049 }
8050
8051 if (!ufshcd_is_ufs_dev_active(hba)) {
8052 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
8053 if (ret)
8054 goto set_old_link_state;
8055 }
8056
4e768e76
SJ		 8057 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
8058 ufshcd_enable_auto_bkops(hba);
8059 else
8060 /*
8061 * If BKOPs operations are urgently needed at this moment then
8062 * keep auto-bkops enabled or else disable it.
8063 */
8064 ufshcd_urgent_bkops(hba);
8065
1ab27c9c
ST		 8066 hba->clk_gating.is_suspended = false;
8067
fcb0c4b0
ST		 8068 if (hba->clk_scaling.is_allowed)
8069 ufshcd_resume_clkscaling(hba);
856b3483 8070
ad448378
AH		 8071 /* Enable Auto-Hibernate if configured */
8072 ufshcd_auto_hibern8_enable(hba);
8073
71d848b8
CG		 8074 /* Schedule clock gating in case of no access to UFS device yet */
8075 ufshcd_release(hba);
8076
57d104c1
SJ		 8077 goto out;
8078
8079set_old_link_state:
8080 ufshcd_link_state_transition(hba, old_link_state, 0);
8081vendor_suspend:
0263bcd0 8082 ufshcd_vops_suspend(hba, pm_op);
57d104c1
SJ		 8083disable_vreg:
8084 ufshcd_vreg_set_lpm(hba);
8085disable_irq_and_vops_clks:
8086 ufshcd_disable_irq(hba);
401f1e44
SJ		 8087 if (hba->clk_scaling.is_allowed)
8088 ufshcd_suspend_clkscaling(hba);
57d104c1
SJ		 8089 ufshcd_setup_clocks(hba, false);
8090out:
8091 hba->pm_op_in_progress = 0;
8808b4e9
SC		 8092 if (ret)
8093 ufshcd_update_reg_hist(&hba->ufs_stats.resume_err, (u32)ret);
57d104c1
SJ		 8094 return ret;
8095}
8096
8097/**
8098 * ufshcd_system_suspend - system suspend routine
8099 * @hba: per adapter instance
57d104c1
SJ		 8100 *
8101 * Check the description of ufshcd_suspend() function for more details.
8102 *
8103 * Returns 0 for success and non-zero for failure
8104 */
8105int ufshcd_system_suspend(struct ufs_hba *hba)
8106{
8107 int ret = 0;
7ff5ab47 8108 ktime_t start = ktime_get();
57d104c1
SJ		 8109
8110 if (!hba || !hba->is_powered)
233b594b 8111 return 0;
57d104c1 8112
0b257734
SJ		 8113 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
8114 hba->curr_dev_pwr_mode) &&
8115 (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
8116 hba->uic_link_state))
8117 goto out;
57d104c1 8118
0b257734 8119 if (pm_runtime_suspended(hba->dev)) {
57d104c1
SJ		 8120 /*
8121 * UFS device and/or UFS link low power states during runtime
8122 * suspend seems to be different than what is expected during
8123 * system suspend. Hence runtime resume the devic & link and
8124 * let the system suspend low power states to take effect.
8125 * TODO: If resume takes longer time, we might have optimize
8126 * it in future by not resuming everything if possible.
8127 */
8128 ret = ufshcd_runtime_resume(hba);
8129 if (ret)
8130 goto out;
8131 }
8132
8133 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
8134out:
7ff5ab47
SJ		 8135 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
8136 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8137 hba->curr_dev_pwr_mode, hba->uic_link_state);
e785060e
DR		 8138 if (!ret)
8139 hba->is_sys_suspended = true;
57d104c1
SJ		 8140 return ret;
8141}
8142EXPORT_SYMBOL(ufshcd_system_suspend);
8143
8144/**
8145 * ufshcd_system_resume - system resume routine
8146 * @hba: per adapter instance
8147 *
8148 * Returns 0 for success and non-zero for failure
8149 */
7a3e97b0 8150
57d104c1
SJ		 8151int ufshcd_system_resume(struct ufs_hba *hba)
8152{
7ff5ab47
SJ		 8153 int ret = 0;
8154 ktime_t start = ktime_get();
8155
e3ce73d6
YG		 8156 if (!hba)
8157 return -EINVAL;
8158
8159 if (!hba->is_powered || pm_runtime_suspended(hba->dev))
57d104c1
SJ		 8160 /*
8161 * Let the runtime resume take care of resuming
8162 * if runtime suspended.
8163 */
7ff5ab47
SJ		 8164 goto out;
8165 else
8166 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
8167out:
8168 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
8169 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8170 hba->curr_dev_pwr_mode, hba->uic_link_state);
ce9e7bce
SC		 8171 if (!ret)
8172 hba->is_sys_suspended = false;
7ff5ab47 8173 return ret;
7a3e97b0 8174}
57d104c1 8175EXPORT_SYMBOL(ufshcd_system_resume);
3b1d0580 8176
57d104c1
SJ		 8177/**
8178 * ufshcd_runtime_suspend - runtime suspend routine
8179 * @hba: per adapter instance
8180 *
8181 * Check the description of ufshcd_suspend() function for more details.
8182 *
8183 * Returns 0 for success and non-zero for failure
8184 */
66ec6d59
SRT		 8185int ufshcd_runtime_suspend(struct ufs_hba *hba)
8186{
7ff5ab47
SJ		 8187 int ret = 0;
8188 ktime_t start = ktime_get();
8189
e3ce73d6
YG		 8190 if (!hba)
8191 return -EINVAL;
8192
8193 if (!hba->is_powered)
7ff5ab47
SJ		 8194 goto out;
8195 else
8196 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
8197out:
8198 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
8199 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8200 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8201 return ret;
66ec6d59
SRT		 8202}
8203EXPORT_SYMBOL(ufshcd_runtime_suspend);
8204
57d104c1
SJ		 8205/**
8206 * ufshcd_runtime_resume - runtime resume routine
8207 * @hba: per adapter instance
8208 *
8209 * This function basically brings the UFS device, UniPro link and controller
8210 * to active state. Following operations are done in this function:
8211 *
8212 * 1. Turn on all the controller related clocks
8213 * 2. Bring the UniPro link out of Hibernate state
8214 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
8215 * to active state.
8216 * 4. If auto-bkops is enabled on the device, disable it.
8217 *
8218 * So following would be the possible power state after this function return
8219 * successfully:
8220 * S1: UFS device in Active state with VCC rail ON
8221 * UniPro link in Active state
8222 * All the UFS/UniPro controller clocks are ON
8223 *
8224 * Returns 0 for success and non-zero for failure
8225 */
66ec6d59
SRT		 8226int ufshcd_runtime_resume(struct ufs_hba *hba)
8227{
7ff5ab47
SJ		 8228 int ret = 0;
8229 ktime_t start = ktime_get();
8230
e3ce73d6
YG		 8231 if (!hba)
8232 return -EINVAL;
8233
8234 if (!hba->is_powered)
7ff5ab47
SJ		 8235 goto out;
8236 else
8237 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
8238out:
8239 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
8240 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8241 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8242 return ret;
66ec6d59
SRT		 8243}
8244EXPORT_SYMBOL(ufshcd_runtime_resume);
8245
8246int ufshcd_runtime_idle(struct ufs_hba *hba)
8247{
8248 return 0;
8249}
8250EXPORT_SYMBOL(ufshcd_runtime_idle);
8251
57d104c1
SJ		 8252/**
8253 * ufshcd_shutdown - shutdown routine
8254 * @hba: per adapter instance
8255 *
8256 * This function would power off both UFS device and UFS link.
8257 *
8258 * Returns 0 always to allow force shutdown even in case of errors.
8259 */
8260int ufshcd_shutdown(struct ufs_hba *hba)
8261{
8262 int ret = 0;
8263
f51913ee
SC		 8264 if (!hba->is_powered)
8265 goto out;
8266
57d104c1
SJ		 8267 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
8268 goto out;
8269
8270 if (pm_runtime_suspended(hba->dev)) {
8271 ret = ufshcd_runtime_resume(hba);
8272 if (ret)
8273 goto out;
8274 }
8275
8276 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
8277out:
8278 if (ret)
8279 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
8280 /* allow force shutdown even in case of errors */
8281 return 0;
8282}
8283EXPORT_SYMBOL(ufshcd_shutdown);
8284
7a3e97b0 8285/**
3b1d0580 8286 * ufshcd_remove - de-allocate SCSI host and host memory space
7a3e97b0 8287 * data structure memory
8aa29f19 8288 * @hba: per adapter instance
7a3e97b0 8289 */
3b1d0580 8290void ufshcd_remove(struct ufs_hba *hba)
7a3e97b0 8291{
df032bf2 8292 ufs_bsg_remove(hba);
cbb6813e 8293 ufs_sysfs_remove_nodes(hba->dev);
69a6c269
BVA		 8294 blk_cleanup_queue(hba->tmf_queue);
8295 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360 8296 blk_cleanup_queue(hba->cmd_queue);
cfdf9c91 8297 scsi_remove_host(hba->host);
7a3e97b0 8298 /* disable interrupts */
2fbd009b 8299 ufshcd_disable_intr(hba, hba->intr_mask);
596585a2 8300 ufshcd_hba_stop(hba, true);
7a3e97b0 8301
eebcc196 8302 ufshcd_exit_clk_scaling(hba);
1ab27c9c 8303 ufshcd_exit_clk_gating(hba);
fcb0c4b0
ST		 8304 if (ufshcd_is_clkscaling_supported(hba))
8305 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
aa497613 8306 ufshcd_hba_exit(hba);
3b1d0580
VH		 8307}
8308EXPORT_SYMBOL_GPL(ufshcd_remove);
8309
47555a5c
YG		 8310/**
8311 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
8312 * @hba: pointer to Host Bus Adapter (HBA)
8313 */
8314void ufshcd_dealloc_host(struct ufs_hba *hba)
8315{
8316 scsi_host_put(hba->host);
8317}
8318EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
8319
ca3d7bf9
AM		 8320/**
8321 * ufshcd_set_dma_mask - Set dma mask based on the controller
8322 * addressing capability
8323 * @hba: per adapter instance
8324 *
8325 * Returns 0 for success, non-zero for failure
8326 */
8327static int ufshcd_set_dma_mask(struct ufs_hba *hba)
8328{
8329 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
8330 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
8331 return 0;
8332 }
8333 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
8334}
8335
7a3e97b0 8336/**
5c0c28a8 8337 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
3b1d0580
VH		 8338 * @dev: pointer to device handle
8339 * @hba_handle: driver private handle
7a3e97b0
SY		 8340 * Returns 0 on success, non-zero value on failure
8341 */
5c0c28a8 8342int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
7a3e97b0
SY		 8343{
8344 struct Scsi_Host *host;
8345 struct ufs_hba *hba;
5c0c28a8 8346 int err = 0;
7a3e97b0 8347
3b1d0580
VH		 8348 if (!dev) {
8349 dev_err(dev,
8350 "Invalid memory reference for dev is NULL\n");
8351 err = -ENODEV;
7a3e97b0
SY		 8352 goto out_error;
8353 }
8354
7a3e97b0
SY		 8355 host = scsi_host_alloc(&ufshcd_driver_template,
8356 sizeof(struct ufs_hba));
8357 if (!host) {
3b1d0580 8358 dev_err(dev, "scsi_host_alloc failed\n");
7a3e97b0 8359 err = -ENOMEM;
3b1d0580 8360 goto out_error;
7a3e97b0
SY		 8361 }
8362 hba = shost_priv(host);
7a3e97b0 8363 hba->host = host;
3b1d0580 8364 hba->dev = dev;
5c0c28a8 8365 *hba_handle = hba;
9e1e8a75 8366 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
5c0c28a8 8367
566ec9ad
SM		 8368 INIT_LIST_HEAD(&hba->clk_list_head);
8369
5c0c28a8
SRT		 8370out_error:
8371 return err;
8372}
8373EXPORT_SYMBOL(ufshcd_alloc_host);
8374
69a6c269
BVA		 8375/* This function exists because blk_mq_alloc_tag_set() requires this. */
8376static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
8377 const struct blk_mq_queue_data *qd)
8378{
8379 WARN_ON_ONCE(true);
8380 return BLK_STS_NOTSUPP;
8381}
8382
8383static const struct blk_mq_ops ufshcd_tmf_ops = {
8384 .queue_rq = ufshcd_queue_tmf,
8385};
8386
5c0c28a8
SRT		 8387/**
8388 * ufshcd_init - Driver initialization routine
8389 * @hba: per-adapter instance
8390 * @mmio_base: base register address
8391 * @irq: Interrupt line of device
8392 * Returns 0 on success, non-zero value on failure
8393 */
8394int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
8395{
8396 int err;
8397 struct Scsi_Host *host = hba->host;
8398 struct device *dev = hba->dev;
8399
8400 if (!mmio_base) {
8401 dev_err(hba->dev,
8402 "Invalid memory reference for mmio_base is NULL\n");
8403 err = -ENODEV;
8404 goto out_error;
8405 }
8406
3b1d0580
VH		 8407 hba->mmio_base = mmio_base;
8408 hba->irq = irq;
b9dc8aca 8409 hba->hba_enable_delay_us = 1000;
7a3e97b0 8410
aa497613 8411 err = ufshcd_hba_init(hba);
5c0c28a8
SRT		 8412 if (err)
8413 goto out_error;
8414
7a3e97b0
SY		 8415 /* Read capabilities registers */
8416 ufshcd_hba_capabilities(hba);
8417
8418 /* Get UFS version supported by the controller */
8419 hba->ufs_version = ufshcd_get_ufs_version(hba);
8420
c01848c6
YG		 8421 if ((hba->ufs_version != UFSHCI_VERSION_10) &&
8422 (hba->ufs_version != UFSHCI_VERSION_11) &&
8423 (hba->ufs_version != UFSHCI_VERSION_20) &&
8424 (hba->ufs_version != UFSHCI_VERSION_21))
8425 dev_err(hba->dev, "invalid UFS version 0x%x\n",
8426 hba->ufs_version);
8427
2fbd009b
SJ		 8428 /* Get Interrupt bit mask per version */
8429 hba->intr_mask = ufshcd_get_intr_mask(hba);
8430
ca3d7bf9
AM		 8431 err = ufshcd_set_dma_mask(hba);
8432 if (err) {
8433 dev_err(hba->dev, "set dma mask failed\n");
8434 goto out_disable;
8435 }
8436
7a3e97b0
SY		 8437 /* Allocate memory for host memory space */
8438 err = ufshcd_memory_alloc(hba);
8439 if (err) {
3b1d0580
VH		 8440 dev_err(hba->dev, "Memory allocation failed\n");
8441 goto out_disable;
7a3e97b0
SY		 8442 }
8443
8444 /* Configure LRB */
8445 ufshcd_host_memory_configure(hba);
8446
8447 host->can_queue = hba->nutrs;
8448 host->cmd_per_lun = hba->nutrs;
8449 host->max_id = UFSHCD_MAX_ID;
0ce147d4 8450 host->max_lun = UFS_MAX_LUNS;
7a3e97b0
SY		 8451 host->max_channel = UFSHCD_MAX_CHANNEL;
8452 host->unique_id = host->host_no;
a851b2bd 8453 host->max_cmd_len = UFS_CDB_SIZE;
7a3e97b0 8454
7eb584db
DR		 8455 hba->max_pwr_info.is_valid = false;
8456
7a3e97b0 8457 /* Initialize work queues */
e8e7f271 8458 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
66ec6d59 8459 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
7a3e97b0 8460
6ccf44fe
SJ		 8461 /* Initialize UIC command mutex */
8462 mutex_init(&hba->uic_cmd_mutex);
8463
5a0b0cb9
SRT		 8464 /* Initialize mutex for device management commands */
8465 mutex_init(&hba->dev_cmd.lock);
8466
a3cd5ec5
SJ		 8467 init_rwsem(&hba->clk_scaling_lock);
8468
1ab27c9c 8469 ufshcd_init_clk_gating(hba);
199ef13c 8470
eebcc196
VG		 8471 ufshcd_init_clk_scaling(hba);
8472
199ef13c
YG		 8473 /*
8474 * In order to avoid any spurious interrupt immediately after
8475 * registering UFS controller interrupt handler, clear any pending UFS
8476 * interrupt status and disable all the UFS interrupts.
8477 */
8478 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
8479 REG_INTERRUPT_STATUS);
8480 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
8481 /*
8482 * Make sure that UFS interrupts are disabled and any pending interrupt
8483 * status is cleared before registering UFS interrupt handler.
8484 */
8485 mb();
8486
7a3e97b0 8487 /* IRQ registration */
2953f850 8488 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
7a3e97b0 8489 if (err) {
3b1d0580 8490 dev_err(hba->dev, "request irq failed\n");
1ab27c9c 8491 goto exit_gating;
57d104c1
SJ		 8492 } else {
8493 hba->is_irq_enabled = true;
7a3e97b0
SY		 8494 }
8495
3b1d0580 8496 err = scsi_add_host(host, hba->dev);
7a3e97b0 8497 if (err) {
3b1d0580 8498 dev_err(hba->dev, "scsi_add_host failed\n");
1ab27c9c 8499 goto exit_gating;
7a3e97b0
SY		 8500 }
8501
7252a360
BVA		 8502 hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set);
8503 if (IS_ERR(hba->cmd_queue)) {
8504 err = PTR_ERR(hba->cmd_queue);
8505 goto out_remove_scsi_host;
8506 }
8507
69a6c269
BVA		 8508 hba->tmf_tag_set = (struct blk_mq_tag_set) {
8509 .nr_hw_queues = 1,
8510 .queue_depth = hba->nutmrs,
8511 .ops = &ufshcd_tmf_ops,
8512 .flags = BLK_MQ_F_NO_SCHED,
8513 };
8514 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
8515 if (err < 0)
8516 goto free_cmd_queue;
8517 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
8518 if (IS_ERR(hba->tmf_queue)) {
8519 err = PTR_ERR(hba->tmf_queue);
8520 goto free_tmf_tag_set;
8521 }
8522
d8d9f793
BA		 8523 /* Reset the attached device */
8524 ufshcd_vops_device_reset(hba);
8525
6ccf44fe
SJ		 8526 /* Host controller enable */
8527 err = ufshcd_hba_enable(hba);
7a3e97b0 8528 if (err) {
6ccf44fe 8529 dev_err(hba->dev, "Host controller enable failed\n");
66cc820f 8530 ufshcd_print_host_regs(hba);
6ba65588 8531 ufshcd_print_host_state(hba);
69a6c269 8532 goto free_tmf_queue;
7a3e97b0 8533 }
6ccf44fe 8534
0c8f7586
SJ		 8535 /*
8536 * Set the default power management level for runtime and system PM.
8537 * Default power saving mode is to keep UFS link in Hibern8 state
8538 * and UFS device in sleep state.
8539 */
8540 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8541 UFS_SLEEP_PWR_MODE,
8542 UIC_LINK_HIBERN8_STATE);
8543 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8544 UFS_SLEEP_PWR_MODE,
8545 UIC_LINK_HIBERN8_STATE);
8546
ad448378 8547 /* Set the default auto-hiberate idle timer value to 150 ms */
f571b377 8548 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
ad448378
AH		 8549 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
8550 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
8551 }
8552
62694735
SRT		 8553 /* Hold auto suspend until async scan completes */
8554 pm_runtime_get_sync(dev);
38135535 8555 atomic_set(&hba->scsi_block_reqs_cnt, 0);
57d104c1 8556 /*
7caf489b
SJ		 8557 * We are assuming that device wasn't put in sleep/power-down
8558 * state exclusively during the boot stage before kernel.
8559 * This assumption helps avoid doing link startup twice during
8560 * ufshcd_probe_hba().
57d104c1 8561 */
7caf489b 8562 ufshcd_set_ufs_dev_active(hba);
57d104c1 8563
6ccf44fe 8564 async_schedule(ufshcd_async_scan, hba);
cbb6813e 8565 ufs_sysfs_add_nodes(hba->dev);
6ccf44fe 8566
7a3e97b0
SY		 8567 return 0;
8568
69a6c269
BVA		 8569free_tmf_queue:
8570 blk_cleanup_queue(hba->tmf_queue);
8571free_tmf_tag_set:
8572 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360
BVA		 8573free_cmd_queue:
8574 blk_cleanup_queue(hba->cmd_queue);
3b1d0580
VH		 8575out_remove_scsi_host:
8576 scsi_remove_host(hba->host);
1ab27c9c 8577exit_gating:
eebcc196 8578 ufshcd_exit_clk_scaling(hba);
1ab27c9c 8579 ufshcd_exit_clk_gating(hba);
3b1d0580 8580out_disable:
57d104c1 8581 hba->is_irq_enabled = false;
aa497613 8582 ufshcd_hba_exit(hba);
3b1d0580
VH		 8583out_error:
8584 return err;
8585}
8586EXPORT_SYMBOL_GPL(ufshcd_init);
8587
3b1d0580
VH		 8588MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
8589MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
e0eca63e 8590MODULE_DESCRIPTION("Generic UFS host controller driver Core");
7a3e97b0
SY		 8591MODULE_LICENSE("GPL");
8592MODULE_VERSION(UFSHCD_DRIVER_VERSION);
Ubuntu Jammy Linux kernel mirror