]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/scsi/ufs/ufshcd.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: ufs: Use DBD setting in mode sense
[mirror_ubuntu-jammy-kernel.git] / drivers / scsi / ufs / ufshcd.c
CommitLineData
7a3e97b0 1/*
e0eca63e 2 * Universal Flash Storage Host controller driver Core
7a3e97b0
SY		 3 *
4 * This code is based on drivers/scsi/ufs/ufshcd.c
3b1d0580 5 * Copyright (C) 2011-2013 Samsung India Software Operations
52ac95fe 6 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7a3e97b0 7 *
3b1d0580
VH		 8 * Authors:
9 * Santosh Yaraganavi <santosh.sy@samsung.com>
10 * Vinayak Holikatti <h.vinayak@samsung.com>
7a3e97b0
SY		 11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
3b1d0580
VH		 16 * See the COPYING file in the top-level directory or visit
17 * <http://www.gnu.org/licenses/gpl-2.0.html>
7a3e97b0
SY		 18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 *
3b1d0580
VH		 24 * This program is provided "AS IS" and "WITH ALL FAULTS" and
25 * without warranty of any kind. You are solely responsible for
26 * determining the appropriateness of using and distributing
27 * the program and assume all risks associated with your exercise
28 * of rights with respect to the program, including but not limited
29 * to infringement of third party rights, the risks and costs of
30 * program errors, damage to or loss of data, programs or equipment,
31 * and unavailability or interruption of operations. Under no
32 * circumstances will the contributor of this Program be liable for
33 * any damages of any kind arising from your use or distribution of
34 * this program.
5c0c28a8
SRT		 35 *
36 * The Linux Foundation chooses to take subject only to the GPLv2
37 * license terms, and distributes only under these terms.
7a3e97b0
SY		 38 */
39
6ccf44fe 40#include <linux/async.h>
856b3483 41#include <linux/devfreq.h>
b573d484 42#include <linux/nls.h>
54b879b7 43#include <linux/of.h>
ad448378 44#include <linux/bitfield.h>
e0eca63e 45#include "ufshcd.h"
c58ab7aa 46#include "ufs_quirks.h"
53b3d9c3 47#include "unipro.h"
cbb6813e 48#include "ufs-sysfs.h"
df032bf2 49#include "ufs_bsg.h"
7a3e97b0 50
7ff5ab47
SJ		 51#define CREATE_TRACE_POINTS
52#include <trace/events/ufs.h>
53
2fbd009b
SJ		 54#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
55 UTP_TASK_REQ_COMPL |\
56 UFSHCD_ERROR_MASK)
6ccf44fe
SJ		 57/* UIC command timeout, unit: ms */
58#define UIC_CMD_TIMEOUT 500
2fbd009b 59
5a0b0cb9
SRT		 60/* NOP OUT retries waiting for NOP IN response */
61#define NOP_OUT_RETRIES 10
62/* Timeout after 30 msecs if NOP OUT hangs without response */
63#define NOP_OUT_TIMEOUT 30 /* msecs */
64
68078d5c 65/* Query request retries */
10fe5888 66#define QUERY_REQ_RETRIES 3
68078d5c 67/* Query request timeout */
10fe5888 68#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
68078d5c 69
e2933132
SRT		 70/* Task management command timeout */
71#define TM_CMD_TIMEOUT 100 /* msecs */
72
64238fbd
YG		 73/* maximum number of retries for a general UIC command */
74#define UFS_UIC_COMMAND_RETRIES 3
75
1d337ec2
SRT		 76/* maximum number of link-startup retries */
77#define DME_LINKSTARTUP_RETRIES 3
78
87d0b4a6
YG		 79/* Maximum retries for Hibern8 enter */
80#define UIC_HIBERN8_ENTER_RETRIES 3
81
1d337ec2
SRT		 82/* maximum number of reset retries before giving up */
83#define MAX_HOST_RESET_RETRIES 5
84
68078d5c
DR		 85/* Expose the flag value from utp_upiu_query.value */
86#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
87
7d568652
SJ		 88/* Interrupt aggregation default timeout, unit: 40us */
89#define INT_AGGR_DEF_TO 0x02
90
49615ba1
SC		 91/* default delay of autosuspend: 2000 ms */
92#define RPM_AUTOSUSPEND_DELAY_MS 2000
93
aa497613
SRT		 94#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
95 ({ \
96 int _ret; \
97 if (_on) \
98 _ret = ufshcd_enable_vreg(_dev, _vreg); \
99 else \
100 _ret = ufshcd_disable_vreg(_dev, _vreg); \
101 _ret; \
102 })
103
ba80917d
TW		 104#define ufshcd_hex_dump(prefix_str, buf, len) do { \
105 size_t __len = (len); \
106 print_hex_dump(KERN_ERR, prefix_str, \
107 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
108 16, 4, buf, __len, false); \
109} while (0)
110
111int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
112 const char *prefix)
113{
d6724756
MG		 114 u32 *regs;
115 size_t pos;
116
117 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
118 return -EINVAL;
ba80917d 119
cddaebaf 120 regs = kzalloc(len, GFP_ATOMIC);
ba80917d
TW		 121 if (!regs)
122 return -ENOMEM;
123
d6724756
MG		 124 for (pos = 0; pos < len; pos += 4)
125 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
126
ba80917d
TW		 127 ufshcd_hex_dump(prefix, regs, len);
128 kfree(regs);
129
130 return 0;
131}
132EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
66cc820f 133
7a3e97b0
SY		 134enum {
135 UFSHCD_MAX_CHANNEL = 0,
136 UFSHCD_MAX_ID = 1,
7a3e97b0
SY		 137 UFSHCD_CMD_PER_LUN = 32,
138 UFSHCD_CAN_QUEUE = 32,
139};
140
141/* UFSHCD states */
142enum {
7a3e97b0
SY		 143 UFSHCD_STATE_RESET,
144 UFSHCD_STATE_ERROR,
3441da7d 145 UFSHCD_STATE_OPERATIONAL,
141f8165 146 UFSHCD_STATE_EH_SCHEDULED,
3441da7d
SRT		 147};
148
149/* UFSHCD error handling flags */
150enum {
151 UFSHCD_EH_IN_PROGRESS = (1 << 0),
7a3e97b0
SY		 152};
153
e8e7f271
SRT		 154/* UFSHCD UIC layer error flags */
155enum {
156 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
9a47ec7c
YG		 157 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
158 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
159 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
160 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
161 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
e8e7f271
SRT		 162};
163
3441da7d 164#define ufshcd_set_eh_in_progress(h) \
9c490d2d 165 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
3441da7d 166#define ufshcd_eh_in_progress(h) \
9c490d2d 167 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
3441da7d 168#define ufshcd_clear_eh_in_progress(h) \
9c490d2d 169 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
3441da7d 170
57d104c1
SJ		 171#define ufshcd_set_ufs_dev_active(h) \
172 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
173#define ufshcd_set_ufs_dev_sleep(h) \
174 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
175#define ufshcd_set_ufs_dev_poweroff(h) \
176 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
177#define ufshcd_is_ufs_dev_active(h) \
178 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
179#define ufshcd_is_ufs_dev_sleep(h) \
180 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
181#define ufshcd_is_ufs_dev_poweroff(h) \
182 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
183
cbb6813e 184struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
57d104c1
SJ		 185 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
186 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
187 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
188 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
189 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
190 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
191};
192
193static inline enum ufs_dev_pwr_mode
194ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
195{
196 return ufs_pm_lvl_states[lvl].dev_state;
197}
198
199static inline enum uic_link_state
200ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
201{
202 return ufs_pm_lvl_states[lvl].link_state;
203}
204
0c8f7586
SJ		 205static inline enum ufs_pm_level
206ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
207 enum uic_link_state link_state)
208{
209 enum ufs_pm_level lvl;
210
211 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
212 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
213 (ufs_pm_lvl_states[lvl].link_state == link_state))
214 return lvl;
215 }
216
217 /* if no match found, return the level 0 */
218 return UFS_PM_LVL_0;
219}
220
56d4a186
SJ		 221static struct ufs_dev_fix ufs_fixups[] = {
222 /* UFS cards deviations table */
223 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
224 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
56d4a186
SJ		 225 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
226 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
56d4a186
SJ		 227 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
228 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
229 UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
230 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
231 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
232 UFS_DEVICE_QUIRK_PA_TACTIVATE),
233 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
234 UFS_DEVICE_QUIRK_PA_TACTIVATE),
56d4a186
SJ		 235 UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
236 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
8e4829c6
WL		 237 UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
238 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
56d4a186
SJ		 239
240 END_FIX
241};
242
9333d775 243static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
3441da7d 244static void ufshcd_async_scan(void *data, async_cookie_t cookie);
e8e7f271 245static int ufshcd_reset_and_restore(struct ufs_hba *hba);
e7d38257 246static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
e8e7f271 247static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
1d337ec2
SRT		 248static void ufshcd_hba_exit(struct ufs_hba *hba);
249static int ufshcd_probe_hba(struct ufs_hba *hba);
1ab27c9c
ST		 250static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
251 bool skip_ref_clk);
252static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
253static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
254static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
cad2e03d 255static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
57d104c1 256static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
fcb0c4b0
ST		 257static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
258static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
401f1e44 259static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
fcb0c4b0 260static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
57d104c1 261static irqreturn_t ufshcd_intr(int irq, void *__hba);
874237f7
YG		 262static int ufshcd_change_power_mode(struct ufs_hba *hba,
263 struct ufs_pa_layer_attr *pwr_mode);
14497328
YG		 264static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
265{
266 return tag >= 0 && tag < hba->nutrs;
267}
57d104c1
SJ		 268
269static inline int ufshcd_enable_irq(struct ufs_hba *hba)
270{
271 int ret = 0;
272
273 if (!hba->is_irq_enabled) {
274 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD,
275 hba);
276 if (ret)
277 dev_err(hba->dev, "%s: request_irq failed, ret=%d\n",
278 __func__, ret);
279 hba->is_irq_enabled = true;
280 }
281
282 return ret;
283}
284
285static inline void ufshcd_disable_irq(struct ufs_hba *hba)
286{
287 if (hba->is_irq_enabled) {
288 free_irq(hba->irq, hba);
289 hba->is_irq_enabled = false;
290 }
291}
3441da7d 292
38135535
SJ		 293static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
294{
295 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
296 scsi_unblock_requests(hba->host);
297}
298
299static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
300{
301 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
302 scsi_block_requests(hba->host);
303}
304
6667e6d9
OS		 305static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
306 const char *str)
307{
308 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
309
310 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
311}
312
313static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
314 const char *str)
315{
316 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
317
318 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
319}
320
321static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
322 const char *str)
323{
6667e6d9 324 int off = (int)tag - hba->nutrs;
391e388f 325 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
6667e6d9 326
391e388f
CH		 327 trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
328 &descp->input_param1);
6667e6d9
OS		 329}
330
1a07f2d9
LS		 331static void ufshcd_add_command_trace(struct ufs_hba *hba,
332 unsigned int tag, const char *str)
333{
334 sector_t lba = -1;
335 u8 opcode = 0;
336 u32 intr, doorbell;
e7c3b379 337 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
1a07f2d9
LS		 338 int transfer_len = -1;
339
e7c3b379
OS		 340 if (!trace_ufshcd_command_enabled()) {
341 /* trace UPIU W/O tracing command */
342 if (lrbp->cmd)
343 ufshcd_add_cmd_upiu_trace(hba, tag, str);
1a07f2d9 344 return;
e7c3b379 345 }
1a07f2d9
LS		 346
347 if (lrbp->cmd) { /* data phase exists */
e7c3b379
OS		 348 /* trace UPIU also */
349 ufshcd_add_cmd_upiu_trace(hba, tag, str);
1a07f2d9
LS		 350 opcode = (u8)(*lrbp->cmd->cmnd);
351 if ((opcode == READ_10) || (opcode == WRITE_10)) {
352 /*
353 * Currently we only fully trace read(10) and write(10)
354 * commands
355 */
356 if (lrbp->cmd->request && lrbp->cmd->request->bio)
357 lba =
358 lrbp->cmd->request->bio->bi_iter.bi_sector;
359 transfer_len = be32_to_cpu(
360 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
361 }
362 }
363
364 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
365 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
366 trace_ufshcd_command(dev_name(hba->dev), str, tag,
367 doorbell, transfer_len, intr, lba, opcode);
368}
369
ff8e20c6
DR		 370static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
371{
372 struct ufs_clk_info *clki;
373 struct list_head *head = &hba->clk_list_head;
374
566ec9ad 375 if (list_empty(head))
ff8e20c6
DR		 376 return;
377
378 list_for_each_entry(clki, head, list) {
379 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
380 clki->max_freq)
381 dev_err(hba->dev, "clk: %s, rate: %u\n",
382 clki->name, clki->curr_freq);
383 }
384}
385
48d5b973
SC		 386static void ufshcd_print_err_hist(struct ufs_hba *hba,
387 struct ufs_err_reg_hist *err_hist,
388 char *err_name)
ff8e20c6
DR		 389{
390 int i;
27752647 391 bool found = false;
ff8e20c6 392
48d5b973
SC		 393 for (i = 0; i < UFS_ERR_REG_HIST_LENGTH; i++) {
394 int p = (i + err_hist->pos) % UFS_ERR_REG_HIST_LENGTH;
ff8e20c6
DR		 395
396 if (err_hist->reg[p] == 0)
397 continue;
c5397f13 398 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
ff8e20c6 399 err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
27752647 400 found = true;
ff8e20c6 401 }
27752647
SC		 402
403 if (!found)
48d5b973 404 dev_err(hba->dev, "No record of %s errors\n", err_name);
ff8e20c6
DR		 405}
406
66cc820f
DR		 407static void ufshcd_print_host_regs(struct ufs_hba *hba)
408{
ba80917d 409 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
66cc820f
DR		 410 dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n",
411 hba->ufs_version, hba->capabilities);
412 dev_err(hba->dev,
413 "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n",
414 (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks);
ff8e20c6
DR		 415 dev_err(hba->dev,
416 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n",
417 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
418 hba->ufs_stats.hibern8_exit_cnt);
419
48d5b973
SC		 420 ufshcd_print_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
421 ufshcd_print_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
422 ufshcd_print_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
423 ufshcd_print_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
424 ufshcd_print_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
d3c615bf
SC		 425 ufshcd_print_err_hist(hba, &hba->ufs_stats.auto_hibern8_err,
426 "auto_hibern8_err");
8808b4e9
SC		 427 ufshcd_print_err_hist(hba, &hba->ufs_stats.fatal_err, "fatal_err");
428 ufshcd_print_err_hist(hba, &hba->ufs_stats.link_startup_err,
429 "link_startup_fail");
430 ufshcd_print_err_hist(hba, &hba->ufs_stats.resume_err, "resume_fail");
431 ufshcd_print_err_hist(hba, &hba->ufs_stats.suspend_err,
432 "suspend_fail");
433 ufshcd_print_err_hist(hba, &hba->ufs_stats.dev_reset, "dev_reset");
434 ufshcd_print_err_hist(hba, &hba->ufs_stats.host_reset, "host_reset");
435 ufshcd_print_err_hist(hba, &hba->ufs_stats.task_abort, "task_abort");
ff8e20c6
DR		 436
437 ufshcd_print_clk_freqs(hba);
438
439 if (hba->vops && hba->vops->dbg_register_dump)
440 hba->vops->dbg_register_dump(hba);
66cc820f
DR		 441}
442
443static
444void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
445{
446 struct ufshcd_lrb *lrbp;
7fabb77b 447 int prdt_length;
66cc820f
DR		 448 int tag;
449
450 for_each_set_bit(tag, &bitmap, hba->nutrs) {
451 lrbp = &hba->lrb[tag];
452
ff8e20c6
DR		 453 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
454 tag, ktime_to_us(lrbp->issue_time_stamp));
09017188
ZL		 455 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
456 tag, ktime_to_us(lrbp->compl_time_stamp));
ff8e20c6
DR		 457 dev_err(hba->dev,
458 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
459 tag, (u64)lrbp->utrd_dma_addr);
460
66cc820f
DR		 461 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
462 sizeof(struct utp_transfer_req_desc));
ff8e20c6
DR		 463 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
464 (u64)lrbp->ucd_req_dma_addr);
66cc820f
DR		 465 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
466 sizeof(struct utp_upiu_req));
ff8e20c6
DR		 467 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
468 (u64)lrbp->ucd_rsp_dma_addr);
66cc820f
DR		 469 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
470 sizeof(struct utp_upiu_rsp));
66cc820f 471
7fabb77b
GB		 472 prdt_length = le16_to_cpu(
473 lrbp->utr_descriptor_ptr->prd_table_length);
474 dev_err(hba->dev,
475 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
476 tag, prdt_length,
477 (u64)lrbp->ucd_prdt_dma_addr);
478
479 if (pr_prdt)
66cc820f 480 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
7fabb77b 481 sizeof(struct ufshcd_sg_entry) * prdt_length);
66cc820f
DR		 482 }
483}
484
485static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
486{
66cc820f
DR		 487 int tag;
488
489 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
391e388f
CH		 490 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
491
66cc820f 492 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
391e388f 493 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
66cc820f
DR		 494 }
495}
496
6ba65588
GB		 497static void ufshcd_print_host_state(struct ufs_hba *hba)
498{
499 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
7252a360
BVA		 500 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
501 hba->outstanding_reqs, hba->outstanding_tasks);
6ba65588
GB		 502 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
503 hba->saved_err, hba->saved_uic_err);
504 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
505 hba->curr_dev_pwr_mode, hba->uic_link_state);
506 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
507 hba->pm_op_in_progress, hba->is_sys_suspended);
508 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
509 hba->auto_bkops_enabled, hba->host->host_self_blocked);
510 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
511 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
512 hba->eh_flags, hba->req_abort_count);
513 dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n",
514 hba->capabilities, hba->caps);
515 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
516 hba->dev_quirks);
517}
518
ff8e20c6
DR		 519/**
520 * ufshcd_print_pwr_info - print power params as saved in hba
521 * power info
522 * @hba: per-adapter instance
523 */
524static void ufshcd_print_pwr_info(struct ufs_hba *hba)
525{
526 static const char * const names[] = {
527 "INVALID MODE",
528 "FAST MODE",
529 "SLOW_MODE",
530 "INVALID MODE",
531 "FASTAUTO_MODE",
532 "SLOWAUTO_MODE",
533 "INVALID MODE",
534 };
535
536 dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
537 __func__,
538 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
539 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
540 names[hba->pwr_info.pwr_rx],
541 names[hba->pwr_info.pwr_tx],
542 hba->pwr_info.hs_rate);
543}
544
5a0b0cb9
SRT		 545/*
546 * ufshcd_wait_for_register - wait for register value to change
547 * @hba - per-adapter interface
548 * @reg - mmio register offset
549 * @mask - mask to apply to read register value
550 * @val - wait condition
551 * @interval_us - polling interval in microsecs
552 * @timeout_ms - timeout in millisecs
596585a2 553 * @can_sleep - perform sleep or just spin
5a0b0cb9
SRT		 554 *
555 * Returns -ETIMEDOUT on error, zero on success
556 */
596585a2
YG		 557int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
558 u32 val, unsigned long interval_us,
559 unsigned long timeout_ms, bool can_sleep)
5a0b0cb9
SRT		 560{
561 int err = 0;
562 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
563
564 /* ignore bits that we don't intend to wait on */
565 val = val & mask;
566
567 while ((ufshcd_readl(hba, reg) & mask) != val) {
596585a2
YG		 568 if (can_sleep)
569 usleep_range(interval_us, interval_us + 50);
570 else
571 udelay(interval_us);
5a0b0cb9
SRT		 572 if (time_after(jiffies, timeout)) {
573 if ((ufshcd_readl(hba, reg) & mask) != val)
574 err = -ETIMEDOUT;
575 break;
576 }
577 }
578
579 return err;
580}
581
2fbd009b
SJ		 582/**
583 * ufshcd_get_intr_mask - Get the interrupt bit mask
8aa29f19 584 * @hba: Pointer to adapter instance
2fbd009b
SJ		 585 *
586 * Returns interrupt bit mask per version
587 */
588static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
589{
c01848c6
YG		 590 u32 intr_mask = 0;
591
592 switch (hba->ufs_version) {
593 case UFSHCI_VERSION_10:
594 intr_mask = INTERRUPT_MASK_ALL_VER_10;
595 break;
c01848c6
YG		 596 case UFSHCI_VERSION_11:
597 case UFSHCI_VERSION_20:
598 intr_mask = INTERRUPT_MASK_ALL_VER_11;
599 break;
c01848c6
YG		 600 case UFSHCI_VERSION_21:
601 default:
602 intr_mask = INTERRUPT_MASK_ALL_VER_21;
031d1e0f 603 break;
c01848c6
YG		 604 }
605
606 return intr_mask;
2fbd009b
SJ		 607}
608
7a3e97b0
SY		 609/**
610 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
8aa29f19 611 * @hba: Pointer to adapter instance
7a3e97b0
SY		 612 *
613 * Returns UFSHCI version supported by the controller
614 */
615static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
616{
0263bcd0
YG		 617 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
618 return ufshcd_vops_get_ufs_hci_version(hba);
9949e702 619
b873a275 620 return ufshcd_readl(hba, REG_UFS_VERSION);
7a3e97b0
SY		 621}
622
623/**
624 * ufshcd_is_device_present - Check if any device connected to
625 * the host controller
5c0c28a8 626 * @hba: pointer to adapter instance
7a3e97b0 627 *
c9e6010b 628 * Returns true if device present, false if no device detected
7a3e97b0 629 */
c9e6010b 630static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
7a3e97b0 631{
5c0c28a8 632 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
c9e6010b 633 DEVICE_PRESENT) ? true : false;
7a3e97b0
SY		 634}
635
636/**
637 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
8aa29f19 638 * @lrbp: pointer to local command reference block
7a3e97b0
SY		 639 *
640 * This function is used to get the OCS field from UTRD
641 * Returns the OCS field in the UTRD
642 */
643static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
644{
e8c8e82a 645 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
7a3e97b0
SY		 646}
647
7a3e97b0
SY		 648/**
649 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
650 * @hba: per adapter instance
651 * @pos: position of the bit to be cleared
652 */
653static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
654{
1399c5b0
AA		 655 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
656 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
657 else
658 ufshcd_writel(hba, ~(1 << pos),
659 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
660}
661
662/**
663 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
664 * @hba: per adapter instance
665 * @pos: position of the bit to be cleared
666 */
667static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
668{
669 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
670 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
671 else
672 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
7a3e97b0
SY		 673}
674
a48353f6
YG		 675/**
676 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
677 * @hba: per adapter instance
678 * @tag: position of the bit to be cleared
679 */
680static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
681{
682 __clear_bit(tag, &hba->outstanding_reqs);
683}
684
7a3e97b0
SY		 685/**
686 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
687 * @reg: Register value of host controller status
688 *
689 * Returns integer, 0 on Success and positive value if failed
690 */
691static inline int ufshcd_get_lists_status(u32 reg)
692{
6cf16115 693 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
7a3e97b0
SY		 694}
695
696/**
697 * ufshcd_get_uic_cmd_result - Get the UIC command result
698 * @hba: Pointer to adapter instance
699 *
700 * This function gets the result of UIC command completion
701 * Returns 0 on success, non zero value on error
702 */
703static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
704{
b873a275 705 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
7a3e97b0
SY		 706 MASK_UIC_COMMAND_RESULT;
707}
708
12b4fdb4
SJ		 709/**
710 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
711 * @hba: Pointer to adapter instance
712 *
713 * This function gets UIC command argument3
714 * Returns 0 on success, non zero value on error
715 */
716static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
717{
718 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
719}
720
7a3e97b0 721/**
5a0b0cb9 722 * ufshcd_get_req_rsp - returns the TR response transaction type
7a3e97b0 723 * @ucd_rsp_ptr: pointer to response UPIU
7a3e97b0
SY		 724 */
725static inline int
5a0b0cb9 726ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
7a3e97b0 727{
5a0b0cb9 728 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
7a3e97b0
SY		 729}
730
731/**
732 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
733 * @ucd_rsp_ptr: pointer to response UPIU
734 *
735 * This function gets the response status and scsi_status from response UPIU
736 * Returns the response result code.
737 */
738static inline int
739ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
740{
741 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
742}
743
1c2623c5
SJ		 744/*
745 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
746 * from response UPIU
747 * @ucd_rsp_ptr: pointer to response UPIU
748 *
749 * Return the data segment length.
750 */
751static inline unsigned int
752ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
753{
754 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
755 MASK_RSP_UPIU_DATA_SEG_LEN;
756}
757
66ec6d59
SRT		 758/**
759 * ufshcd_is_exception_event - Check if the device raised an exception event
760 * @ucd_rsp_ptr: pointer to response UPIU
761 *
762 * The function checks if the device raised an exception event indicated in
763 * the Device Information field of response UPIU.
764 *
765 * Returns true if exception is raised, false otherwise.
766 */
767static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
768{
769 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
770 MASK_RSP_EXCEPTION_EVENT ? true : false;
771}
772
7a3e97b0 773/**
7d568652 774 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
7a3e97b0 775 * @hba: per adapter instance
7a3e97b0
SY		 776 */
777static inline void
7d568652 778ufshcd_reset_intr_aggr(struct ufs_hba *hba)
7a3e97b0 779{
7d568652
SJ		 780 ufshcd_writel(hba, INT_AGGR_ENABLE |
781 INT_AGGR_COUNTER_AND_TIMER_RESET,
782 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
783}
784
785/**
786 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
787 * @hba: per adapter instance
788 * @cnt: Interrupt aggregation counter threshold
789 * @tmout: Interrupt aggregation timeout value
790 */
791static inline void
792ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
793{
794 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
795 INT_AGGR_COUNTER_THLD_VAL(cnt) |
796 INT_AGGR_TIMEOUT_VAL(tmout),
797 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
7a3e97b0
SY		 798}
799
b852190e
YG		 800/**
801 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
802 * @hba: per adapter instance
803 */
804static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
805{
806 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
807}
808
7a3e97b0
SY		 809/**
810 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
811 * When run-stop registers are set to 1, it indicates the
812 * host controller that it can process the requests
813 * @hba: per adapter instance
814 */
815static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
816{
b873a275
SJ		 817 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
818 REG_UTP_TASK_REQ_LIST_RUN_STOP);
819 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
820 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
7a3e97b0
SY		 821}
822
7a3e97b0
SY		 823/**
824 * ufshcd_hba_start - Start controller initialization sequence
825 * @hba: per adapter instance
826 */
827static inline void ufshcd_hba_start(struct ufs_hba *hba)
828{
b873a275 829 ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
7a3e97b0
SY		 830}
831
832/**
833 * ufshcd_is_hba_active - Get controller state
834 * @hba: per adapter instance
835 *
c9e6010b 836 * Returns false if controller is active, true otherwise
7a3e97b0 837 */
c9e6010b 838static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
7a3e97b0 839{
4a8eec2b
TK		 840 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
841 ? false : true;
7a3e97b0
SY		 842}
843
37113106
YG		 844u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
845{
846 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
847 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
848 (hba->ufs_version == UFSHCI_VERSION_11))
849 return UFS_UNIPRO_VER_1_41;
850 else
851 return UFS_UNIPRO_VER_1_6;
852}
853EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
854
855static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
856{
857 /*
858 * If both host and device support UniPro ver1.6 or later, PA layer
859 * parameters tuning happens during link startup itself.
860 *
861 * We can manually tune PA layer parameters if either host or device
862 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
863 * logic simple, we will only do manual tuning if local unipro version
864 * doesn't support ver1.6 or later.
865 */
866 if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
867 return true;
868 else
869 return false;
870}
871
a3cd5ec5
SJ		 872static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
873{
874 int ret = 0;
875 struct ufs_clk_info *clki;
876 struct list_head *head = &hba->clk_list_head;
877 ktime_t start = ktime_get();
878 bool clk_state_changed = false;
879
566ec9ad 880 if (list_empty(head))
a3cd5ec5
SJ		 881 goto out;
882
883 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
884 if (ret)
885 return ret;
886
887 list_for_each_entry(clki, head, list) {
888 if (!IS_ERR_OR_NULL(clki->clk)) {
889 if (scale_up && clki->max_freq) {
890 if (clki->curr_freq == clki->max_freq)
891 continue;
892
893 clk_state_changed = true;
894 ret = clk_set_rate(clki->clk, clki->max_freq);
895 if (ret) {
896 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
897 __func__, clki->name,
898 clki->max_freq, ret);
899 break;
900 }
901 trace_ufshcd_clk_scaling(dev_name(hba->dev),
902 "scaled up", clki->name,
903 clki->curr_freq,
904 clki->max_freq);
905
906 clki->curr_freq = clki->max_freq;
907
908 } else if (!scale_up && clki->min_freq) {
909 if (clki->curr_freq == clki->min_freq)
910 continue;
911
912 clk_state_changed = true;
913 ret = clk_set_rate(clki->clk, clki->min_freq);
914 if (ret) {
915 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
916 __func__, clki->name,
917 clki->min_freq, ret);
918 break;
919 }
920 trace_ufshcd_clk_scaling(dev_name(hba->dev),
921 "scaled down", clki->name,
922 clki->curr_freq,
923 clki->min_freq);
924 clki->curr_freq = clki->min_freq;
925 }
926 }
927 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
928 clki->name, clk_get_rate(clki->clk));
929 }
930
931 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
932
933out:
934 if (clk_state_changed)
935 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
936 (scale_up ? "up" : "down"),
937 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
938 return ret;
939}
940
941/**
942 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
943 * @hba: per adapter instance
944 * @scale_up: True if scaling up and false if scaling down
945 *
946 * Returns true if scaling is required, false otherwise.
947 */
948static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
949 bool scale_up)
950{
951 struct ufs_clk_info *clki;
952 struct list_head *head = &hba->clk_list_head;
953
566ec9ad 954 if (list_empty(head))
a3cd5ec5
SJ		 955 return false;
956
957 list_for_each_entry(clki, head, list) {
958 if (!IS_ERR_OR_NULL(clki->clk)) {
959 if (scale_up && clki->max_freq) {
960 if (clki->curr_freq == clki->max_freq)
961 continue;
962 return true;
963 } else if (!scale_up && clki->min_freq) {
964 if (clki->curr_freq == clki->min_freq)
965 continue;
966 return true;
967 }
968 }
969 }
970
971 return false;
972}
973
974static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
975 u64 wait_timeout_us)
976{
977 unsigned long flags;
978 int ret = 0;
979 u32 tm_doorbell;
980 u32 tr_doorbell;
981 bool timeout = false, do_last_check = false;
982 ktime_t start;
983
984 ufshcd_hold(hba, false);
985 spin_lock_irqsave(hba->host->host_lock, flags);
986 /*
987 * Wait for all the outstanding tasks/transfer requests.
988 * Verify by checking the doorbell registers are clear.
989 */
990 start = ktime_get();
991 do {
992 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
993 ret = -EBUSY;
994 goto out;
995 }
996
997 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
998 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
999 if (!tm_doorbell && !tr_doorbell) {
1000 timeout = false;
1001 break;
1002 } else if (do_last_check) {
1003 break;
1004 }
1005
1006 spin_unlock_irqrestore(hba->host->host_lock, flags);
1007 schedule();
1008 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1009 wait_timeout_us) {
1010 timeout = true;
1011 /*
1012 * We might have scheduled out for long time so make
1013 * sure to check if doorbells are cleared by this time
1014 * or not.
1015 */
1016 do_last_check = true;
1017 }
1018 spin_lock_irqsave(hba->host->host_lock, flags);
1019 } while (tm_doorbell || tr_doorbell);
1020
1021 if (timeout) {
1022 dev_err(hba->dev,
1023 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1024 __func__, tm_doorbell, tr_doorbell);
1025 ret = -EBUSY;
1026 }
1027out:
1028 spin_unlock_irqrestore(hba->host->host_lock, flags);
1029 ufshcd_release(hba);
1030 return ret;
1031}
1032
1033/**
1034 * ufshcd_scale_gear - scale up/down UFS gear
1035 * @hba: per adapter instance
1036 * @scale_up: True for scaling up gear and false for scaling down
1037 *
1038 * Returns 0 for success,
1039 * Returns -EBUSY if scaling can't happen at this time
1040 * Returns non-zero for any other errors
1041 */
1042static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1043{
1044 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1045 int ret = 0;
1046 struct ufs_pa_layer_attr new_pwr_info;
1047
1048 if (scale_up) {
1049 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1050 sizeof(struct ufs_pa_layer_attr));
1051 } else {
1052 memcpy(&new_pwr_info, &hba->pwr_info,
1053 sizeof(struct ufs_pa_layer_attr));
1054
1055 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1056 || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1057 /* save the current power mode */
1058 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1059 &hba->pwr_info,
1060 sizeof(struct ufs_pa_layer_attr));
1061
1062 /* scale down gear */
1063 new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1064 new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1065 }
1066 }
1067
1068 /* check if the power mode needs to be changed or not? */
1069 ret = ufshcd_change_power_mode(hba, &new_pwr_info);
1070
1071 if (ret)
1072 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1073 __func__, ret,
1074 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1075 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1076
1077 return ret;
1078}
1079
1080static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1081{
1082 #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
1083 int ret = 0;
1084 /*
1085 * make sure that there are no outstanding requests when
1086 * clock scaling is in progress
1087 */
38135535 1088 ufshcd_scsi_block_requests(hba);
a3cd5ec5
SJ		 1089 down_write(&hba->clk_scaling_lock);
1090 if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1091 ret = -EBUSY;
1092 up_write(&hba->clk_scaling_lock);
38135535 1093 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5
SJ		 1094 }
1095
1096 return ret;
1097}
1098
1099static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1100{
1101 up_write(&hba->clk_scaling_lock);
38135535 1102 ufshcd_scsi_unblock_requests(hba);
a3cd5ec5
SJ		 1103}
1104
1105/**
1106 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1107 * @hba: per adapter instance
1108 * @scale_up: True for scaling up and false for scalin down
1109 *
1110 * Returns 0 for success,
1111 * Returns -EBUSY if scaling can't happen at this time
1112 * Returns non-zero for any other errors
1113 */
1114static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1115{
1116 int ret = 0;
1117
401f1e44
SJ		 1118 /* let's not get into low power until clock scaling is completed */
1119 ufshcd_hold(hba, false);
1120
a3cd5ec5
SJ		 1121 ret = ufshcd_clock_scaling_prepare(hba);
1122 if (ret)
1123 return ret;
1124
1125 /* scale down the gear before scaling down clocks */
1126 if (!scale_up) {
1127 ret = ufshcd_scale_gear(hba, false);
1128 if (ret)
1129 goto out;
1130 }
1131
1132 ret = ufshcd_scale_clks(hba, scale_up);
1133 if (ret) {
1134 if (!scale_up)
1135 ufshcd_scale_gear(hba, true);
1136 goto out;
1137 }
1138
1139 /* scale up the gear after scaling up clocks */
1140 if (scale_up) {
1141 ret = ufshcd_scale_gear(hba, true);
1142 if (ret) {
1143 ufshcd_scale_clks(hba, false);
1144 goto out;
1145 }
1146 }
1147
1148 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1149
1150out:
1151 ufshcd_clock_scaling_unprepare(hba);
401f1e44 1152 ufshcd_release(hba);
a3cd5ec5
SJ		 1153 return ret;
1154}
1155
401f1e44
SJ		 1156static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1157{
1158 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1159 clk_scaling.suspend_work);
1160 unsigned long irq_flags;
1161
1162 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1163 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1164 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1165 return;
1166 }
1167 hba->clk_scaling.is_suspended = true;
1168 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1169
1170 __ufshcd_suspend_clkscaling(hba);
1171}
1172
1173static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1174{
1175 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1176 clk_scaling.resume_work);
1177 unsigned long irq_flags;
1178
1179 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1180 if (!hba->clk_scaling.is_suspended) {
1181 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1182 return;
1183 }
1184 hba->clk_scaling.is_suspended = false;
1185 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1186
1187 devfreq_resume_device(hba->devfreq);
1188}
1189
a3cd5ec5
SJ		 1190static int ufshcd_devfreq_target(struct device *dev,
1191 unsigned long *freq, u32 flags)
1192{
1193 int ret = 0;
1194 struct ufs_hba *hba = dev_get_drvdata(dev);
1195 ktime_t start;
401f1e44 1196 bool scale_up, sched_clk_scaling_suspend_work = false;
092b4558
BA		 1197 struct list_head *clk_list = &hba->clk_list_head;
1198 struct ufs_clk_info *clki;
a3cd5ec5
SJ		 1199 unsigned long irq_flags;
1200
1201 if (!ufshcd_is_clkscaling_supported(hba))
1202 return -EINVAL;
1203
a3cd5ec5
SJ		 1204 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1205 if (ufshcd_eh_in_progress(hba)) {
1206 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1207 return 0;
1208 }
1209
401f1e44
SJ		 1210 if (!hba->clk_scaling.active_reqs)
1211 sched_clk_scaling_suspend_work = true;
1212
092b4558
BA		 1213 if (list_empty(clk_list)) {
1214 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1215 goto out;
1216 }
1217
1218 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1219 scale_up = (*freq == clki->max_freq) ? true : false;
401f1e44
SJ		 1220 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1221 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1222 ret = 0;
1223 goto out; /* no state change required */
a3cd5ec5
SJ		 1224 }
1225 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1226
1227 start = ktime_get();
a3cd5ec5
SJ		 1228 ret = ufshcd_devfreq_scale(hba, scale_up);
1229
a3cd5ec5
SJ		 1230 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1231 (scale_up ? "up" : "down"),
1232 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1233
401f1e44
SJ		 1234out:
1235 if (sched_clk_scaling_suspend_work)
1236 queue_work(hba->clk_scaling.workq,
1237 &hba->clk_scaling.suspend_work);
1238
a3cd5ec5
SJ		 1239 return ret;
1240}
1241
7252a360
BVA		 1242static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1243{
1244 int *busy = priv;
1245
1246 WARN_ON_ONCE(reserved);
1247 (*busy)++;
1248 return false;
1249}
1250
1251/* Whether or not any tag is in use by a request that is in progress. */
1252static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1253{
1254 struct request_queue *q = hba->cmd_queue;
1255 int busy = 0;
1256
1257 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1258 return busy;
1259}
a3cd5ec5
SJ		 1260
1261static int ufshcd_devfreq_get_dev_status(struct device *dev,
1262 struct devfreq_dev_status *stat)
1263{
1264 struct ufs_hba *hba = dev_get_drvdata(dev);
1265 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1266 unsigned long flags;
1267
1268 if (!ufshcd_is_clkscaling_supported(hba))
1269 return -EINVAL;
1270
1271 memset(stat, 0, sizeof(*stat));
1272
1273 spin_lock_irqsave(hba->host->host_lock, flags);
1274 if (!scaling->window_start_t)
1275 goto start_window;
1276
1277 if (scaling->is_busy_started)
1278 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1279 scaling->busy_start_t));
1280
1281 stat->total_time = jiffies_to_usecs((long)jiffies -
1282 (long)scaling->window_start_t);
1283 stat->busy_time = scaling->tot_busy_t;
1284start_window:
1285 scaling->window_start_t = jiffies;
1286 scaling->tot_busy_t = 0;
1287
1288 if (hba->outstanding_reqs) {
1289 scaling->busy_start_t = ktime_get();
1290 scaling->is_busy_started = true;
1291 } else {
1292 scaling->busy_start_t = 0;
1293 scaling->is_busy_started = false;
1294 }
1295 spin_unlock_irqrestore(hba->host->host_lock, flags);
1296 return 0;
1297}
1298
1299static struct devfreq_dev_profile ufs_devfreq_profile = {
1300 .polling_ms = 100,
1301 .target = ufshcd_devfreq_target,
1302 .get_dev_status = ufshcd_devfreq_get_dev_status,
1303};
1304
deac444f
BA		 1305static int ufshcd_devfreq_init(struct ufs_hba *hba)
1306{
092b4558
BA		 1307 struct list_head *clk_list = &hba->clk_list_head;
1308 struct ufs_clk_info *clki;
deac444f
BA		 1309 struct devfreq *devfreq;
1310 int ret;
1311
092b4558
BA		 1312 /* Skip devfreq if we don't have any clocks in the list */
1313 if (list_empty(clk_list))
1314 return 0;
1315
1316 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1317 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1318 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1319
1320 devfreq = devfreq_add_device(hba->dev,
deac444f
BA		 1321 &ufs_devfreq_profile,
1322 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1323 NULL);
1324 if (IS_ERR(devfreq)) {
1325 ret = PTR_ERR(devfreq);
1326 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
092b4558
BA		 1327
1328 dev_pm_opp_remove(hba->dev, clki->min_freq);
1329 dev_pm_opp_remove(hba->dev, clki->max_freq);
deac444f
BA		 1330 return ret;
1331 }
1332
1333 hba->devfreq = devfreq;
1334
1335 return 0;
1336}
1337
092b4558
BA		 1338static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1339{
1340 struct list_head *clk_list = &hba->clk_list_head;
1341 struct ufs_clk_info *clki;
1342
1343 if (!hba->devfreq)
1344 return;
1345
1346 devfreq_remove_device(hba->devfreq);
1347 hba->devfreq = NULL;
1348
1349 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1350 dev_pm_opp_remove(hba->dev, clki->min_freq);
1351 dev_pm_opp_remove(hba->dev, clki->max_freq);
1352}
1353
401f1e44
SJ		 1354static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1355{
1356 unsigned long flags;
1357
1358 devfreq_suspend_device(hba->devfreq);
1359 spin_lock_irqsave(hba->host->host_lock, flags);
1360 hba->clk_scaling.window_start_t = 0;
1361 spin_unlock_irqrestore(hba->host->host_lock, flags);
1362}
a3cd5ec5 1363
a508253d
GB		 1364static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1365{
401f1e44
SJ		 1366 unsigned long flags;
1367 bool suspend = false;
1368
fcb0c4b0
ST		 1369 if (!ufshcd_is_clkscaling_supported(hba))
1370 return;
1371
401f1e44
SJ		 1372 spin_lock_irqsave(hba->host->host_lock, flags);
1373 if (!hba->clk_scaling.is_suspended) {
1374 suspend = true;
1375 hba->clk_scaling.is_suspended = true;
1376 }
1377 spin_unlock_irqrestore(hba->host->host_lock, flags);
1378
1379 if (suspend)
1380 __ufshcd_suspend_clkscaling(hba);
a508253d
GB		 1381}
1382
1383static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1384{
401f1e44
SJ		 1385 unsigned long flags;
1386 bool resume = false;
1387
1388 if (!ufshcd_is_clkscaling_supported(hba))
1389 return;
1390
1391 spin_lock_irqsave(hba->host->host_lock, flags);
1392 if (hba->clk_scaling.is_suspended) {
1393 resume = true;
1394 hba->clk_scaling.is_suspended = false;
1395 }
1396 spin_unlock_irqrestore(hba->host->host_lock, flags);
1397
1398 if (resume)
1399 devfreq_resume_device(hba->devfreq);
fcb0c4b0
ST		 1400}
1401
1402static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1403 struct device_attribute *attr, char *buf)
1404{
1405 struct ufs_hba *hba = dev_get_drvdata(dev);
1406
1407 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1408}
1409
1410static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1411 struct device_attribute *attr, const char *buf, size_t count)
1412{
1413 struct ufs_hba *hba = dev_get_drvdata(dev);
1414 u32 value;
1415 int err;
1416
1417 if (kstrtou32(buf, 0, &value))
1418 return -EINVAL;
1419
1420 value = !!value;
1421 if (value == hba->clk_scaling.is_allowed)
1422 goto out;
1423
1424 pm_runtime_get_sync(hba->dev);
1425 ufshcd_hold(hba, false);
1426
401f1e44
SJ		 1427 cancel_work_sync(&hba->clk_scaling.suspend_work);
1428 cancel_work_sync(&hba->clk_scaling.resume_work);
1429
1430 hba->clk_scaling.is_allowed = value;
1431
fcb0c4b0
ST		 1432 if (value) {
1433 ufshcd_resume_clkscaling(hba);
1434 } else {
1435 ufshcd_suspend_clkscaling(hba);
a3cd5ec5 1436 err = ufshcd_devfreq_scale(hba, true);
fcb0c4b0
ST		 1437 if (err)
1438 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1439 __func__, err);
1440 }
fcb0c4b0
ST		 1441
1442 ufshcd_release(hba);
1443 pm_runtime_put_sync(hba->dev);
1444out:
1445 return count;
a508253d
GB		 1446}
1447
a3cd5ec5
SJ		 1448static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1449{
1450 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1451 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1452 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1453 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1454 hba->clk_scaling.enable_attr.attr.mode = 0644;
1455 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1456 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1457}
1458
1ab27c9c
ST		 1459static void ufshcd_ungate_work(struct work_struct *work)
1460{
1461 int ret;
1462 unsigned long flags;
1463 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1464 clk_gating.ungate_work);
1465
1466 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1467
1468 spin_lock_irqsave(hba->host->host_lock, flags);
1469 if (hba->clk_gating.state == CLKS_ON) {
1470 spin_unlock_irqrestore(hba->host->host_lock, flags);
1471 goto unblock_reqs;
1472 }
1473
1474 spin_unlock_irqrestore(hba->host->host_lock, flags);
1475 ufshcd_setup_clocks(hba, true);
1476
1477 /* Exit from hibern8 */
1478 if (ufshcd_can_hibern8_during_gating(hba)) {
1479 /* Prevent gating in this path */
1480 hba->clk_gating.is_suspended = true;
1481 if (ufshcd_is_link_hibern8(hba)) {
1482 ret = ufshcd_uic_hibern8_exit(hba);
1483 if (ret)
1484 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1485 __func__, ret);
1486 else
1487 ufshcd_set_link_active(hba);
1488 }
1489 hba->clk_gating.is_suspended = false;
1490 }
1491unblock_reqs:
38135535 1492 ufshcd_scsi_unblock_requests(hba);
1ab27c9c
ST		 1493}
1494
1495/**
1496 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1497 * Also, exit from hibern8 mode and set the link as active.
1498 * @hba: per adapter instance
1499 * @async: This indicates whether caller should ungate clocks asynchronously.
1500 */
1501int ufshcd_hold(struct ufs_hba *hba, bool async)
1502{
1503 int rc = 0;
1504 unsigned long flags;
1505
1506 if (!ufshcd_is_clkgating_allowed(hba))
1507 goto out;
1ab27c9c
ST		 1508 spin_lock_irqsave(hba->host->host_lock, flags);
1509 hba->clk_gating.active_reqs++;
1510
53c12d0e
YG		 1511 if (ufshcd_eh_in_progress(hba)) {
1512 spin_unlock_irqrestore(hba->host->host_lock, flags);
1513 return 0;
1514 }
1515
856b3483 1516start:
1ab27c9c
ST		 1517 switch (hba->clk_gating.state) {
1518 case CLKS_ON:
f2a785ac
VG		 1519 /*
1520 * Wait for the ungate work to complete if in progress.
1521 * Though the clocks may be in ON state, the link could
1522 * still be in hibner8 state if hibern8 is allowed
1523 * during clock gating.
1524 * Make sure we exit hibern8 state also in addition to
1525 * clocks being ON.
1526 */
1527 if (ufshcd_can_hibern8_during_gating(hba) &&
1528 ufshcd_is_link_hibern8(hba)) {
1529 spin_unlock_irqrestore(hba->host->host_lock, flags);
1530 flush_work(&hba->clk_gating.ungate_work);
1531 spin_lock_irqsave(hba->host->host_lock, flags);
1532 goto start;
1533 }
1ab27c9c
ST		 1534 break;
1535 case REQ_CLKS_OFF:
1536 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1537 hba->clk_gating.state = CLKS_ON;
7ff5ab47
SJ		 1538 trace_ufshcd_clk_gating(dev_name(hba->dev),
1539 hba->clk_gating.state);
1ab27c9c
ST		 1540 break;
1541 }
1542 /*
9c490d2d 1543 * If we are here, it means gating work is either done or
1ab27c9c
ST		 1544 * currently running. Hence, fall through to cancel gating
1545 * work and to enable clocks.
1546 */
30eb2e4c 1547 /* fallthrough */
1ab27c9c 1548 case CLKS_OFF:
38135535 1549 ufshcd_scsi_block_requests(hba);
1ab27c9c 1550 hba->clk_gating.state = REQ_CLKS_ON;
7ff5ab47
SJ		 1551 trace_ufshcd_clk_gating(dev_name(hba->dev),
1552 hba->clk_gating.state);
10e5e375
VV		 1553 queue_work(hba->clk_gating.clk_gating_workq,
1554 &hba->clk_gating.ungate_work);
1ab27c9c
ST		 1555 /*
1556 * fall through to check if we should wait for this
1557 * work to be done or not.
1558 */
30eb2e4c 1559 /* fallthrough */
1ab27c9c
ST		 1560 case REQ_CLKS_ON:
1561 if (async) {
1562 rc = -EAGAIN;
1563 hba->clk_gating.active_reqs--;
1564 break;
1565 }
1566
1567 spin_unlock_irqrestore(hba->host->host_lock, flags);
1568 flush_work(&hba->clk_gating.ungate_work);
1569 /* Make sure state is CLKS_ON before returning */
856b3483 1570 spin_lock_irqsave(hba->host->host_lock, flags);
1ab27c9c
ST		 1571 goto start;
1572 default:
1573 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1574 __func__, hba->clk_gating.state);
1575 break;
1576 }
1577 spin_unlock_irqrestore(hba->host->host_lock, flags);
1578out:
1579 return rc;
1580}
6e3fd44d 1581EXPORT_SYMBOL_GPL(ufshcd_hold);
1ab27c9c
ST		 1582
1583static void ufshcd_gate_work(struct work_struct *work)
1584{
1585 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1586 clk_gating.gate_work.work);
1587 unsigned long flags;
1588
1589 spin_lock_irqsave(hba->host->host_lock, flags);
3f0c06de
VG		 1590 /*
1591 * In case you are here to cancel this work the gating state
1592 * would be marked as REQ_CLKS_ON. In this case save time by
1593 * skipping the gating work and exit after changing the clock
1594 * state to CLKS_ON.
1595 */
1596 if (hba->clk_gating.is_suspended ||
18f01374 1597 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1ab27c9c 1598 hba->clk_gating.state = CLKS_ON;
7ff5ab47
SJ		 1599 trace_ufshcd_clk_gating(dev_name(hba->dev),
1600 hba->clk_gating.state);
1ab27c9c
ST		 1601 goto rel_lock;
1602 }
1603
1604 if (hba->clk_gating.active_reqs
1605 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
7252a360 1606 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1ab27c9c
ST		 1607 || hba->active_uic_cmd || hba->uic_async_done)
1608 goto rel_lock;
1609
1610 spin_unlock_irqrestore(hba->host->host_lock, flags);
1611
1612 /* put the link into hibern8 mode before turning off clocks */
1613 if (ufshcd_can_hibern8_during_gating(hba)) {
1614 if (ufshcd_uic_hibern8_enter(hba)) {
1615 hba->clk_gating.state = CLKS_ON;
7ff5ab47
SJ		 1616 trace_ufshcd_clk_gating(dev_name(hba->dev),
1617 hba->clk_gating.state);
1ab27c9c
ST		 1618 goto out;
1619 }
1620 ufshcd_set_link_hibern8(hba);
1621 }
1622
1623 if (!ufshcd_is_link_active(hba))
1624 ufshcd_setup_clocks(hba, false);
1625 else
1626 /* If link is active, device ref_clk can't be switched off */
1627 __ufshcd_setup_clocks(hba, false, true);
1628
1629 /*
1630 * In case you are here to cancel this work the gating state
1631 * would be marked as REQ_CLKS_ON. In this case keep the state
1632 * as REQ_CLKS_ON which would anyway imply that clocks are off
1633 * and a request to turn them on is pending. By doing this way,
1634 * we keep the state machine in tact and this would ultimately
1635 * prevent from doing cancel work multiple times when there are
1636 * new requests arriving before the current cancel work is done.
1637 */
1638 spin_lock_irqsave(hba->host->host_lock, flags);
7ff5ab47 1639 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1ab27c9c 1640 hba->clk_gating.state = CLKS_OFF;
7ff5ab47
SJ		 1641 trace_ufshcd_clk_gating(dev_name(hba->dev),
1642 hba->clk_gating.state);
1643 }
1ab27c9c
ST		 1644rel_lock:
1645 spin_unlock_irqrestore(hba->host->host_lock, flags);
1646out:
1647 return;
1648}
1649
1650/* host lock must be held before calling this variant */
1651static void __ufshcd_release(struct ufs_hba *hba)
1652{
1653 if (!ufshcd_is_clkgating_allowed(hba))
1654 return;
1655
1656 hba->clk_gating.active_reqs--;
1657
1658 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
1659 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
7252a360 1660 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
53c12d0e
YG		 1661 || hba->active_uic_cmd || hba->uic_async_done
1662 || ufshcd_eh_in_progress(hba))
1ab27c9c
ST		 1663 return;
1664
1665 hba->clk_gating.state = REQ_CLKS_OFF;
7ff5ab47 1666 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
f4bb7704
EG		 1667 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1668 &hba->clk_gating.gate_work,
1669 msecs_to_jiffies(hba->clk_gating.delay_ms));
1ab27c9c
ST		 1670}
1671
1672void ufshcd_release(struct ufs_hba *hba)
1673{
1674 unsigned long flags;
1675
1676 spin_lock_irqsave(hba->host->host_lock, flags);
1677 __ufshcd_release(hba);
1678 spin_unlock_irqrestore(hba->host->host_lock, flags);
1679}
6e3fd44d 1680EXPORT_SYMBOL_GPL(ufshcd_release);
1ab27c9c
ST		 1681
1682static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1683 struct device_attribute *attr, char *buf)
1684{
1685 struct ufs_hba *hba = dev_get_drvdata(dev);
1686
1687 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1688}
1689
1690static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1691 struct device_attribute *attr, const char *buf, size_t count)
1692{
1693 struct ufs_hba *hba = dev_get_drvdata(dev);
1694 unsigned long flags, value;
1695
1696 if (kstrtoul(buf, 0, &value))
1697 return -EINVAL;
1698
1699 spin_lock_irqsave(hba->host->host_lock, flags);
1700 hba->clk_gating.delay_ms = value;
1701 spin_unlock_irqrestore(hba->host->host_lock, flags);
1702 return count;
1703}
1704
b427411a
ST		 1705static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1706 struct device_attribute *attr, char *buf)
1707{
1708 struct ufs_hba *hba = dev_get_drvdata(dev);
1709
1710 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1711}
1712
1713static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1714 struct device_attribute *attr, const char *buf, size_t count)
1715{
1716 struct ufs_hba *hba = dev_get_drvdata(dev);
1717 unsigned long flags;
1718 u32 value;
1719
1720 if (kstrtou32(buf, 0, &value))
1721 return -EINVAL;
1722
1723 value = !!value;
1724 if (value == hba->clk_gating.is_enabled)
1725 goto out;
1726
1727 if (value) {
1728 ufshcd_release(hba);
1729 } else {
1730 spin_lock_irqsave(hba->host->host_lock, flags);
1731 hba->clk_gating.active_reqs++;
1732 spin_unlock_irqrestore(hba->host->host_lock, flags);
1733 }
1734
1735 hba->clk_gating.is_enabled = value;
1736out:
1737 return count;
1738}
1739
eebcc196
VG		 1740static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1741{
1742 char wq_name[sizeof("ufs_clkscaling_00")];
1743
1744 if (!ufshcd_is_clkscaling_supported(hba))
1745 return;
1746
1747 INIT_WORK(&hba->clk_scaling.suspend_work,
1748 ufshcd_clk_scaling_suspend_work);
1749 INIT_WORK(&hba->clk_scaling.resume_work,
1750 ufshcd_clk_scaling_resume_work);
1751
1752 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1753 hba->host->host_no);
1754 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1755
1756 ufshcd_clkscaling_init_sysfs(hba);
1757}
1758
1759static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1760{
1761 if (!ufshcd_is_clkscaling_supported(hba))
1762 return;
1763
1764 destroy_workqueue(hba->clk_scaling.workq);
1765 ufshcd_devfreq_remove(hba);
1766}
1767
1ab27c9c
ST		 1768static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1769{
10e5e375
VV		 1770 char wq_name[sizeof("ufs_clk_gating_00")];
1771
1ab27c9c
ST		 1772 if (!ufshcd_is_clkgating_allowed(hba))
1773 return;
1774
1775 hba->clk_gating.delay_ms = 150;
1776 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1777 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1778
10e5e375
VV		 1779 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1780 hba->host->host_no);
1781 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1782 WQ_MEM_RECLAIM);
1783
b427411a
ST		 1784 hba->clk_gating.is_enabled = true;
1785
1ab27c9c
ST		 1786 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1787 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1788 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1789 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
b427411a 1790 hba->clk_gating.delay_attr.attr.mode = 0644;
1ab27c9c
ST		 1791 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1792 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
b427411a
ST		 1793
1794 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1795 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1796 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1797 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1798 hba->clk_gating.enable_attr.attr.mode = 0644;
1799 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1800 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1ab27c9c
ST		 1801}
1802
1803static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1804{
1805 if (!ufshcd_is_clkgating_allowed(hba))
1806 return;
1807 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
b427411a 1808 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
97cd6805
AM		 1809 cancel_work_sync(&hba->clk_gating.ungate_work);
1810 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
10e5e375 1811 destroy_workqueue(hba->clk_gating.clk_gating_workq);
1ab27c9c
ST		 1812}
1813
856b3483
ST		 1814/* Must be called with host lock acquired */
1815static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1816{
401f1e44
SJ		 1817 bool queue_resume_work = false;
1818
fcb0c4b0 1819 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1820 return;
1821
401f1e44
SJ		 1822 if (!hba->clk_scaling.active_reqs++)
1823 queue_resume_work = true;
1824
1825 if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1826 return;
1827
1828 if (queue_resume_work)
1829 queue_work(hba->clk_scaling.workq,
1830 &hba->clk_scaling.resume_work);
1831
1832 if (!hba->clk_scaling.window_start_t) {
1833 hba->clk_scaling.window_start_t = jiffies;
1834 hba->clk_scaling.tot_busy_t = 0;
1835 hba->clk_scaling.is_busy_started = false;
1836 }
1837
856b3483
ST		 1838 if (!hba->clk_scaling.is_busy_started) {
1839 hba->clk_scaling.busy_start_t = ktime_get();
1840 hba->clk_scaling.is_busy_started = true;
1841 }
1842}
1843
1844static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1845{
1846 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1847
fcb0c4b0 1848 if (!ufshcd_is_clkscaling_supported(hba))
856b3483
ST		 1849 return;
1850
1851 if (!hba->outstanding_reqs && scaling->is_busy_started) {
1852 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1853 scaling->busy_start_t));
8b0e1953 1854 scaling->busy_start_t = 0;
856b3483
ST		 1855 scaling->is_busy_started = false;
1856 }
1857}
7a3e97b0
SY		 1858/**
1859 * ufshcd_send_command - Send SCSI or device management commands
1860 * @hba: per adapter instance
1861 * @task_tag: Task tag of the command
1862 */
1863static inline
1864void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1865{
ff8e20c6 1866 hba->lrb[task_tag].issue_time_stamp = ktime_get();
09017188 1867 hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0);
856b3483 1868 ufshcd_clk_scaling_start_busy(hba);
7a3e97b0 1869 __set_bit(task_tag, &hba->outstanding_reqs);
b873a275 1870 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
ad1a1b9c
GB		 1871 /* Make sure that doorbell is committed immediately */
1872 wmb();
1a07f2d9 1873 ufshcd_add_command_trace(hba, task_tag, "send");
7a3e97b0
SY		 1874}
1875
1876/**
1877 * ufshcd_copy_sense_data - Copy sense data in case of check condition
8aa29f19 1878 * @lrbp: pointer to local reference block
7a3e97b0
SY		 1879 */
1880static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1881{
1882 int len;
1c2623c5
SJ		 1883 if (lrbp->sense_buffer &&
1884 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
e3ce73d6
YG		 1885 int len_to_copy;
1886
5a0b0cb9 1887 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
09a5a24f 1888 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
e3ce73d6 1889
09a5a24f
AA		 1890 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
1891 len_to_copy);
7a3e97b0
SY		 1892 }
1893}
1894
68078d5c
DR		 1895/**
1896 * ufshcd_copy_query_response() - Copy the Query Response and the data
1897 * descriptor
1898 * @hba: per adapter instance
8aa29f19 1899 * @lrbp: pointer to local reference block
68078d5c
DR		 1900 */
1901static
c6d4a831 1902int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
68078d5c
DR		 1903{
1904 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1905
68078d5c 1906 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
68078d5c 1907
68078d5c 1908 /* Get the descriptor */
1c90836f
AA		 1909 if (hba->dev_cmd.query.descriptor &&
1910 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
d44a5f98 1911 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
68078d5c 1912 GENERAL_UPIU_REQUEST_SIZE;
c6d4a831
DR		 1913 u16 resp_len;
1914 u16 buf_len;
68078d5c
DR		 1915
1916 /* data segment length */
c6d4a831 1917 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
68078d5c 1918 MASK_QUERY_DATA_SEG_LEN;
ea2aab24
SRT		 1919 buf_len = be16_to_cpu(
1920 hba->dev_cmd.query.request.upiu_req.length);
c6d4a831
DR		 1921 if (likely(buf_len >= resp_len)) {
1922 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
1923 } else {
1924 dev_warn(hba->dev,
3d4881d1
BH		 1925 "%s: rsp size %d is bigger than buffer size %d",
1926 __func__, resp_len, buf_len);
c6d4a831
DR		 1927 return -EINVAL;
1928 }
68078d5c 1929 }
c6d4a831
DR		 1930
1931 return 0;
68078d5c
DR		 1932}
1933
7a3e97b0
SY		 1934/**
1935 * ufshcd_hba_capabilities - Read controller capabilities
1936 * @hba: per adapter instance
1937 */
1938static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
1939{
b873a275 1940 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
7a3e97b0
SY		 1941
1942 /* nutrs and nutmrs are 0 based values */
1943 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
1944 hba->nutmrs =
1945 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
1946}
1947
1948/**
6ccf44fe
SJ		 1949 * ufshcd_ready_for_uic_cmd - Check if controller is ready
1950 * to accept UIC commands
7a3e97b0 1951 * @hba: per adapter instance
6ccf44fe
SJ		 1952 * Return true on success, else false
1953 */
1954static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
1955{
1956 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
1957 return true;
1958 else
1959 return false;
1960}
1961
53b3d9c3
SJ		 1962/**
1963 * ufshcd_get_upmcrs - Get the power mode change request status
1964 * @hba: Pointer to adapter instance
1965 *
1966 * This function gets the UPMCRS field of HCS register
1967 * Returns value of UPMCRS field
1968 */
1969static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
1970{
1971 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
1972}
1973
6ccf44fe
SJ		 1974/**
1975 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1976 * @hba: per adapter instance
1977 * @uic_cmd: UIC command
1978 *
1979 * Mutex must be held.
7a3e97b0
SY		 1980 */
1981static inline void
6ccf44fe 1982ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
7a3e97b0 1983{
6ccf44fe
SJ		 1984 WARN_ON(hba->active_uic_cmd);
1985
1986 hba->active_uic_cmd = uic_cmd;
1987
7a3e97b0 1988 /* Write Args */
6ccf44fe
SJ		 1989 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
1990 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
1991 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
7a3e97b0
SY		 1992
1993 /* Write UIC Cmd */
6ccf44fe 1994 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
b873a275 1995 REG_UIC_COMMAND);
7a3e97b0
SY		 1996}
1997
6ccf44fe
SJ		 1998/**
1999 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2000 * @hba: per adapter instance
8aa29f19 2001 * @uic_cmd: UIC command
6ccf44fe
SJ		 2002 *
2003 * Must be called with mutex held.
2004 * Returns 0 only if success.
2005 */
2006static int
2007ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2008{
2009 int ret;
2010 unsigned long flags;
2011
2012 if (wait_for_completion_timeout(&uic_cmd->done,
2013 msecs_to_jiffies(UIC_CMD_TIMEOUT)))
2014 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2015 else
2016 ret = -ETIMEDOUT;
2017
2018 spin_lock_irqsave(hba->host->host_lock, flags);
2019 hba->active_uic_cmd = NULL;
2020 spin_unlock_irqrestore(hba->host->host_lock, flags);
2021
2022 return ret;
2023}
2024
2025/**
2026 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2027 * @hba: per adapter instance
2028 * @uic_cmd: UIC command
d75f7fe4 2029 * @completion: initialize the completion only if this is set to true
6ccf44fe
SJ		 2030 *
2031 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
57d104c1 2032 * with mutex held and host_lock locked.
6ccf44fe
SJ		 2033 * Returns 0 only if success.
2034 */
2035static int
d75f7fe4
YG		 2036__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2037 bool completion)
6ccf44fe 2038{
6ccf44fe
SJ		 2039 if (!ufshcd_ready_for_uic_cmd(hba)) {
2040 dev_err(hba->dev,
2041 "Controller not ready to accept UIC commands\n");
2042 return -EIO;
2043 }
2044
d75f7fe4
YG		 2045 if (completion)
2046 init_completion(&uic_cmd->done);
6ccf44fe 2047
6ccf44fe 2048 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
6ccf44fe 2049
57d104c1 2050 return 0;
6ccf44fe
SJ		 2051}
2052
2053/**
2054 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2055 * @hba: per adapter instance
2056 * @uic_cmd: UIC command
2057 *
2058 * Returns 0 only if success.
2059 */
e77044c5 2060int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
6ccf44fe
SJ		 2061{
2062 int ret;
57d104c1 2063 unsigned long flags;
6ccf44fe 2064
1ab27c9c 2065 ufshcd_hold(hba, false);
6ccf44fe 2066 mutex_lock(&hba->uic_cmd_mutex);
cad2e03d
YG		 2067 ufshcd_add_delay_before_dme_cmd(hba);
2068
57d104c1 2069 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 2070 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
57d104c1
SJ		 2071 spin_unlock_irqrestore(hba->host->host_lock, flags);
2072 if (!ret)
2073 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2074
6ccf44fe
SJ		 2075 mutex_unlock(&hba->uic_cmd_mutex);
2076
1ab27c9c 2077 ufshcd_release(hba);
6ccf44fe
SJ		 2078 return ret;
2079}
2080
7a3e97b0
SY		 2081/**
2082 * ufshcd_map_sg - Map scatter-gather list to prdt
8aa29f19
BVA		 2083 * @hba: per adapter instance
2084 * @lrbp: pointer to local reference block
7a3e97b0
SY		 2085 *
2086 * Returns 0 in case of success, non-zero value in case of failure
2087 */
75b1cc4a 2088static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0
SY		 2089{
2090 struct ufshcd_sg_entry *prd_table;
2091 struct scatterlist *sg;
2092 struct scsi_cmnd *cmd;
2093 int sg_segments;
2094 int i;
2095
2096 cmd = lrbp->cmd;
2097 sg_segments = scsi_dma_map(cmd);
2098 if (sg_segments < 0)
2099 return sg_segments;
2100
2101 if (sg_segments) {
75b1cc4a
KK		 2102 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2103 lrbp->utr_descriptor_ptr->prd_table_length =
2104 cpu_to_le16((u16)(sg_segments *
2105 sizeof(struct ufshcd_sg_entry)));
2106 else
2107 lrbp->utr_descriptor_ptr->prd_table_length =
2108 cpu_to_le16((u16) (sg_segments));
7a3e97b0
SY		 2109
2110 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2111
2112 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2113 prd_table[i].size =
2114 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2115 prd_table[i].base_addr =
2116 cpu_to_le32(lower_32_bits(sg->dma_address));
2117 prd_table[i].upper_addr =
2118 cpu_to_le32(upper_32_bits(sg->dma_address));
52ac95fe 2119 prd_table[i].reserved = 0;
7a3e97b0
SY		 2120 }
2121 } else {
2122 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2123 }
2124
2125 return 0;
2126}
2127
2128/**
2fbd009b 2129 * ufshcd_enable_intr - enable interrupts
7a3e97b0 2130 * @hba: per adapter instance
2fbd009b 2131 * @intrs: interrupt bits
7a3e97b0 2132 */
2fbd009b 2133static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
7a3e97b0 2134{
2fbd009b
SJ		 2135 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2136
2137 if (hba->ufs_version == UFSHCI_VERSION_10) {
2138 u32 rw;
2139 rw = set & INTERRUPT_MASK_RW_VER_10;
2140 set = rw | ((set ^ intrs) & intrs);
2141 } else {
2142 set |= intrs;
2143 }
2144
2145 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2146}
2147
2148/**
2149 * ufshcd_disable_intr - disable interrupts
2150 * @hba: per adapter instance
2151 * @intrs: interrupt bits
2152 */
2153static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2154{
2155 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2156
2157 if (hba->ufs_version == UFSHCI_VERSION_10) {
2158 u32 rw;
2159 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2160 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2161 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2162
2163 } else {
2164 set &= ~intrs;
7a3e97b0 2165 }
2fbd009b
SJ		 2166
2167 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
7a3e97b0
SY		 2168}
2169
5a0b0cb9
SRT		 2170/**
2171 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2172 * descriptor according to request
2173 * @lrbp: pointer to local reference block
2174 * @upiu_flags: flags required in the header
2175 * @cmd_dir: requests data direction
2176 */
2177static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
300bb13f 2178 u32 *upiu_flags, enum dma_data_direction cmd_dir)
5a0b0cb9
SRT		 2179{
2180 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2181 u32 data_direction;
2182 u32 dword_0;
2183
2184 if (cmd_dir == DMA_FROM_DEVICE) {
2185 data_direction = UTP_DEVICE_TO_HOST;
2186 *upiu_flags = UPIU_CMD_FLAGS_READ;
2187 } else if (cmd_dir == DMA_TO_DEVICE) {
2188 data_direction = UTP_HOST_TO_DEVICE;
2189 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2190 } else {
2191 data_direction = UTP_NO_DATA_TRANSFER;
2192 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2193 }
2194
2195 dword_0 = data_direction | (lrbp->command_type
2196 << UPIU_COMMAND_TYPE_OFFSET);
2197 if (lrbp->intr_cmd)
2198 dword_0 |= UTP_REQ_DESC_INT_CMD;
2199
2200 /* Transfer request descriptor header fields */
2201 req_desc->header.dword_0 = cpu_to_le32(dword_0);
52ac95fe
YG		 2202 /* dword_1 is reserved, hence it is set to 0 */
2203 req_desc->header.dword_1 = 0;
5a0b0cb9
SRT		 2204 /*
2205 * assigning invalid value for command status. Controller
2206 * updates OCS on command completion, with the command
2207 * status
2208 */
2209 req_desc->header.dword_2 =
2210 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
52ac95fe
YG		 2211 /* dword_3 is reserved, hence it is set to 0 */
2212 req_desc->header.dword_3 = 0;
51047266
YG		 2213
2214 req_desc->prd_table_length = 0;
5a0b0cb9
SRT		 2215}
2216
2217/**
2218 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2219 * for scsi commands
8aa29f19
BVA		 2220 * @lrbp: local reference block pointer
2221 * @upiu_flags: flags
5a0b0cb9
SRT		 2222 */
2223static
2224void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
2225{
2226 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
52ac95fe 2227 unsigned short cdb_len;
5a0b0cb9
SRT		 2228
2229 /* command descriptor fields */
2230 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2231 UPIU_TRANSACTION_COMMAND, upiu_flags,
2232 lrbp->lun, lrbp->task_tag);
2233 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2234 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2235
2236 /* Total EHS length and Data segment length will be zero */
2237 ucd_req_ptr->header.dword_2 = 0;
2238
2239 ucd_req_ptr->sc.exp_data_transfer_len =
2240 cpu_to_be32(lrbp->cmd->sdb.length);
2241
a851b2bd
AA		 2242 cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, UFS_CDB_SIZE);
2243 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
52ac95fe
YG		 2244 memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
2245
2246 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2247}
2248
68078d5c
DR		 2249/**
2250 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2251 * for query requsts
2252 * @hba: UFS hba
2253 * @lrbp: local reference block pointer
2254 * @upiu_flags: flags
2255 */
2256static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2257 struct ufshcd_lrb *lrbp, u32 upiu_flags)
2258{
2259 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2260 struct ufs_query *query = &hba->dev_cmd.query;
e8c8e82a 2261 u16 len = be16_to_cpu(query->request.upiu_req.length);
68078d5c
DR		 2262
2263 /* Query request header */
2264 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2265 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2266 lrbp->lun, lrbp->task_tag);
2267 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2268 0, query->request.query_func, 0, 0);
2269
6861285c
ZL		 2270 /* Data segment length only need for WRITE_DESC */
2271 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2272 ucd_req_ptr->header.dword_2 =
2273 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2274 else
2275 ucd_req_ptr->header.dword_2 = 0;
68078d5c
DR		 2276
2277 /* Copy the Query Request buffer as is */
2278 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2279 QUERY_OSF_SIZE);
68078d5c
DR		 2280
2281 /* Copy the Descriptor */
c6d4a831 2282 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
220d17a6 2283 memcpy(ucd_req_ptr + 1, query->descriptor, len);
c6d4a831 2284
51047266 2285 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
68078d5c
DR		 2286}
2287
5a0b0cb9
SRT		 2288static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2289{
2290 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2291
2292 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2293
2294 /* command descriptor fields */
2295 ucd_req_ptr->header.dword_0 =
2296 UPIU_HEADER_DWORD(
2297 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
51047266
YG		 2298 /* clear rest of the fields of basic header */
2299 ucd_req_ptr->header.dword_1 = 0;
2300 ucd_req_ptr->header.dword_2 = 0;
2301
2302 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5a0b0cb9
SRT		 2303}
2304
7a3e97b0 2305/**
300bb13f
JP		 2306 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2307 * for Device Management Purposes
8aa29f19
BVA		 2308 * @hba: per adapter instance
2309 * @lrbp: pointer to local reference block
7a3e97b0 2310 */
300bb13f 2311static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
7a3e97b0 2312{
7a3e97b0 2313 u32 upiu_flags;
5a0b0cb9 2314 int ret = 0;
7a3e97b0 2315
83dc7e3d 2316 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2317 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2318 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
83dc7e3d 2319 else
2320 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2321
2322 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2323 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2324 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2325 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2326 ufshcd_prepare_utp_nop_upiu(lrbp);
2327 else
2328 ret = -EINVAL;
2329
2330 return ret;
2331}
2332
2333/**
2334 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2335 * for SCSI Purposes
8aa29f19
BVA		 2336 * @hba: per adapter instance
2337 * @lrbp: pointer to local reference block
300bb13f
JP		 2338 */
2339static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2340{
2341 u32 upiu_flags;
2342 int ret = 0;
2343
83dc7e3d 2344 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2345 (hba->ufs_version == UFSHCI_VERSION_11))
300bb13f 2346 lrbp->command_type = UTP_CMD_TYPE_SCSI;
83dc7e3d 2347 else
2348 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
300bb13f
JP		 2349
2350 if (likely(lrbp->cmd)) {
2351 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2352 lrbp->cmd->sc_data_direction);
2353 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2354 } else {
2355 ret = -EINVAL;
2356 }
5a0b0cb9
SRT		 2357
2358 return ret;
7a3e97b0
SY		 2359}
2360
2a8fa600
SJ		 2361/**
2362 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
8aa29f19 2363 * @upiu_wlun_id: UPIU W-LUN id
2a8fa600
SJ		 2364 *
2365 * Returns SCSI W-LUN id
2366 */
2367static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2368{
2369 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2370}
2371
7a3e97b0
SY		 2372/**
2373 * ufshcd_queuecommand - main entry point for SCSI requests
8aa29f19 2374 * @host: SCSI host pointer
7a3e97b0 2375 * @cmd: command from SCSI Midlayer
7a3e97b0
SY		 2376 *
2377 * Returns 0 for success, non-zero in case of failure
2378 */
2379static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2380{
2381 struct ufshcd_lrb *lrbp;
2382 struct ufs_hba *hba;
2383 unsigned long flags;
2384 int tag;
2385 int err = 0;
2386
2387 hba = shost_priv(host);
2388
2389 tag = cmd->request->tag;
14497328
YG		 2390 if (!ufshcd_valid_tag(hba, tag)) {
2391 dev_err(hba->dev,
2392 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2393 __func__, tag, cmd, cmd->request);
2394 BUG();
2395 }
7a3e97b0 2396
a3cd5ec5
SJ		 2397 if (!down_read_trylock(&hba->clk_scaling_lock))
2398 return SCSI_MLQUEUE_HOST_BUSY;
2399
3441da7d
SRT		 2400 spin_lock_irqsave(hba->host->host_lock, flags);
2401 switch (hba->ufshcd_state) {
2402 case UFSHCD_STATE_OPERATIONAL:
2403 break;
141f8165 2404 case UFSHCD_STATE_EH_SCHEDULED:
3441da7d 2405 case UFSHCD_STATE_RESET:
7a3e97b0 2406 err = SCSI_MLQUEUE_HOST_BUSY;
3441da7d
SRT		 2407 goto out_unlock;
2408 case UFSHCD_STATE_ERROR:
2409 set_host_byte(cmd, DID_ERROR);
2410 cmd->scsi_done(cmd);
2411 goto out_unlock;
2412 default:
2413 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2414 __func__, hba->ufshcd_state);
2415 set_host_byte(cmd, DID_BAD_TARGET);
2416 cmd->scsi_done(cmd);
2417 goto out_unlock;
7a3e97b0 2418 }
53c12d0e
YG		 2419
2420 /* if error handling is in progress, don't issue commands */
2421 if (ufshcd_eh_in_progress(hba)) {
2422 set_host_byte(cmd, DID_ERROR);
2423 cmd->scsi_done(cmd);
2424 goto out_unlock;
2425 }
3441da7d 2426 spin_unlock_irqrestore(hba->host->host_lock, flags);
7a3e97b0 2427
7fabb77b
GB		 2428 hba->req_abort_count = 0;
2429
1ab27c9c
ST		 2430 err = ufshcd_hold(hba, true);
2431 if (err) {
2432 err = SCSI_MLQUEUE_HOST_BUSY;
1ab27c9c
ST		 2433 goto out;
2434 }
2435 WARN_ON(hba->clk_gating.state != CLKS_ON);
2436
7a3e97b0
SY		 2437 lrbp = &hba->lrb[tag];
2438
5a0b0cb9 2439 WARN_ON(lrbp->cmd);
7a3e97b0 2440 lrbp->cmd = cmd;
09a5a24f 2441 lrbp->sense_bufflen = UFS_SENSE_SIZE;
7a3e97b0
SY		 2442 lrbp->sense_buffer = cmd->sense_buffer;
2443 lrbp->task_tag = tag;
0ce147d4 2444 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
b852190e 2445 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
e0b299e3 2446 lrbp->req_abort_skip = false;
7a3e97b0 2447
300bb13f
JP		 2448 ufshcd_comp_scsi_upiu(hba, lrbp);
2449
75b1cc4a 2450 err = ufshcd_map_sg(hba, lrbp);
5a0b0cb9
SRT		 2451 if (err) {
2452 lrbp->cmd = NULL;
7a3e97b0 2453 goto out;
5a0b0cb9 2454 }
ad1a1b9c
GB		 2455 /* Make sure descriptors are ready before ringing the doorbell */
2456 wmb();
7a3e97b0
SY		 2457
2458 /* issue command to the controller */
2459 spin_lock_irqsave(hba->host->host_lock, flags);
0e675efa 2460 ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
7a3e97b0 2461 ufshcd_send_command(hba, tag);
3441da7d 2462out_unlock:
7a3e97b0
SY		 2463 spin_unlock_irqrestore(hba->host->host_lock, flags);
2464out:
a3cd5ec5 2465 up_read(&hba->clk_scaling_lock);
7a3e97b0
SY		 2466 return err;
2467}
2468
5a0b0cb9
SRT		 2469static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2470 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2471{
2472 lrbp->cmd = NULL;
2473 lrbp->sense_bufflen = 0;
2474 lrbp->sense_buffer = NULL;
2475 lrbp->task_tag = tag;
2476 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
5a0b0cb9
SRT		 2477 lrbp->intr_cmd = true; /* No interrupt aggregation */
2478 hba->dev_cmd.type = cmd_type;
2479
300bb13f 2480 return ufshcd_comp_devman_upiu(hba, lrbp);
5a0b0cb9
SRT		 2481}
2482
2483static int
2484ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2485{
2486 int err = 0;
2487 unsigned long flags;
2488 u32 mask = 1 << tag;
2489
2490 /* clear outstanding transaction before retry */
2491 spin_lock_irqsave(hba->host->host_lock, flags);
2492 ufshcd_utrl_clear(hba, tag);
2493 spin_unlock_irqrestore(hba->host->host_lock, flags);
2494
2495 /*
2496 * wait for for h/w to clear corresponding bit in door-bell.
2497 * max. wait is 1 sec.
2498 */
2499 err = ufshcd_wait_for_register(hba,
2500 REG_UTP_TRANSFER_REQ_DOOR_BELL,
596585a2 2501 mask, ~mask, 1000, 1000, true);
5a0b0cb9
SRT		 2502
2503 return err;
2504}
2505
c6d4a831
DR		 2506static int
2507ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2508{
2509 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2510
2511 /* Get the UPIU response */
2512 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2513 UPIU_RSP_CODE_OFFSET;
2514 return query_res->response;
2515}
2516
5a0b0cb9
SRT		 2517/**
2518 * ufshcd_dev_cmd_completion() - handles device management command responses
2519 * @hba: per adapter instance
2520 * @lrbp: pointer to local reference block
2521 */
2522static int
2523ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2524{
2525 int resp;
2526 int err = 0;
2527
ff8e20c6 2528 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 2529 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2530
2531 switch (resp) {
2532 case UPIU_TRANSACTION_NOP_IN:
2533 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2534 err = -EINVAL;
2535 dev_err(hba->dev, "%s: unexpected response %x\n",
2536 __func__, resp);
2537 }
2538 break;
68078d5c 2539 case UPIU_TRANSACTION_QUERY_RSP:
c6d4a831
DR		 2540 err = ufshcd_check_query_response(hba, lrbp);
2541 if (!err)
2542 err = ufshcd_copy_query_response(hba, lrbp);
68078d5c 2543 break;
5a0b0cb9
SRT		 2544 case UPIU_TRANSACTION_REJECT_UPIU:
2545 /* TODO: handle Reject UPIU Response */
2546 err = -EPERM;
2547 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2548 __func__);
2549 break;
2550 default:
2551 err = -EINVAL;
2552 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2553 __func__, resp);
2554 break;
2555 }
2556
2557 return err;
2558}
2559
2560static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2561 struct ufshcd_lrb *lrbp, int max_timeout)
2562{
2563 int err = 0;
2564 unsigned long time_left;
2565 unsigned long flags;
2566
2567 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2568 msecs_to_jiffies(max_timeout));
2569
ad1a1b9c
GB		 2570 /* Make sure descriptors are ready before ringing the doorbell */
2571 wmb();
5a0b0cb9
SRT		 2572 spin_lock_irqsave(hba->host->host_lock, flags);
2573 hba->dev_cmd.complete = NULL;
2574 if (likely(time_left)) {
2575 err = ufshcd_get_tr_ocs(lrbp);
2576 if (!err)
2577 err = ufshcd_dev_cmd_completion(hba, lrbp);
2578 }
2579 spin_unlock_irqrestore(hba->host->host_lock, flags);
2580
2581 if (!time_left) {
2582 err = -ETIMEDOUT;
a48353f6
YG		 2583 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2584 __func__, lrbp->task_tag);
5a0b0cb9 2585 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
a48353f6 2586 /* successfully cleared the command, retry if needed */
5a0b0cb9 2587 err = -EAGAIN;
a48353f6
YG		 2588 /*
2589 * in case of an error, after clearing the doorbell,
2590 * we also need to clear the outstanding_request
2591 * field in hba
2592 */
2593 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
5a0b0cb9
SRT		 2594 }
2595
2596 return err;
2597}
2598
5a0b0cb9
SRT		 2599/**
2600 * ufshcd_exec_dev_cmd - API for sending device management requests
8aa29f19
BVA		 2601 * @hba: UFS hba
2602 * @cmd_type: specifies the type (NOP, Query...)
2603 * @timeout: time in seconds
5a0b0cb9 2604 *
68078d5c
DR		 2605 * NOTE: Since there is only one available tag for device management commands,
2606 * it is expected you hold the hba->dev_cmd.lock mutex.
5a0b0cb9
SRT		 2607 */
2608static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2609 enum dev_cmd_type cmd_type, int timeout)
2610{
7252a360
BVA		 2611 struct request_queue *q = hba->cmd_queue;
2612 struct request *req;
5a0b0cb9
SRT		 2613 struct ufshcd_lrb *lrbp;
2614 int err;
2615 int tag;
2616 struct completion wait;
2617 unsigned long flags;
2618
a3cd5ec5
SJ		 2619 down_read(&hba->clk_scaling_lock);
2620
5a0b0cb9
SRT		 2621 /*
2622 * Get free slot, sleep if slots are unavailable.
2623 * Even though we use wait_event() which sleeps indefinitely,
2624 * the maximum wait time is bounded by SCSI request timeout.
2625 */
7252a360
BVA		 2626 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
2627 if (IS_ERR(req))
2628 return PTR_ERR(req);
2629 tag = req->tag;
2630 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5a0b0cb9
SRT		 2631
2632 init_completion(&wait);
2633 lrbp = &hba->lrb[tag];
2634 WARN_ON(lrbp->cmd);
2635 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2636 if (unlikely(err))
2637 goto out_put_tag;
2638
2639 hba->dev_cmd.complete = &wait;
2640
6667e6d9 2641 ufshcd_add_query_upiu_trace(hba, tag, "query_send");
e3dfdc53
YG		 2642 /* Make sure descriptors are ready before ringing the doorbell */
2643 wmb();
5a0b0cb9 2644 spin_lock_irqsave(hba->host->host_lock, flags);
0e675efa 2645 ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
5a0b0cb9
SRT		 2646 ufshcd_send_command(hba, tag);
2647 spin_unlock_irqrestore(hba->host->host_lock, flags);
2648
2649 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2650
6667e6d9
OS		 2651 ufshcd_add_query_upiu_trace(hba, tag,
2652 err ? "query_complete_err" : "query_complete");
2653
5a0b0cb9 2654out_put_tag:
7252a360 2655 blk_put_request(req);
a3cd5ec5 2656 up_read(&hba->clk_scaling_lock);
5a0b0cb9
SRT		 2657 return err;
2658}
2659
d44a5f98
DR		 2660/**
2661 * ufshcd_init_query() - init the query response and request parameters
2662 * @hba: per-adapter instance
2663 * @request: address of the request pointer to be initialized
2664 * @response: address of the response pointer to be initialized
2665 * @opcode: operation to perform
2666 * @idn: flag idn to access
2667 * @index: LU number to access
2668 * @selector: query/flag/descriptor further identification
2669 */
2670static inline void ufshcd_init_query(struct ufs_hba *hba,
2671 struct ufs_query_req **request, struct ufs_query_res **response,
2672 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2673{
2674 *request = &hba->dev_cmd.query.request;
2675 *response = &hba->dev_cmd.query.response;
2676 memset(*request, 0, sizeof(struct ufs_query_req));
2677 memset(*response, 0, sizeof(struct ufs_query_res));
2678 (*request)->upiu_req.opcode = opcode;
2679 (*request)->upiu_req.idn = idn;
2680 (*request)->upiu_req.index = index;
2681 (*request)->upiu_req.selector = selector;
2682}
2683
dc3c8d3a
YG		 2684static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2685 enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
2686{
2687 int ret;
2688 int retries;
2689
2690 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2691 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
2692 if (ret)
2693 dev_dbg(hba->dev,
2694 "%s: failed with error %d, retries %d\n",
2695 __func__, ret, retries);
2696 else
2697 break;
2698 }
2699
2700 if (ret)
2701 dev_err(hba->dev,
2702 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2703 __func__, opcode, idn, ret, retries);
2704 return ret;
2705}
2706
68078d5c
DR		 2707/**
2708 * ufshcd_query_flag() - API function for sending flag query requests
8aa29f19
BVA		 2709 * @hba: per-adapter instance
2710 * @opcode: flag query to perform
2711 * @idn: flag idn to access
2712 * @flag_res: the flag value after the query request completes
68078d5c
DR		 2713 *
2714 * Returns 0 for success, non-zero in case of failure
2715 */
dc3c8d3a 2716int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
68078d5c
DR		 2717 enum flag_idn idn, bool *flag_res)
2718{
d44a5f98
DR		 2719 struct ufs_query_req *request = NULL;
2720 struct ufs_query_res *response = NULL;
2721 int err, index = 0, selector = 0;
e5ad406c 2722 int timeout = QUERY_REQ_TIMEOUT;
68078d5c
DR		 2723
2724 BUG_ON(!hba);
2725
1ab27c9c 2726 ufshcd_hold(hba, false);
68078d5c 2727 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2728 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2729 selector);
68078d5c
DR		 2730
2731 switch (opcode) {
2732 case UPIU_QUERY_OPCODE_SET_FLAG:
2733 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2734 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2735 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2736 break;
2737 case UPIU_QUERY_OPCODE_READ_FLAG:
2738 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2739 if (!flag_res) {
2740 /* No dummy reads */
2741 dev_err(hba->dev, "%s: Invalid argument for read request\n",
2742 __func__);
2743 err = -EINVAL;
2744 goto out_unlock;
2745 }
2746 break;
2747 default:
2748 dev_err(hba->dev,
2749 "%s: Expected query flag opcode but got = %d\n",
2750 __func__, opcode);
2751 err = -EINVAL;
2752 goto out_unlock;
2753 }
68078d5c 2754
e5ad406c 2755 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
68078d5c
DR		 2756
2757 if (err) {
2758 dev_err(hba->dev,
2759 "%s: Sending flag query for idn %d failed, err = %d\n",
2760 __func__, idn, err);
2761 goto out_unlock;
2762 }
2763
2764 if (flag_res)
e8c8e82a 2765 *flag_res = (be32_to_cpu(response->upiu_res.value) &
68078d5c
DR		 2766 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2767
2768out_unlock:
2769 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 2770 ufshcd_release(hba);
68078d5c
DR		 2771 return err;
2772}
2773
66ec6d59
SRT		 2774/**
2775 * ufshcd_query_attr - API function for sending attribute requests
8aa29f19
BVA		 2776 * @hba: per-adapter instance
2777 * @opcode: attribute opcode
2778 * @idn: attribute idn to access
2779 * @index: index field
2780 * @selector: selector field
2781 * @attr_val: the attribute value after the query request completes
66ec6d59
SRT		 2782 *
2783 * Returns 0 for success, non-zero in case of failure
2784*/
ec92b59c
SN		 2785int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2786 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
66ec6d59 2787{
d44a5f98
DR		 2788 struct ufs_query_req *request = NULL;
2789 struct ufs_query_res *response = NULL;
66ec6d59
SRT		 2790 int err;
2791
2792 BUG_ON(!hba);
2793
1ab27c9c 2794 ufshcd_hold(hba, false);
66ec6d59
SRT		 2795 if (!attr_val) {
2796 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2797 __func__, opcode);
2798 err = -EINVAL;
2799 goto out;
2800 }
2801
2802 mutex_lock(&hba->dev_cmd.lock);
d44a5f98
DR		 2803 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2804 selector);
66ec6d59
SRT		 2805
2806 switch (opcode) {
2807 case UPIU_QUERY_OPCODE_WRITE_ATTR:
2808 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
e8c8e82a 2809 request->upiu_req.value = cpu_to_be32(*attr_val);
66ec6d59
SRT		 2810 break;
2811 case UPIU_QUERY_OPCODE_READ_ATTR:
2812 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2813 break;
2814 default:
2815 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2816 __func__, opcode);
2817 err = -EINVAL;
2818 goto out_unlock;
2819 }
2820
d44a5f98 2821 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
66ec6d59
SRT		 2822
2823 if (err) {
4b761b58
YG		 2824 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2825 __func__, opcode, idn, index, err);
66ec6d59
SRT		 2826 goto out_unlock;
2827 }
2828
e8c8e82a 2829 *attr_val = be32_to_cpu(response->upiu_res.value);
66ec6d59
SRT		 2830
2831out_unlock:
2832 mutex_unlock(&hba->dev_cmd.lock);
2833out:
1ab27c9c 2834 ufshcd_release(hba);
66ec6d59
SRT		 2835 return err;
2836}
2837
5e86ae44
YG		 2838/**
2839 * ufshcd_query_attr_retry() - API function for sending query
2840 * attribute with retries
2841 * @hba: per-adapter instance
2842 * @opcode: attribute opcode
2843 * @idn: attribute idn to access
2844 * @index: index field
2845 * @selector: selector field
2846 * @attr_val: the attribute value after the query request
2847 * completes
2848 *
2849 * Returns 0 for success, non-zero in case of failure
2850*/
2851static int ufshcd_query_attr_retry(struct ufs_hba *hba,
2852 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
2853 u32 *attr_val)
2854{
2855 int ret = 0;
2856 u32 retries;
2857
2858 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2859 ret = ufshcd_query_attr(hba, opcode, idn, index,
2860 selector, attr_val);
2861 if (ret)
2862 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
2863 __func__, ret, retries);
2864 else
2865 break;
2866 }
2867
2868 if (ret)
2869 dev_err(hba->dev,
2870 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
2871 __func__, idn, ret, QUERY_REQ_RETRIES);
2872 return ret;
2873}
2874
a70e91b8 2875static int __ufshcd_query_descriptor(struct ufs_hba *hba,
d44a5f98
DR		 2876 enum query_opcode opcode, enum desc_idn idn, u8 index,
2877 u8 selector, u8 *desc_buf, int *buf_len)
2878{
2879 struct ufs_query_req *request = NULL;
2880 struct ufs_query_res *response = NULL;
2881 int err;
2882
2883 BUG_ON(!hba);
2884
1ab27c9c 2885 ufshcd_hold(hba, false);
d44a5f98
DR		 2886 if (!desc_buf) {
2887 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
2888 __func__, opcode);
2889 err = -EINVAL;
2890 goto out;
2891 }
2892
a4b0e8a4 2893 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
d44a5f98
DR		 2894 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
2895 __func__, *buf_len);
2896 err = -EINVAL;
2897 goto out;
2898 }
2899
2900 mutex_lock(&hba->dev_cmd.lock);
2901 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2902 selector);
2903 hba->dev_cmd.query.descriptor = desc_buf;
ea2aab24 2904 request->upiu_req.length = cpu_to_be16(*buf_len);
d44a5f98
DR		 2905
2906 switch (opcode) {
2907 case UPIU_QUERY_OPCODE_WRITE_DESC:
2908 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2909 break;
2910 case UPIU_QUERY_OPCODE_READ_DESC:
2911 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2912 break;
2913 default:
2914 dev_err(hba->dev,
2915 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
2916 __func__, opcode);
2917 err = -EINVAL;
2918 goto out_unlock;
2919 }
2920
2921 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2922
2923 if (err) {
4b761b58
YG		 2924 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2925 __func__, opcode, idn, index, err);
d44a5f98
DR		 2926 goto out_unlock;
2927 }
2928
ea2aab24 2929 *buf_len = be16_to_cpu(response->upiu_res.length);
d44a5f98
DR		 2930
2931out_unlock:
cfcbae38 2932 hba->dev_cmd.query.descriptor = NULL;
d44a5f98
DR		 2933 mutex_unlock(&hba->dev_cmd.lock);
2934out:
1ab27c9c 2935 ufshcd_release(hba);
d44a5f98
DR		 2936 return err;
2937}
2938
a70e91b8 2939/**
8aa29f19
BVA		 2940 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
2941 * @hba: per-adapter instance
2942 * @opcode: attribute opcode
2943 * @idn: attribute idn to access
2944 * @index: index field
2945 * @selector: selector field
2946 * @desc_buf: the buffer that contains the descriptor
2947 * @buf_len: length parameter passed to the device
a70e91b8
YG		 2948 *
2949 * Returns 0 for success, non-zero in case of failure.
2950 * The buf_len parameter will contain, on return, the length parameter
2951 * received on the response.
2952 */
2238d31c
SN		 2953int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
2954 enum query_opcode opcode,
2955 enum desc_idn idn, u8 index,
2956 u8 selector,
2957 u8 *desc_buf, int *buf_len)
a70e91b8
YG		 2958{
2959 int err;
2960 int retries;
2961
2962 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2963 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
2964 selector, desc_buf, buf_len);
2965 if (!err || err == -EINVAL)
2966 break;
2967 }
2968
2969 return err;
2970}
a70e91b8 2971
a4b0e8a4
PM		 2972/**
2973 * ufshcd_read_desc_length - read the specified descriptor length from header
2974 * @hba: Pointer to adapter instance
2975 * @desc_id: descriptor idn value
2976 * @desc_index: descriptor index
2977 * @desc_length: pointer to variable to read the length of descriptor
2978 *
2979 * Return 0 in case of success, non-zero otherwise
2980 */
2981static int ufshcd_read_desc_length(struct ufs_hba *hba,
2982 enum desc_idn desc_id,
2983 int desc_index,
2984 int *desc_length)
2985{
2986 int ret;
2987 u8 header[QUERY_DESC_HDR_SIZE];
2988 int header_len = QUERY_DESC_HDR_SIZE;
2989
2990 if (desc_id >= QUERY_DESC_IDN_MAX)
2991 return -EINVAL;
2992
2993 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
2994 desc_id, desc_index, 0, header,
2995 &header_len);
2996
2997 if (ret) {
2998 dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
2999 __func__, desc_id);
3000 return ret;
3001 } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
3002 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
3003 __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
3004 desc_id);
3005 ret = -EINVAL;
3006 }
3007
3008 *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
3009 return ret;
3010
3011}
3012
3013/**
3014 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3015 * @hba: Pointer to adapter instance
3016 * @desc_id: descriptor idn value
3017 * @desc_len: mapped desc length (out)
3018 *
3019 * Return 0 in case of success, non-zero otherwise
3020 */
3021int ufshcd_map_desc_id_to_length(struct ufs_hba *hba,
3022 enum desc_idn desc_id, int *desc_len)
3023{
3024 switch (desc_id) {
3025 case QUERY_DESC_IDN_DEVICE:
3026 *desc_len = hba->desc_size.dev_desc;
3027 break;
3028 case QUERY_DESC_IDN_POWER:
3029 *desc_len = hba->desc_size.pwr_desc;
3030 break;
3031 case QUERY_DESC_IDN_GEOMETRY:
3032 *desc_len = hba->desc_size.geom_desc;
3033 break;
3034 case QUERY_DESC_IDN_CONFIGURATION:
3035 *desc_len = hba->desc_size.conf_desc;
3036 break;
3037 case QUERY_DESC_IDN_UNIT:
3038 *desc_len = hba->desc_size.unit_desc;
3039 break;
3040 case QUERY_DESC_IDN_INTERCONNECT:
3041 *desc_len = hba->desc_size.interc_desc;
3042 break;
3043 case QUERY_DESC_IDN_STRING:
3044 *desc_len = QUERY_DESC_MAX_SIZE;
3045 break;
c648c2d2
SN		 3046 case QUERY_DESC_IDN_HEALTH:
3047 *desc_len = hba->desc_size.hlth_desc;
3048 break;
a4b0e8a4
PM		 3049 case QUERY_DESC_IDN_RFU_0:
3050 case QUERY_DESC_IDN_RFU_1:
3051 *desc_len = 0;
3052 break;
3053 default:
3054 *desc_len = 0;
3055 return -EINVAL;
3056 }
3057 return 0;
3058}
3059EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3060
da461cec
SJ		 3061/**
3062 * ufshcd_read_desc_param - read the specified descriptor parameter
3063 * @hba: Pointer to adapter instance
3064 * @desc_id: descriptor idn value
3065 * @desc_index: descriptor index
3066 * @param_offset: offset of the parameter to read
3067 * @param_read_buf: pointer to buffer where parameter would be read
3068 * @param_size: sizeof(param_read_buf)
3069 *
3070 * Return 0 in case of success, non-zero otherwise
3071 */
45bced87
SN		 3072int ufshcd_read_desc_param(struct ufs_hba *hba,
3073 enum desc_idn desc_id,
3074 int desc_index,
3075 u8 param_offset,
3076 u8 *param_read_buf,
3077 u8 param_size)
da461cec
SJ		 3078{
3079 int ret;
3080 u8 *desc_buf;
a4b0e8a4 3081 int buff_len;
da461cec
SJ		 3082 bool is_kmalloc = true;
3083
a4b0e8a4
PM		 3084 /* Safety check */
3085 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
da461cec
SJ		 3086 return -EINVAL;
3087
a4b0e8a4
PM		 3088 /* Get the max length of descriptor from structure filled up at probe
3089 * time.
3090 */
3091 ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
da461cec 3092
a4b0e8a4
PM		 3093 /* Sanity checks */
3094 if (ret || !buff_len) {
3095 dev_err(hba->dev, "%s: Failed to get full descriptor length",
3096 __func__);
3097 return ret;
3098 }
3099
3100 /* Check whether we need temp memory */
3101 if (param_offset != 0 || param_size < buff_len) {
da461cec
SJ		 3102 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3103 if (!desc_buf)
3104 return -ENOMEM;
a4b0e8a4
PM		 3105 } else {
3106 desc_buf = param_read_buf;
3107 is_kmalloc = false;
da461cec
SJ		 3108 }
3109
a4b0e8a4 3110 /* Request for full descriptor */
a70e91b8 3111 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
a4b0e8a4
PM		 3112 desc_id, desc_index, 0,
3113 desc_buf, &buff_len);
da461cec 3114
bde44bb6
SJ		 3115 if (ret) {
3116 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3117 __func__, desc_id, desc_index, param_offset, ret);
da461cec
SJ		 3118 goto out;
3119 }
3120
bde44bb6
SJ		 3121 /* Sanity check */
3122 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3123 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3124 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3125 ret = -EINVAL;
3126 goto out;
3127 }
3128
a4b0e8a4
PM		 3129 /* Check wherher we will not copy more data, than available */
3130 if (is_kmalloc && param_size > buff_len)
3131 param_size = buff_len;
bde44bb6 3132
da461cec
SJ		 3133 if (is_kmalloc)
3134 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3135out:
3136 if (is_kmalloc)
3137 kfree(desc_buf);
3138 return ret;
3139}
3140
3141static inline int ufshcd_read_desc(struct ufs_hba *hba,
3142 enum desc_idn desc_id,
3143 int desc_index,
4b828fe1 3144 void *buf,
da461cec
SJ		 3145 u32 size)
3146{
3147 return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
3148}
3149
3150static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
3151 u8 *buf,
3152 u32 size)
3153{
dbd34a61 3154 return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
da461cec
SJ		 3155}
3156
8209b6d5 3157static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
b573d484
YG		 3158{
3159 return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
3160}
b573d484 3161
4b828fe1
TW		 3162/**
3163 * struct uc_string_id - unicode string
3164 *
3165 * @len: size of this descriptor inclusive
3166 * @type: descriptor type
3167 * @uc: unicode string character
3168 */
3169struct uc_string_id {
3170 u8 len;
3171 u8 type;
3172 wchar_t uc[0];
3173} __packed;
3174
3175/* replace non-printable or non-ASCII characters with spaces */
3176static inline char ufshcd_remove_non_printable(u8 ch)
3177{
3178 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3179}
3180
b573d484
YG		 3181/**
3182 * ufshcd_read_string_desc - read string descriptor
3183 * @hba: pointer to adapter instance
3184 * @desc_index: descriptor index
4b828fe1
TW		 3185 * @buf: pointer to buffer where descriptor would be read,
3186 * the caller should free the memory.
b573d484 3187 * @ascii: if true convert from unicode to ascii characters
4b828fe1 3188 * null terminated string.
b573d484 3189 *
4b828fe1
TW		 3190 * Return:
3191 * * string size on success.
3192 * * -ENOMEM: on allocation failure
3193 * * -EINVAL: on a wrong parameter
b573d484 3194 */
4b828fe1
TW		 3195int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3196 u8 **buf, bool ascii)
b573d484 3197{
4b828fe1
TW		 3198 struct uc_string_id *uc_str;
3199 u8 *str;
3200 int ret;
b573d484 3201
4b828fe1
TW		 3202 if (!buf)
3203 return -EINVAL;
b573d484 3204
4b828fe1
TW		 3205 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3206 if (!uc_str)
3207 return -ENOMEM;
b573d484 3208
4b828fe1
TW		 3209 ret = ufshcd_read_desc(hba, QUERY_DESC_IDN_STRING,
3210 desc_index, uc_str,
3211 QUERY_DESC_MAX_SIZE);
3212 if (ret < 0) {
3213 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3214 QUERY_REQ_RETRIES, ret);
3215 str = NULL;
3216 goto out;
3217 }
3218
3219 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3220 dev_dbg(hba->dev, "String Desc is of zero length\n");
3221 str = NULL;
3222 ret = 0;
b573d484
YG		 3223 goto out;
3224 }
3225
3226 if (ascii) {
4b828fe1 3227 ssize_t ascii_len;
b573d484 3228 int i;
b573d484 3229 /* remove header and divide by 2 to move from UTF16 to UTF8 */
4b828fe1
TW		 3230 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3231 str = kzalloc(ascii_len, GFP_KERNEL);
3232 if (!str) {
3233 ret = -ENOMEM;
fcbefc3b 3234 goto out;
b573d484
YG		 3235 }
3236
3237 /*
3238 * the descriptor contains string in UTF16 format
3239 * we need to convert to utf-8 so it can be displayed
3240 */
4b828fe1
TW		 3241 ret = utf16s_to_utf8s(uc_str->uc,
3242 uc_str->len - QUERY_DESC_HDR_SIZE,
3243 UTF16_BIG_ENDIAN, str, ascii_len);
b573d484
YG		 3244
3245 /* replace non-printable or non-ASCII characters with spaces */
4b828fe1
TW		 3246 for (i = 0; i < ret; i++)
3247 str[i] = ufshcd_remove_non_printable(str[i]);
b573d484 3248
4b828fe1
TW		 3249 str[ret++] = '\0';
3250
3251 } else {
5f57704d 3252 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
4b828fe1
TW		 3253 if (!str) {
3254 ret = -ENOMEM;
3255 goto out;
3256 }
4b828fe1 3257 ret = uc_str->len;
b573d484
YG		 3258 }
3259out:
4b828fe1
TW		 3260 *buf = str;
3261 kfree(uc_str);
3262 return ret;
b573d484 3263}
b573d484 3264
da461cec
SJ		 3265/**
3266 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3267 * @hba: Pointer to adapter instance
3268 * @lun: lun id
3269 * @param_offset: offset of the parameter to read
3270 * @param_read_buf: pointer to buffer where parameter would be read
3271 * @param_size: sizeof(param_read_buf)
3272 *
3273 * Return 0 in case of success, non-zero otherwise
3274 */
3275static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3276 int lun,
3277 enum unit_desc_param param_offset,
3278 u8 *param_read_buf,
3279 u32 param_size)
3280{
3281 /*
3282 * Unit descriptors are only available for general purpose LUs (LUN id
3283 * from 0 to 7) and RPMB Well known LU.
3284 */
d829fc8a 3285 if (!ufs_is_valid_unit_desc_lun(lun))
da461cec
SJ		 3286 return -EOPNOTSUPP;
3287
3288 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3289 param_offset, param_read_buf, param_size);
3290}
3291
7a3e97b0
SY		 3292/**
3293 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3294 * @hba: per adapter instance
3295 *
3296 * 1. Allocate DMA memory for Command Descriptor array
3297 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3298 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3299 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3300 * (UTMRDL)
3301 * 4. Allocate memory for local reference block(lrb).
3302 *
3303 * Returns 0 for success, non-zero in case of failure
3304 */
3305static int ufshcd_memory_alloc(struct ufs_hba *hba)
3306{
3307 size_t utmrdl_size, utrdl_size, ucdl_size;
3308
3309 /* Allocate memory for UTP command descriptors */
3310 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2953f850
SJ		 3311 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3312 ucdl_size,
3313 &hba->ucdl_dma_addr,
3314 GFP_KERNEL);
7a3e97b0
SY		 3315
3316 /*
3317 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3318 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3319 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3320 * be aligned to 128 bytes as well
3321 */
3322 if (!hba->ucdl_base_addr ||
3323 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3324 dev_err(hba->dev,
7a3e97b0
SY		 3325 "Command Descriptor Memory allocation failed\n");
3326 goto out;
3327 }
3328
3329 /*
3330 * Allocate memory for UTP Transfer descriptors
3331 * UFSHCI requires 1024 byte alignment of UTRD
3332 */
3333 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2953f850
SJ		 3334 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3335 utrdl_size,
3336 &hba->utrdl_dma_addr,
3337 GFP_KERNEL);
7a3e97b0
SY		 3338 if (!hba->utrdl_base_addr ||
3339 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3340 dev_err(hba->dev,
7a3e97b0
SY		 3341 "Transfer Descriptor Memory allocation failed\n");
3342 goto out;
3343 }
3344
3345 /*
3346 * Allocate memory for UTP Task Management descriptors
3347 * UFSHCI requires 1024 byte alignment of UTMRD
3348 */
3349 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2953f850
SJ		 3350 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3351 utmrdl_size,
3352 &hba->utmrdl_dma_addr,
3353 GFP_KERNEL);
7a3e97b0
SY		 3354 if (!hba->utmrdl_base_addr ||
3355 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3b1d0580 3356 dev_err(hba->dev,
7a3e97b0
SY		 3357 "Task Management Descriptor Memory allocation failed\n");
3358 goto out;
3359 }
3360
3361 /* Allocate memory for local reference block */
a86854d0
KC		 3362 hba->lrb = devm_kcalloc(hba->dev,
3363 hba->nutrs, sizeof(struct ufshcd_lrb),
2953f850 3364 GFP_KERNEL);
7a3e97b0 3365 if (!hba->lrb) {
3b1d0580 3366 dev_err(hba->dev, "LRB Memory allocation failed\n");
7a3e97b0
SY		 3367 goto out;
3368 }
3369 return 0;
3370out:
7a3e97b0
SY		 3371 return -ENOMEM;
3372}
3373
3374/**
3375 * ufshcd_host_memory_configure - configure local reference block with
3376 * memory offsets
3377 * @hba: per adapter instance
3378 *
3379 * Configure Host memory space
3380 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3381 * address.
3382 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3383 * and PRDT offset.
3384 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3385 * into local reference block.
3386 */
3387static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3388{
3389 struct utp_transfer_cmd_desc *cmd_descp;
3390 struct utp_transfer_req_desc *utrdlp;
3391 dma_addr_t cmd_desc_dma_addr;
3392 dma_addr_t cmd_desc_element_addr;
3393 u16 response_offset;
3394 u16 prdt_offset;
3395 int cmd_desc_size;
3396 int i;
3397
3398 utrdlp = hba->utrdl_base_addr;
3399 cmd_descp = hba->ucdl_base_addr;
3400
3401 response_offset =
3402 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3403 prdt_offset =
3404 offsetof(struct utp_transfer_cmd_desc, prd_table);
3405
3406 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3407 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3408
3409 for (i = 0; i < hba->nutrs; i++) {
3410 /* Configure UTRD with command descriptor base address */
3411 cmd_desc_element_addr =
3412 (cmd_desc_dma_addr + (cmd_desc_size * i));
3413 utrdlp[i].command_desc_base_addr_lo =
3414 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3415 utrdlp[i].command_desc_base_addr_hi =
3416 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3417
3418 /* Response upiu and prdt offset should be in double words */
75b1cc4a
KK		 3419 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3420 utrdlp[i].response_upiu_offset =
3421 cpu_to_le16(response_offset);
3422 utrdlp[i].prd_table_offset =
3423 cpu_to_le16(prdt_offset);
3424 utrdlp[i].response_upiu_length =
3425 cpu_to_le16(ALIGNED_UPIU_SIZE);
3426 } else {
3427 utrdlp[i].response_upiu_offset =
7a3e97b0 3428 cpu_to_le16((response_offset >> 2));
75b1cc4a 3429 utrdlp[i].prd_table_offset =
7a3e97b0 3430 cpu_to_le16((prdt_offset >> 2));
75b1cc4a 3431 utrdlp[i].response_upiu_length =
3ca316c5 3432 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
75b1cc4a 3433 }
7a3e97b0
SY		 3434
3435 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
ff8e20c6
DR		 3436 hba->lrb[i].utrd_dma_addr = hba->utrdl_dma_addr +
3437 (i * sizeof(struct utp_transfer_req_desc));
5a0b0cb9
SRT		 3438 hba->lrb[i].ucd_req_ptr =
3439 (struct utp_upiu_req *)(cmd_descp + i);
ff8e20c6 3440 hba->lrb[i].ucd_req_dma_addr = cmd_desc_element_addr;
7a3e97b0
SY		 3441 hba->lrb[i].ucd_rsp_ptr =
3442 (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
ff8e20c6
DR		 3443 hba->lrb[i].ucd_rsp_dma_addr = cmd_desc_element_addr +
3444 response_offset;
7a3e97b0
SY		 3445 hba->lrb[i].ucd_prdt_ptr =
3446 (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
ff8e20c6
DR		 3447 hba->lrb[i].ucd_prdt_dma_addr = cmd_desc_element_addr +
3448 prdt_offset;
7a3e97b0
SY		 3449 }
3450}
3451
3452/**
3453 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3454 * @hba: per adapter instance
3455 *
3456 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3457 * in order to initialize the Unipro link startup procedure.
3458 * Once the Unipro links are up, the device connected to the controller
3459 * is detected.
3460 *
3461 * Returns 0 on success, non-zero value on failure
3462 */
3463static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3464{
6ccf44fe
SJ		 3465 struct uic_command uic_cmd = {0};
3466 int ret;
7a3e97b0 3467
6ccf44fe 3468 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
7a3e97b0 3469
6ccf44fe
SJ		 3470 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3471 if (ret)
ff8e20c6 3472 dev_dbg(hba->dev,
6ccf44fe
SJ		 3473 "dme-link-startup: error code %d\n", ret);
3474 return ret;
7a3e97b0 3475}
4404c5de
AA		 3476/**
3477 * ufshcd_dme_reset - UIC command for DME_RESET
3478 * @hba: per adapter instance
3479 *
3480 * DME_RESET command is issued in order to reset UniPro stack.
3481 * This function now deal with cold reset.
3482 *
3483 * Returns 0 on success, non-zero value on failure
3484 */
3485static int ufshcd_dme_reset(struct ufs_hba *hba)
3486{
3487 struct uic_command uic_cmd = {0};
3488 int ret;
3489
3490 uic_cmd.command = UIC_CMD_DME_RESET;
3491
3492 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3493 if (ret)
3494 dev_err(hba->dev,
3495 "dme-reset: error code %d\n", ret);
3496
3497 return ret;
3498}
3499
3500/**
3501 * ufshcd_dme_enable - UIC command for DME_ENABLE
3502 * @hba: per adapter instance
3503 *
3504 * DME_ENABLE command is issued in order to enable UniPro stack.
3505 *
3506 * Returns 0 on success, non-zero value on failure
3507 */
3508static int ufshcd_dme_enable(struct ufs_hba *hba)
3509{
3510 struct uic_command uic_cmd = {0};
3511 int ret;
3512
3513 uic_cmd.command = UIC_CMD_DME_ENABLE;
3514
3515 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3516 if (ret)
3517 dev_err(hba->dev,
3518 "dme-reset: error code %d\n", ret);
3519
3520 return ret;
3521}
7a3e97b0 3522
cad2e03d
YG		 3523static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3524{
3525 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3526 unsigned long min_sleep_time_us;
3527
3528 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3529 return;
3530
3531 /*
3532 * last_dme_cmd_tstamp will be 0 only for 1st call to
3533 * this function
3534 */
3535 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3536 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3537 } else {
3538 unsigned long delta =
3539 (unsigned long) ktime_to_us(
3540 ktime_sub(ktime_get(),
3541 hba->last_dme_cmd_tstamp));
3542
3543 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3544 min_sleep_time_us =
3545 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3546 else
3547 return; /* no more delay required */
3548 }
3549
3550 /* allow sleep for extra 50us if needed */
3551 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3552}
3553
12b4fdb4
SJ		 3554/**
3555 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3556 * @hba: per adapter instance
3557 * @attr_sel: uic command argument1
3558 * @attr_set: attribute set type as uic command argument2
3559 * @mib_val: setting value as uic command argument3
3560 * @peer: indicate whether peer or local
3561 *
3562 * Returns 0 on success, non-zero value on failure
3563 */
3564int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3565 u8 attr_set, u32 mib_val, u8 peer)
3566{
3567 struct uic_command uic_cmd = {0};
3568 static const char *const action[] = {
3569 "dme-set",
3570 "dme-peer-set"
3571 };
3572 const char *set = action[!!peer];
3573 int ret;
64238fbd 3574 int retries = UFS_UIC_COMMAND_RETRIES;
12b4fdb4
SJ		 3575
3576 uic_cmd.command = peer ?
3577 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3578 uic_cmd.argument1 = attr_sel;
3579 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3580 uic_cmd.argument3 = mib_val;
3581
64238fbd
YG		 3582 do {
3583 /* for peer attributes we retry upon failure */
3584 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3585 if (ret)
3586 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3587 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3588 } while (ret && peer && --retries);
3589
f37e9f8c 3590 if (ret)
64238fbd 3591 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
f37e9f8c
YG		 3592 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3593 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4
SJ		 3594
3595 return ret;
3596}
3597EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3598
3599/**
3600 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3601 * @hba: per adapter instance
3602 * @attr_sel: uic command argument1
3603 * @mib_val: the value of the attribute as returned by the UIC command
3604 * @peer: indicate whether peer or local
3605 *
3606 * Returns 0 on success, non-zero value on failure
3607 */
3608int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3609 u32 *mib_val, u8 peer)
3610{
3611 struct uic_command uic_cmd = {0};
3612 static const char *const action[] = {
3613 "dme-get",
3614 "dme-peer-get"
3615 };
3616 const char *get = action[!!peer];
3617 int ret;
64238fbd 3618 int retries = UFS_UIC_COMMAND_RETRIES;
874237f7
YG		 3619 struct ufs_pa_layer_attr orig_pwr_info;
3620 struct ufs_pa_layer_attr temp_pwr_info;
3621 bool pwr_mode_change = false;
3622
3623 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3624 orig_pwr_info = hba->pwr_info;
3625 temp_pwr_info = orig_pwr_info;
3626
3627 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3628 orig_pwr_info.pwr_rx == FAST_MODE) {
3629 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3630 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3631 pwr_mode_change = true;
3632 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3633 orig_pwr_info.pwr_rx == SLOW_MODE) {
3634 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3635 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3636 pwr_mode_change = true;
3637 }
3638 if (pwr_mode_change) {
3639 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3640 if (ret)
3641 goto out;
3642 }
3643 }
12b4fdb4
SJ		 3644
3645 uic_cmd.command = peer ?
3646 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3647 uic_cmd.argument1 = attr_sel;
3648
64238fbd
YG		 3649 do {
3650 /* for peer attributes we retry upon failure */
3651 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3652 if (ret)
3653 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3654 get, UIC_GET_ATTR_ID(attr_sel), ret);
3655 } while (ret && peer && --retries);
3656
f37e9f8c 3657 if (ret)
64238fbd 3658 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
f37e9f8c
YG		 3659 get, UIC_GET_ATTR_ID(attr_sel),
3660 UFS_UIC_COMMAND_RETRIES - retries);
12b4fdb4 3661
64238fbd 3662 if (mib_val && !ret)
12b4fdb4 3663 *mib_val = uic_cmd.argument3;
874237f7
YG		 3664
3665 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3666 && pwr_mode_change)
3667 ufshcd_change_power_mode(hba, &orig_pwr_info);
12b4fdb4
SJ		 3668out:
3669 return ret;
3670}
3671EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3672
53b3d9c3 3673/**
57d104c1
SJ		 3674 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3675 * state) and waits for it to take effect.
3676 *
53b3d9c3 3677 * @hba: per adapter instance
57d104c1
SJ		 3678 * @cmd: UIC command to execute
3679 *
3680 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3681 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3682 * and device UniPro link and hence it's final completion would be indicated by
3683 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3684 * addition to normal UIC command completion Status (UCCS). This function only
3685 * returns after the relevant status bits indicate the completion.
53b3d9c3
SJ		 3686 *
3687 * Returns 0 on success, non-zero value on failure
3688 */
57d104c1 3689static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
53b3d9c3 3690{
57d104c1 3691 struct completion uic_async_done;
53b3d9c3
SJ		 3692 unsigned long flags;
3693 u8 status;
3694 int ret;
d75f7fe4 3695 bool reenable_intr = false;
53b3d9c3 3696
53b3d9c3 3697 mutex_lock(&hba->uic_cmd_mutex);
57d104c1 3698 init_completion(&uic_async_done);
cad2e03d 3699 ufshcd_add_delay_before_dme_cmd(hba);
53b3d9c3
SJ		 3700
3701 spin_lock_irqsave(hba->host->host_lock, flags);
57d104c1 3702 hba->uic_async_done = &uic_async_done;
d75f7fe4
YG		 3703 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3704 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3705 /*
3706 * Make sure UIC command completion interrupt is disabled before
3707 * issuing UIC command.
3708 */
3709 wmb();
3710 reenable_intr = true;
57d104c1 3711 }
d75f7fe4
YG		 3712 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3713 spin_unlock_irqrestore(hba->host->host_lock, flags);
57d104c1
SJ		 3714 if (ret) {
3715 dev_err(hba->dev,
3716 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3717 cmd->command, cmd->argument3, ret);
53b3d9c3
SJ		 3718 goto out;
3719 }
3720
57d104c1 3721 if (!wait_for_completion_timeout(hba->uic_async_done,
53b3d9c3
SJ		 3722 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3723 dev_err(hba->dev,
57d104c1
SJ		 3724 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3725 cmd->command, cmd->argument3);
53b3d9c3
SJ		 3726 ret = -ETIMEDOUT;
3727 goto out;
3728 }
3729
3730 status = ufshcd_get_upmcrs(hba);
3731 if (status != PWR_LOCAL) {
3732 dev_err(hba->dev,
479da360 3733 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
57d104c1 3734 cmd->command, status);
53b3d9c3
SJ		 3735 ret = (status != PWR_OK) ? status : -1;
3736 }
3737out:
7942f7b5
VG		 3738 if (ret) {
3739 ufshcd_print_host_state(hba);
3740 ufshcd_print_pwr_info(hba);
3741 ufshcd_print_host_regs(hba);
3742 }
3743
53b3d9c3 3744 spin_lock_irqsave(hba->host->host_lock, flags);
d75f7fe4 3745 hba->active_uic_cmd = NULL;
57d104c1 3746 hba->uic_async_done = NULL;
d75f7fe4
YG		 3747 if (reenable_intr)
3748 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
53b3d9c3
SJ		 3749 spin_unlock_irqrestore(hba->host->host_lock, flags);
3750 mutex_unlock(&hba->uic_cmd_mutex);
1ab27c9c 3751
53b3d9c3
SJ		 3752 return ret;
3753}
3754
57d104c1
SJ		 3755/**
3756 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3757 * using DME_SET primitives.
3758 * @hba: per adapter instance
3759 * @mode: powr mode value
3760 *
3761 * Returns 0 on success, non-zero value on failure
3762 */
3763static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
3764{
3765 struct uic_command uic_cmd = {0};
1ab27c9c 3766 int ret;
57d104c1 3767
c3a2f9ee
YG		 3768 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
3769 ret = ufshcd_dme_set(hba,
3770 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
3771 if (ret) {
3772 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
3773 __func__, ret);
3774 goto out;
3775 }
3776 }
3777
57d104c1
SJ		 3778 uic_cmd.command = UIC_CMD_DME_SET;
3779 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
3780 uic_cmd.argument3 = mode;
1ab27c9c
ST		 3781 ufshcd_hold(hba, false);
3782 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3783 ufshcd_release(hba);
57d104c1 3784
c3a2f9ee 3785out:
1ab27c9c 3786 return ret;
57d104c1
SJ		 3787}
3788
53c12d0e
YG		 3789static int ufshcd_link_recovery(struct ufs_hba *hba)
3790{
3791 int ret;
3792 unsigned long flags;
3793
3794 spin_lock_irqsave(hba->host->host_lock, flags);
3795 hba->ufshcd_state = UFSHCD_STATE_RESET;
3796 ufshcd_set_eh_in_progress(hba);
3797 spin_unlock_irqrestore(hba->host->host_lock, flags);
3798
ebdd1dfd
CG		 3799 /* Reset the attached device */
3800 ufshcd_vops_device_reset(hba);
3801
53c12d0e
YG		 3802 ret = ufshcd_host_reset_and_restore(hba);
3803
3804 spin_lock_irqsave(hba->host->host_lock, flags);
3805 if (ret)
3806 hba->ufshcd_state = UFSHCD_STATE_ERROR;
3807 ufshcd_clear_eh_in_progress(hba);
3808 spin_unlock_irqrestore(hba->host->host_lock, flags);
3809
3810 if (ret)
3811 dev_err(hba->dev, "%s: link recovery failed, err %d",
3812 __func__, ret);
3813
3814 return ret;
3815}
3816
87d0b4a6 3817static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
57d104c1 3818{
87d0b4a6 3819 int ret;
57d104c1 3820 struct uic_command uic_cmd = {0};
911a0771 3821 ktime_t start = ktime_get();
57d104c1 3822
ee32c909
KK		 3823 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
3824
57d104c1 3825 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
87d0b4a6 3826 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771
SJ		 3827 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
3828 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
87d0b4a6 3829
53c12d0e 3830 if (ret) {
6d303e4b
SJ		 3831 int err;
3832
87d0b4a6
YG		 3833 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
3834 __func__, ret);
3835
53c12d0e 3836 /*
6d303e4b
SJ		 3837 * If link recovery fails then return error code returned from
3838 * ufshcd_link_recovery().
3839 * If link recovery succeeds then return -EAGAIN to attempt
3840 * hibern8 enter retry again.
53c12d0e 3841 */
6d303e4b
SJ		 3842 err = ufshcd_link_recovery(hba);
3843 if (err) {
3844 dev_err(hba->dev, "%s: link recovery failed", __func__);
3845 ret = err;
3846 } else {
3847 ret = -EAGAIN;
3848 }
ee32c909
KK		 3849 } else
3850 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
3851 POST_CHANGE);
53c12d0e 3852
87d0b4a6
YG		 3853 return ret;
3854}
3855
3856static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
3857{
3858 int ret = 0, retries;
57d104c1 3859
87d0b4a6
YG		 3860 for (retries = UIC_HIBERN8_ENTER_RETRIES; retries > 0; retries--) {
3861 ret = __ufshcd_uic_hibern8_enter(hba);
6d303e4b 3862 if (!ret)
87d0b4a6
YG		 3863 goto out;
3864 }
3865out:
3866 return ret;
57d104c1
SJ		 3867}
3868
3869static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
3870{
3871 struct uic_command uic_cmd = {0};
3872 int ret;
911a0771 3873 ktime_t start = ktime_get();
57d104c1 3874
ee32c909
KK		 3875 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
3876
57d104c1
SJ		 3877 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
3878 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
911a0771
SJ		 3879 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
3880 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3881
57d104c1 3882 if (ret) {
53c12d0e
YG		 3883 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
3884 __func__, ret);
3885 ret = ufshcd_link_recovery(hba);
ff8e20c6 3886 } else {
ee32c909
KK		 3887 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
3888 POST_CHANGE);
ff8e20c6
DR		 3889 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
3890 hba->ufs_stats.hibern8_exit_cnt++;
3891 }
57d104c1
SJ		 3892
3893 return ret;
3894}
3895
71d848b8 3896void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
ad448378
AH		 3897{
3898 unsigned long flags;
3899
ee5f1042 3900 if (!ufshcd_is_auto_hibern8_supported(hba) || !hba->ahit)
ad448378
AH		 3901 return;
3902
3903 spin_lock_irqsave(hba->host->host_lock, flags);
3904 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
3905 spin_unlock_irqrestore(hba->host->host_lock, flags);
3906}
3907
5064636c
YG		 3908 /**
3909 * ufshcd_init_pwr_info - setting the POR (power on reset)
3910 * values in hba power info
3911 * @hba: per-adapter instance
3912 */
3913static void ufshcd_init_pwr_info(struct ufs_hba *hba)
3914{
3915 hba->pwr_info.gear_rx = UFS_PWM_G1;
3916 hba->pwr_info.gear_tx = UFS_PWM_G1;
3917 hba->pwr_info.lane_rx = 1;
3918 hba->pwr_info.lane_tx = 1;
3919 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
3920 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
3921 hba->pwr_info.hs_rate = 0;
3922}
3923
d3e89bac 3924/**
7eb584db
DR		 3925 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3926 * @hba: per-adapter instance
d3e89bac 3927 */
7eb584db 3928static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
d3e89bac 3929{
7eb584db
DR		 3930 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
3931
3932 if (hba->max_pwr_info.is_valid)
3933 return 0;
3934
2349b533
SJ		 3935 pwr_info->pwr_tx = FAST_MODE;
3936 pwr_info->pwr_rx = FAST_MODE;
7eb584db 3937 pwr_info->hs_rate = PA_HS_MODE_B;
d3e89bac
SJ		 3938
3939 /* Get the connected lane count */
7eb584db
DR		 3940 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
3941 &pwr_info->lane_rx);
3942 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
3943 &pwr_info->lane_tx);
3944
3945 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
3946 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
3947 __func__,
3948 pwr_info->lane_rx,
3949 pwr_info->lane_tx);
3950 return -EINVAL;
3951 }
d3e89bac
SJ		 3952
3953 /*
3954 * First, get the maximum gears of HS speed.
3955 * If a zero value, it means there is no HSGEAR capability.
3956 * Then, get the maximum gears of PWM speed.
3957 */
7eb584db
DR		 3958 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
3959 if (!pwr_info->gear_rx) {
3960 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
3961 &pwr_info->gear_rx);
3962 if (!pwr_info->gear_rx) {
3963 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
3964 __func__, pwr_info->gear_rx);
3965 return -EINVAL;
3966 }
2349b533 3967 pwr_info->pwr_rx = SLOW_MODE;
d3e89bac
SJ		 3968 }
3969
7eb584db
DR		 3970 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
3971 &pwr_info->gear_tx);
3972 if (!pwr_info->gear_tx) {
d3e89bac 3973 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
7eb584db
DR		 3974 &pwr_info->gear_tx);
3975 if (!pwr_info->gear_tx) {
3976 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
3977 __func__, pwr_info->gear_tx);
3978 return -EINVAL;
3979 }
2349b533 3980 pwr_info->pwr_tx = SLOW_MODE;
7eb584db
DR		 3981 }
3982
3983 hba->max_pwr_info.is_valid = true;
3984 return 0;
3985}
3986
3987static int ufshcd_change_power_mode(struct ufs_hba *hba,
3988 struct ufs_pa_layer_attr *pwr_mode)
3989{
3990 int ret;
3991
3992 /* if already configured to the requested pwr_mode */
3993 if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
3994 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
3995 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
3996 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
3997 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
3998 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
3999 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4000 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4001 return 0;
d3e89bac
SJ		 4002 }
4003
4004 /*
4005 * Configure attributes for power mode change with below.
4006 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4007 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4008 * - PA_HSSERIES
4009 */
7eb584db
DR		 4010 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4011 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4012 pwr_mode->lane_rx);
4013 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4014 pwr_mode->pwr_rx == FAST_MODE)
d3e89bac 4015 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
7eb584db
DR		 4016 else
4017 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
d3e89bac 4018
7eb584db
DR		 4019 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4020 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4021 pwr_mode->lane_tx);
4022 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4023 pwr_mode->pwr_tx == FAST_MODE)
d3e89bac 4024 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
7eb584db
DR		 4025 else
4026 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
d3e89bac 4027
7eb584db
DR		 4028 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4029 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4030 pwr_mode->pwr_rx == FAST_MODE ||
4031 pwr_mode->pwr_tx == FAST_MODE)
4032 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4033 pwr_mode->hs_rate);
d3e89bac 4034
7eb584db
DR		 4035 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4036 | pwr_mode->pwr_tx);
4037
4038 if (ret) {
d3e89bac 4039 dev_err(hba->dev,
7eb584db
DR		 4040 "%s: power mode change failed %d\n", __func__, ret);
4041 } else {
0263bcd0
YG		 4042 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4043 pwr_mode);
7eb584db
DR		 4044
4045 memcpy(&hba->pwr_info, pwr_mode,
4046 sizeof(struct ufs_pa_layer_attr));
4047 }
4048
4049 return ret;
4050}
4051
4052/**
4053 * ufshcd_config_pwr_mode - configure a new power mode
4054 * @hba: per-adapter instance
4055 * @desired_pwr_mode: desired power configuration
4056 */
0d846e70 4057int ufshcd_config_pwr_mode(struct ufs_hba *hba,
7eb584db
DR		 4058 struct ufs_pa_layer_attr *desired_pwr_mode)
4059{
4060 struct ufs_pa_layer_attr final_params = { 0 };
4061 int ret;
4062
0263bcd0
YG		 4063 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4064 desired_pwr_mode, &final_params);
4065
4066 if (ret)
7eb584db
DR		 4067 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4068
4069 ret = ufshcd_change_power_mode(hba, &final_params);
a3cd5ec5
SJ		 4070 if (!ret)
4071 ufshcd_print_pwr_info(hba);
d3e89bac
SJ		 4072
4073 return ret;
4074}
0d846e70 4075EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
d3e89bac 4076
68078d5c
DR		 4077/**
4078 * ufshcd_complete_dev_init() - checks device readiness
8aa29f19 4079 * @hba: per-adapter instance
68078d5c
DR		 4080 *
4081 * Set fDeviceInit flag and poll until device toggles it.
4082 */
4083static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4084{
dc3c8d3a
YG		 4085 int i;
4086 int err;
68078d5c
DR		 4087 bool flag_res = 1;
4088
dc3c8d3a
YG		 4089 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4090 QUERY_FLAG_IDN_FDEVICEINIT, NULL);
68078d5c
DR		 4091 if (err) {
4092 dev_err(hba->dev,
4093 "%s setting fDeviceInit flag failed with error %d\n",
4094 __func__, err);
4095 goto out;
4096 }
4097
dc3c8d3a
YG		 4098 /* poll for max. 1000 iterations for fDeviceInit flag to clear */
4099 for (i = 0; i < 1000 && !err && flag_res; i++)
4100 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4101 QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
4102
68078d5c
DR		 4103 if (err)
4104 dev_err(hba->dev,
4105 "%s reading fDeviceInit flag failed with error %d\n",
4106 __func__, err);
4107 else if (flag_res)
4108 dev_err(hba->dev,
4109 "%s fDeviceInit was not cleared by the device\n",
4110 __func__);
4111
4112out:
4113 return err;
4114}
4115
7a3e97b0
SY		 4116/**
4117 * ufshcd_make_hba_operational - Make UFS controller operational
4118 * @hba: per adapter instance
4119 *
4120 * To bring UFS host controller to operational state,
5c0c28a8
SRT		 4121 * 1. Enable required interrupts
4122 * 2. Configure interrupt aggregation
897efe62 4123 * 3. Program UTRL and UTMRL base address
5c0c28a8 4124 * 4. Configure run-stop-registers
7a3e97b0
SY		 4125 *
4126 * Returns 0 on success, non-zero value on failure
4127 */
4128static int ufshcd_make_hba_operational(struct ufs_hba *hba)
4129{
4130 int err = 0;
4131 u32 reg;
4132
6ccf44fe
SJ		 4133 /* Enable required interrupts */
4134 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4135
4136 /* Configure interrupt aggregation */
b852190e
YG		 4137 if (ufshcd_is_intr_aggr_allowed(hba))
4138 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4139 else
4140 ufshcd_disable_intr_aggr(hba);
6ccf44fe
SJ		 4141
4142 /* Configure UTRL and UTMRL base address registers */
4143 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4144 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4145 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4146 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4147 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4148 REG_UTP_TASK_REQ_LIST_BASE_L);
4149 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4150 REG_UTP_TASK_REQ_LIST_BASE_H);
4151
897efe62
YG		 4152 /*
4153 * Make sure base address and interrupt setup are updated before
4154 * enabling the run/stop registers below.
4155 */
4156 wmb();
4157
7a3e97b0
SY		 4158 /*
4159 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
7a3e97b0 4160 */
5c0c28a8 4161 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
7a3e97b0
SY		 4162 if (!(ufshcd_get_lists_status(reg))) {
4163 ufshcd_enable_run_stop_reg(hba);
4164 } else {
3b1d0580 4165 dev_err(hba->dev,
7a3e97b0
SY		 4166 "Host controller not ready to process requests");
4167 err = -EIO;
4168 goto out;
4169 }
4170
7a3e97b0
SY		 4171out:
4172 return err;
4173}
4174
596585a2
YG		 4175/**
4176 * ufshcd_hba_stop - Send controller to reset state
4177 * @hba: per adapter instance
4178 * @can_sleep: perform sleep or just spin
4179 */
4180static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
4181{
4182 int err;
4183
4184 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
4185 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4186 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4187 10, 1, can_sleep);
4188 if (err)
4189 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4190}
4191
7a3e97b0 4192/**
4404c5de 4193 * ufshcd_hba_execute_hce - initialize the controller
7a3e97b0
SY		 4194 * @hba: per adapter instance
4195 *
4196 * The controller resets itself and controller firmware initialization
4197 * sequence kicks off. When controller is ready it will set
4198 * the Host Controller Enable bit to 1.
4199 *
4200 * Returns 0 on success, non-zero value on failure
4201 */
4404c5de 4202static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
7a3e97b0
SY		 4203{
4204 int retry;
4205
596585a2 4206 if (!ufshcd_is_hba_active(hba))
7a3e97b0 4207 /* change controller state to "reset state" */
596585a2 4208 ufshcd_hba_stop(hba, true);
7a3e97b0 4209
57d104c1
SJ		 4210 /* UniPro link is disabled at this point */
4211 ufshcd_set_link_off(hba);
4212
0263bcd0 4213 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
5c0c28a8 4214
7a3e97b0
SY		 4215 /* start controller initialization sequence */
4216 ufshcd_hba_start(hba);
4217
4218 /*
4219 * To initialize a UFS host controller HCE bit must be set to 1.
4220 * During initialization the HCE bit value changes from 1->0->1.
4221 * When the host controller completes initialization sequence
4222 * it sets the value of HCE bit to 1. The same HCE bit is read back
4223 * to check if the controller has completed initialization sequence.
4224 * So without this delay the value HCE = 1, set in the previous
4225 * instruction might be read back.
4226 * This delay can be changed based on the controller.
4227 */
838c1efc 4228 usleep_range(1000, 1100);
7a3e97b0
SY		 4229
4230 /* wait for the host controller to complete initialization */
4231 retry = 10;
4232 while (ufshcd_is_hba_active(hba)) {
4233 if (retry) {
4234 retry--;
4235 } else {
3b1d0580 4236 dev_err(hba->dev,
7a3e97b0
SY		 4237 "Controller enable failed\n");
4238 return -EIO;
4239 }
838c1efc 4240 usleep_range(5000, 5100);
7a3e97b0 4241 }
5c0c28a8 4242
1d337ec2 4243 /* enable UIC related interrupts */
57d104c1 4244 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
1d337ec2 4245
0263bcd0 4246 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
5c0c28a8 4247
7a3e97b0
SY		 4248 return 0;
4249}
4250
4404c5de
AA		 4251static int ufshcd_hba_enable(struct ufs_hba *hba)
4252{
4253 int ret;
4254
4255 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4256 ufshcd_set_link_off(hba);
4257 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4258
4259 /* enable UIC related interrupts */
4260 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4261 ret = ufshcd_dme_reset(hba);
4262 if (!ret) {
4263 ret = ufshcd_dme_enable(hba);
4264 if (!ret)
4265 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4266 if (ret)
4267 dev_err(hba->dev,
4268 "Host controller enable failed with non-hce\n");
4269 }
4270 } else {
4271 ret = ufshcd_hba_execute_hce(hba);
4272 }
4273
4274 return ret;
4275}
7ca38cf3
YG		 4276static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4277{
4278 int tx_lanes, i, err = 0;
4279
4280 if (!peer)
4281 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4282 &tx_lanes);
4283 else
4284 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4285 &tx_lanes);
4286 for (i = 0; i < tx_lanes; i++) {
4287 if (!peer)
4288 err = ufshcd_dme_set(hba,
4289 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4290 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4291 0);
4292 else
4293 err = ufshcd_dme_peer_set(hba,
4294 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4295 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4296 0);
4297 if (err) {
4298 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4299 __func__, peer, i, err);
4300 break;
4301 }
4302 }
4303
4304 return err;
4305}
4306
4307static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4308{
4309 return ufshcd_disable_tx_lcc(hba, true);
4310}
4311
8808b4e9
SC		 4312static void ufshcd_update_reg_hist(struct ufs_err_reg_hist *reg_hist,
4313 u32 reg)
4314{
4315 reg_hist->reg[reg_hist->pos] = reg;
4316 reg_hist->tstamp[reg_hist->pos] = ktime_get();
4317 reg_hist->pos = (reg_hist->pos + 1) % UFS_ERR_REG_HIST_LENGTH;
4318}
4319
7a3e97b0 4320/**
6ccf44fe 4321 * ufshcd_link_startup - Initialize unipro link startup
7a3e97b0
SY		 4322 * @hba: per adapter instance
4323 *
6ccf44fe 4324 * Returns 0 for success, non-zero in case of failure
7a3e97b0 4325 */
6ccf44fe 4326static int ufshcd_link_startup(struct ufs_hba *hba)
7a3e97b0 4327{
6ccf44fe 4328 int ret;
1d337ec2 4329 int retries = DME_LINKSTARTUP_RETRIES;
7caf489b 4330 bool link_startup_again = false;
7a3e97b0 4331
7caf489b
SJ		 4332 /*
4333 * If UFS device isn't active then we will have to issue link startup
4334 * 2 times to make sure the device state move to active.
4335 */
4336 if (!ufshcd_is_ufs_dev_active(hba))
4337 link_startup_again = true;
7a3e97b0 4338
7caf489b 4339link_startup:
1d337ec2 4340 do {
0263bcd0 4341 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
6ccf44fe 4342
1d337ec2 4343 ret = ufshcd_dme_link_startup(hba);
5c0c28a8 4344
1d337ec2
SRT		 4345 /* check if device is detected by inter-connect layer */
4346 if (!ret && !ufshcd_is_device_present(hba)) {
8808b4e9
SC		 4347 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4348 0);
1d337ec2
SRT		 4349 dev_err(hba->dev, "%s: Device not present\n", __func__);
4350 ret = -ENXIO;
4351 goto out;
4352 }
6ccf44fe 4353
1d337ec2
SRT		 4354 /*
4355 * DME link lost indication is only received when link is up,
4356 * but we can't be sure if the link is up until link startup
4357 * succeeds. So reset the local Uni-Pro and try again.
4358 */
8808b4e9
SC		 4359 if (ret && ufshcd_hba_enable(hba)) {
4360 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4361 (u32)ret);
1d337ec2 4362 goto out;
8808b4e9 4363 }
1d337ec2
SRT		 4364 } while (ret && retries--);
4365
8808b4e9 4366 if (ret) {
1d337ec2 4367 /* failed to get the link up... retire */
8808b4e9
SC		 4368 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4369 (u32)ret);
5c0c28a8 4370 goto out;
8808b4e9 4371 }
5c0c28a8 4372
7caf489b
SJ		 4373 if (link_startup_again) {
4374 link_startup_again = false;
4375 retries = DME_LINKSTARTUP_RETRIES;
4376 goto link_startup;
4377 }
4378
d2aebb9b
SJ		 4379 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4380 ufshcd_init_pwr_info(hba);
4381 ufshcd_print_pwr_info(hba);
4382
7ca38cf3
YG		 4383 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4384 ret = ufshcd_disable_device_tx_lcc(hba);
4385 if (ret)
4386 goto out;
4387 }
4388
5c0c28a8 4389 /* Include any host controller configuration via UIC commands */
0263bcd0
YG		 4390 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4391 if (ret)
4392 goto out;
7a3e97b0 4393
5c0c28a8 4394 ret = ufshcd_make_hba_operational(hba);
6ccf44fe 4395out:
7942f7b5 4396 if (ret) {
6ccf44fe 4397 dev_err(hba->dev, "link startup failed %d\n", ret);
7942f7b5
VG		 4398 ufshcd_print_host_state(hba);
4399 ufshcd_print_pwr_info(hba);
4400 ufshcd_print_host_regs(hba);
4401 }
6ccf44fe 4402 return ret;
7a3e97b0
SY		 4403}
4404
5a0b0cb9
SRT		 4405/**
4406 * ufshcd_verify_dev_init() - Verify device initialization
4407 * @hba: per-adapter instance
4408 *
4409 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4410 * device Transport Protocol (UTP) layer is ready after a reset.
4411 * If the UTP layer at the device side is not initialized, it may
4412 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4413 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4414 */
4415static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4416{
4417 int err = 0;
4418 int retries;
4419
1ab27c9c 4420 ufshcd_hold(hba, false);
5a0b0cb9
SRT		 4421 mutex_lock(&hba->dev_cmd.lock);
4422 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4423 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4424 NOP_OUT_TIMEOUT);
4425
4426 if (!err || err == -ETIMEDOUT)
4427 break;
4428
4429 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4430 }
4431 mutex_unlock(&hba->dev_cmd.lock);
1ab27c9c 4432 ufshcd_release(hba);
5a0b0cb9
SRT		 4433
4434 if (err)
4435 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4436 return err;
4437}
4438
0ce147d4
SJ		 4439/**
4440 * ufshcd_set_queue_depth - set lun queue depth
4441 * @sdev: pointer to SCSI device
4442 *
4443 * Read bLUQueueDepth value and activate scsi tagged command
4444 * queueing. For WLUN, queue depth is set to 1. For best-effort
4445 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4446 * value that host can queue.
4447 */
4448static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4449{
4450 int ret = 0;
4451 u8 lun_qdepth;
4452 struct ufs_hba *hba;
4453
4454 hba = shost_priv(sdev->host);
4455
4456 lun_qdepth = hba->nutrs;
dbd34a61
SM		 4457 ret = ufshcd_read_unit_desc_param(hba,
4458 ufshcd_scsi_to_upiu_lun(sdev->lun),
4459 UNIT_DESC_PARAM_LU_Q_DEPTH,
4460 &lun_qdepth,
4461 sizeof(lun_qdepth));
0ce147d4
SJ		 4462
4463 /* Some WLUN doesn't support unit descriptor */
4464 if (ret == -EOPNOTSUPP)
4465 lun_qdepth = 1;
4466 else if (!lun_qdepth)
4467 /* eventually, we can figure out the real queue depth */
4468 lun_qdepth = hba->nutrs;
4469 else
4470 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4471
4472 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4473 __func__, lun_qdepth);
db5ed4df 4474 scsi_change_queue_depth(sdev, lun_qdepth);
0ce147d4
SJ		 4475}
4476
57d104c1
SJ		 4477/*
4478 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4479 * @hba: per-adapter instance
4480 * @lun: UFS device lun id
4481 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4482 *
4483 * Returns 0 in case of success and b_lu_write_protect status would be returned
4484 * @b_lu_write_protect parameter.
4485 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4486 * Returns -EINVAL in case of invalid parameters passed to this function.
4487 */
4488static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4489 u8 lun,
4490 u8 *b_lu_write_protect)
4491{
4492 int ret;
4493
4494 if (!b_lu_write_protect)
4495 ret = -EINVAL;
4496 /*
4497 * According to UFS device spec, RPMB LU can't be write
4498 * protected so skip reading bLUWriteProtect parameter for
4499 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4500 */
4501 else if (lun >= UFS_UPIU_MAX_GENERAL_LUN)
4502 ret = -ENOTSUPP;
4503 else
4504 ret = ufshcd_read_unit_desc_param(hba,
4505 lun,
4506 UNIT_DESC_PARAM_LU_WR_PROTECT,
4507 b_lu_write_protect,
4508 sizeof(*b_lu_write_protect));
4509 return ret;
4510}
4511
4512/**
4513 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4514 * status
4515 * @hba: per-adapter instance
4516 * @sdev: pointer to SCSI device
4517 *
4518 */
4519static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4520 struct scsi_device *sdev)
4521{
4522 if (hba->dev_info.f_power_on_wp_en &&
4523 !hba->dev_info.is_lu_power_on_wp) {
4524 u8 b_lu_write_protect;
4525
4526 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4527 &b_lu_write_protect) &&
4528 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4529 hba->dev_info.is_lu_power_on_wp = true;
4530 }
4531}
4532
7a3e97b0
SY		 4533/**
4534 * ufshcd_slave_alloc - handle initial SCSI device configurations
4535 * @sdev: pointer to SCSI device
4536 *
4537 * Returns success
4538 */
4539static int ufshcd_slave_alloc(struct scsi_device *sdev)
4540{
4541 struct ufs_hba *hba;
4542
4543 hba = shost_priv(sdev->host);
7a3e97b0
SY		 4544
4545 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4546 sdev->use_10_for_ms = 1;
a3a76391
CG		 4547
4548 /* DBD field should be set to 1 in mode sense(10) */
4549 sdev->set_dbd_for_ms = 1;
7a3e97b0 4550
e8e7f271
SRT		 4551 /* allow SCSI layer to restart the device in case of errors */
4552 sdev->allow_restart = 1;
4264fd61 4553
b2a6c522
SRT		 4554 /* REPORT SUPPORTED OPERATION CODES is not supported */
4555 sdev->no_report_opcodes = 1;
4556
84af7e8b
SRT		 4557 /* WRITE_SAME command is not supported */
4558 sdev->no_write_same = 1;
e8e7f271 4559
0ce147d4 4560 ufshcd_set_queue_depth(sdev);
4264fd61 4561
57d104c1
SJ		 4562 ufshcd_get_lu_power_on_wp_status(hba, sdev);
4563
7a3e97b0
SY		 4564 return 0;
4565}
4566
4264fd61
SRT		 4567/**
4568 * ufshcd_change_queue_depth - change queue depth
4569 * @sdev: pointer to SCSI device
4570 * @depth: required depth to set
4264fd61 4571 *
db5ed4df 4572 * Change queue depth and make sure the max. limits are not crossed.
4264fd61 4573 */
db5ed4df 4574static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4264fd61
SRT		 4575{
4576 struct ufs_hba *hba = shost_priv(sdev->host);
4577
4578 if (depth > hba->nutrs)
4579 depth = hba->nutrs;
db5ed4df 4580 return scsi_change_queue_depth(sdev, depth);
4264fd61
SRT		 4581}
4582
eeda4749
AM		 4583/**
4584 * ufshcd_slave_configure - adjust SCSI device configurations
4585 * @sdev: pointer to SCSI device
4586 */
4587static int ufshcd_slave_configure(struct scsi_device *sdev)
4588{
49615ba1 4589 struct ufs_hba *hba = shost_priv(sdev->host);
eeda4749
AM		 4590 struct request_queue *q = sdev->request_queue;
4591
4592 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
49615ba1
SC		 4593
4594 if (ufshcd_is_rpm_autosuspend_allowed(hba))
4595 sdev->rpm_autosuspend = 1;
4596
eeda4749
AM		 4597 return 0;
4598}
4599
7a3e97b0
SY		 4600/**
4601 * ufshcd_slave_destroy - remove SCSI device configurations
4602 * @sdev: pointer to SCSI device
4603 */
4604static void ufshcd_slave_destroy(struct scsi_device *sdev)
4605{
4606 struct ufs_hba *hba;
4607
4608 hba = shost_priv(sdev->host);
0ce147d4 4609 /* Drop the reference as it won't be needed anymore */
7c48bfd0
AM		 4610 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4611 unsigned long flags;
4612
4613 spin_lock_irqsave(hba->host->host_lock, flags);
0ce147d4 4614 hba->sdev_ufs_device = NULL;
7c48bfd0
AM		 4615 spin_unlock_irqrestore(hba->host->host_lock, flags);
4616 }
7a3e97b0
SY		 4617}
4618
7a3e97b0
SY		 4619/**
4620 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
8aa29f19 4621 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4622 * @scsi_status: SCSI command status
4623 *
4624 * Returns value base on SCSI command status
4625 */
4626static inline int
4627ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4628{
4629 int result = 0;
4630
4631 switch (scsi_status) {
7a3e97b0 4632 case SAM_STAT_CHECK_CONDITION:
1c2623c5 4633 ufshcd_copy_sense_data(lrbp);
30eb2e4c 4634 /* fallthrough */
1c2623c5 4635 case SAM_STAT_GOOD:
7a3e97b0
SY		 4636 result |= DID_OK << 16 |
4637 COMMAND_COMPLETE << 8 |
1c2623c5 4638 scsi_status;
7a3e97b0
SY		 4639 break;
4640 case SAM_STAT_TASK_SET_FULL:
1c2623c5 4641 case SAM_STAT_BUSY:
7a3e97b0 4642 case SAM_STAT_TASK_ABORTED:
1c2623c5
SJ		 4643 ufshcd_copy_sense_data(lrbp);
4644 result |= scsi_status;
7a3e97b0
SY		 4645 break;
4646 default:
4647 result |= DID_ERROR << 16;
4648 break;
4649 } /* end of switch */
4650
4651 return result;
4652}
4653
4654/**
4655 * ufshcd_transfer_rsp_status - Get overall status of the response
4656 * @hba: per adapter instance
8aa29f19 4657 * @lrbp: pointer to local reference block of completed command
7a3e97b0
SY		 4658 *
4659 * Returns result of the command to notify SCSI midlayer
4660 */
4661static inline int
4662ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
4663{
4664 int result = 0;
4665 int scsi_status;
4666 int ocs;
4667
4668 /* overall command status of utrd */
4669 ocs = ufshcd_get_tr_ocs(lrbp);
4670
4671 switch (ocs) {
4672 case OCS_SUCCESS:
5a0b0cb9 4673 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
ff8e20c6 4674 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5a0b0cb9
SRT		 4675 switch (result) {
4676 case UPIU_TRANSACTION_RESPONSE:
4677 /*
4678 * get the response UPIU result to extract
4679 * the SCSI command status
4680 */
4681 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
4682
4683 /*
4684 * get the result based on SCSI status response
4685 * to notify the SCSI midlayer of the command status
4686 */
4687 scsi_status = result & MASK_SCSI_STATUS;
4688 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
66ec6d59 4689
f05ac2e5
YG		 4690 /*
4691 * Currently we are only supporting BKOPs exception
4692 * events hence we can ignore BKOPs exception event
4693 * during power management callbacks. BKOPs exception
4694 * event is not expected to be raised in runtime suspend
4695 * callback as it allows the urgent bkops.
4696 * During system suspend, we are anyway forcefully
4697 * disabling the bkops and if urgent bkops is needed
4698 * it will be enabled on system resume. Long term
4699 * solution could be to abort the system suspend if
4700 * UFS device needs urgent BKOPs.
4701 */
4702 if (!hba->pm_op_in_progress &&
4703 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
66ec6d59 4704 schedule_work(&hba->eeh_work);
5a0b0cb9
SRT		 4705 break;
4706 case UPIU_TRANSACTION_REJECT_UPIU:
4707 /* TODO: handle Reject UPIU Response */
4708 result = DID_ERROR << 16;
3b1d0580 4709 dev_err(hba->dev,
5a0b0cb9
SRT		 4710 "Reject UPIU not fully implemented\n");
4711 break;
4712 default:
5a0b0cb9
SRT		 4713 dev_err(hba->dev,
4714 "Unexpected request response code = %x\n",
4715 result);
e0347d89 4716 result = DID_ERROR << 16;
7a3e97b0
SY		 4717 break;
4718 }
7a3e97b0
SY		 4719 break;
4720 case OCS_ABORTED:
4721 result |= DID_ABORT << 16;
4722 break;
e8e7f271
SRT		 4723 case OCS_INVALID_COMMAND_STATUS:
4724 result |= DID_REQUEUE << 16;
4725 break;
7a3e97b0
SY		 4726 case OCS_INVALID_CMD_TABLE_ATTR:
4727 case OCS_INVALID_PRDT_ATTR:
4728 case OCS_MISMATCH_DATA_BUF_SIZE:
4729 case OCS_MISMATCH_RESP_UPIU_SIZE:
4730 case OCS_PEER_COMM_FAILURE:
4731 case OCS_FATAL_ERROR:
4732 default:
4733 result |= DID_ERROR << 16;
3b1d0580 4734 dev_err(hba->dev,
ff8e20c6
DR		 4735 "OCS error from controller = %x for tag %d\n",
4736 ocs, lrbp->task_tag);
4737 ufshcd_print_host_regs(hba);
6ba65588 4738 ufshcd_print_host_state(hba);
7a3e97b0
SY		 4739 break;
4740 } /* end of switch */
4741
2df74b69 4742 if ((host_byte(result) != DID_OK) && !hba->silence_err_logs)
66cc820f 4743 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
7a3e97b0
SY		 4744 return result;
4745}
4746
6ccf44fe
SJ		 4747/**
4748 * ufshcd_uic_cmd_compl - handle completion of uic command
4749 * @hba: per adapter instance
53b3d9c3 4750 * @intr_status: interrupt status generated by the controller
9333d775
VG		 4751 *
4752 * Returns
4753 * IRQ_HANDLED - If interrupt is valid
4754 * IRQ_NONE - If invalid interrupt
6ccf44fe 4755 */
9333d775 4756static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
6ccf44fe 4757{
9333d775
VG		 4758 irqreturn_t retval = IRQ_NONE;
4759
53b3d9c3 4760 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
6ccf44fe
SJ		 4761 hba->active_uic_cmd->argument2 |=
4762 ufshcd_get_uic_cmd_result(hba);
12b4fdb4
SJ		 4763 hba->active_uic_cmd->argument3 =
4764 ufshcd_get_dme_attr_val(hba);
6ccf44fe 4765 complete(&hba->active_uic_cmd->done);
9333d775 4766 retval = IRQ_HANDLED;
6ccf44fe 4767 }
53b3d9c3 4768
9333d775 4769 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
57d104c1 4770 complete(hba->uic_async_done);
9333d775
VG		 4771 retval = IRQ_HANDLED;
4772 }
4773 return retval;
6ccf44fe
SJ		 4774}
4775
7a3e97b0 4776/**
9a47ec7c 4777 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
7a3e97b0 4778 * @hba: per adapter instance
9a47ec7c 4779 * @completed_reqs: requests to complete
7a3e97b0 4780 */
9a47ec7c
YG		 4781static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
4782 unsigned long completed_reqs)
7a3e97b0 4783{
5a0b0cb9
SRT		 4784 struct ufshcd_lrb *lrbp;
4785 struct scsi_cmnd *cmd;
7a3e97b0
SY		 4786 int result;
4787 int index;
e9d501b1 4788
e9d501b1
DR		 4789 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
4790 lrbp = &hba->lrb[index];
4791 cmd = lrbp->cmd;
4792 if (cmd) {
1a07f2d9 4793 ufshcd_add_command_trace(hba, index, "complete");
e9d501b1
DR		 4794 result = ufshcd_transfer_rsp_status(hba, lrbp);
4795 scsi_dma_unmap(cmd);
4796 cmd->result = result;
4797 /* Mark completed command as NULL in LRB */
4798 lrbp->cmd = NULL;
74a527a2 4799 lrbp->compl_time_stamp = ktime_get();
e9d501b1
DR		 4800 /* Do not touch lrbp after scsi done */
4801 cmd->scsi_done(cmd);
1ab27c9c 4802 __ufshcd_release(hba);
300bb13f
JP		 4803 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
4804 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
74a527a2 4805 lrbp->compl_time_stamp = ktime_get();
1a07f2d9
LS		 4806 if (hba->dev_cmd.complete) {
4807 ufshcd_add_command_trace(hba, index,
4808 "dev_complete");
e9d501b1 4809 complete(hba->dev_cmd.complete);
1a07f2d9 4810 }
e9d501b1 4811 }
401f1e44
SJ		 4812 if (ufshcd_is_clkscaling_supported(hba))
4813 hba->clk_scaling.active_reqs--;
e9d501b1 4814 }
7a3e97b0
SY		 4815
4816 /* clear corresponding bits of completed commands */
4817 hba->outstanding_reqs ^= completed_reqs;
4818
856b3483 4819 ufshcd_clk_scaling_update_busy(hba);
7a3e97b0
SY		 4820}
4821
9a47ec7c
YG		 4822/**
4823 * ufshcd_transfer_req_compl - handle SCSI and query command completion
4824 * @hba: per adapter instance
9333d775
VG		 4825 *
4826 * Returns
4827 * IRQ_HANDLED - If interrupt is valid
4828 * IRQ_NONE - If invalid interrupt
9a47ec7c 4829 */
9333d775 4830static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
9a47ec7c
YG		 4831{
4832 unsigned long completed_reqs;
4833 u32 tr_doorbell;
4834
4835 /* Resetting interrupt aggregation counters first and reading the
4836 * DOOR_BELL afterward allows us to handle all the completed requests.
4837 * In order to prevent other interrupts starvation the DB is read once
4838 * after reset. The down side of this solution is the possibility of
4839 * false interrupt if device completes another request after resetting
4840 * aggregation and before reading the DB.
4841 */
5ac6abc9
AA		 4842 if (ufshcd_is_intr_aggr_allowed(hba) &&
4843 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
9a47ec7c
YG		 4844 ufshcd_reset_intr_aggr(hba);
4845
4846 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4847 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
4848
9333d775
VG		 4849 if (completed_reqs) {
4850 __ufshcd_transfer_req_compl(hba, completed_reqs);
4851 return IRQ_HANDLED;
4852 } else {
4853 return IRQ_NONE;
4854 }
9a47ec7c
YG		 4855}
4856
66ec6d59
SRT		 4857/**
4858 * ufshcd_disable_ee - disable exception event
4859 * @hba: per-adapter instance
4860 * @mask: exception event to disable
4861 *
4862 * Disables exception event in the device so that the EVENT_ALERT
4863 * bit is not set.
4864 *
4865 * Returns zero on success, non-zero error value on failure.
4866 */
4867static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
4868{
4869 int err = 0;
4870 u32 val;
4871
4872 if (!(hba->ee_ctrl_mask & mask))
4873 goto out;
4874
4875 val = hba->ee_ctrl_mask & ~mask;
d7e2ddd5 4876 val &= MASK_EE_STATUS;
5e86ae44 4877 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 4878 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4879 if (!err)
4880 hba->ee_ctrl_mask &= ~mask;
4881out:
4882 return err;
4883}
4884
4885/**
4886 * ufshcd_enable_ee - enable exception event
4887 * @hba: per-adapter instance
4888 * @mask: exception event to enable
4889 *
4890 * Enable corresponding exception event in the device to allow
4891 * device to alert host in critical scenarios.
4892 *
4893 * Returns zero on success, non-zero error value on failure.
4894 */
4895static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
4896{
4897 int err = 0;
4898 u32 val;
4899
4900 if (hba->ee_ctrl_mask & mask)
4901 goto out;
4902
4903 val = hba->ee_ctrl_mask | mask;
d7e2ddd5 4904 val &= MASK_EE_STATUS;
5e86ae44 4905 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
66ec6d59
SRT		 4906 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4907 if (!err)
4908 hba->ee_ctrl_mask |= mask;
4909out:
4910 return err;
4911}
4912
4913/**
4914 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4915 * @hba: per-adapter instance
4916 *
4917 * Allow device to manage background operations on its own. Enabling
4918 * this might lead to inconsistent latencies during normal data transfers
4919 * as the device is allowed to manage its own way of handling background
4920 * operations.
4921 *
4922 * Returns zero on success, non-zero on failure.
4923 */
4924static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
4925{
4926 int err = 0;
4927
4928 if (hba->auto_bkops_enabled)
4929 goto out;
4930
dc3c8d3a 4931 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
66ec6d59
SRT		 4932 QUERY_FLAG_IDN_BKOPS_EN, NULL);
4933 if (err) {
4934 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
4935 __func__, err);
4936 goto out;
4937 }
4938
4939 hba->auto_bkops_enabled = true;
7ff5ab47 4940 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
66ec6d59
SRT		 4941
4942 /* No need of URGENT_BKOPS exception from the device */
4943 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
4944 if (err)
4945 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
4946 __func__, err);
4947out:
4948 return err;
4949}
4950
4951/**
4952 * ufshcd_disable_auto_bkops - block device in doing background operations
4953 * @hba: per-adapter instance
4954 *
4955 * Disabling background operations improves command response latency but
4956 * has drawback of device moving into critical state where the device is
4957 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
4958 * host is idle so that BKOPS are managed effectively without any negative
4959 * impacts.
4960 *
4961 * Returns zero on success, non-zero on failure.
4962 */
4963static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
4964{
4965 int err = 0;
4966
4967 if (!hba->auto_bkops_enabled)
4968 goto out;
4969
4970 /*
4971 * If host assisted BKOPs is to be enabled, make sure
4972 * urgent bkops exception is allowed.
4973 */
4974 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
4975 if (err) {
4976 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
4977 __func__, err);
4978 goto out;
4979 }
4980
dc3c8d3a 4981 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
66ec6d59
SRT		 4982 QUERY_FLAG_IDN_BKOPS_EN, NULL);
4983 if (err) {
4984 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
4985 __func__, err);
4986 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
4987 goto out;
4988 }
4989
4990 hba->auto_bkops_enabled = false;
7ff5ab47 4991 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
24366c2a 4992 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 4993out:
4994 return err;
4995}
4996
4997/**
4e768e76 4998 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
66ec6d59
SRT		 4999 * @hba: per adapter instance
5000 *
5001 * After a device reset the device may toggle the BKOPS_EN flag
5002 * to default value. The s/w tracking variables should be updated
4e768e76
SJ		 5003 * as well. This function would change the auto-bkops state based on
5004 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
66ec6d59 5005 */
4e768e76 5006static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
66ec6d59 5007{
4e768e76
SJ		 5008 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5009 hba->auto_bkops_enabled = false;
5010 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5011 ufshcd_enable_auto_bkops(hba);
5012 } else {
5013 hba->auto_bkops_enabled = true;
5014 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5015 ufshcd_disable_auto_bkops(hba);
5016 }
24366c2a 5017 hba->is_urgent_bkops_lvl_checked = false;
66ec6d59
SRT		 5018}
5019
5020static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5021{
5e86ae44 5022 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5023 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5024}
5025
5026/**
57d104c1 5027 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
66ec6d59 5028 * @hba: per-adapter instance
57d104c1 5029 * @status: bkops_status value
66ec6d59 5030 *
57d104c1
SJ		 5031 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5032 * flag in the device to permit background operations if the device
5033 * bkops_status is greater than or equal to "status" argument passed to
5034 * this function, disable otherwise.
5035 *
5036 * Returns 0 for success, non-zero in case of failure.
5037 *
5038 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5039 * to know whether auto bkops is enabled or disabled after this function
5040 * returns control to it.
66ec6d59 5041 */
57d104c1
SJ		 5042static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5043 enum bkops_status status)
66ec6d59
SRT		 5044{
5045 int err;
57d104c1 5046 u32 curr_status = 0;
66ec6d59 5047
57d104c1 5048 err = ufshcd_get_bkops_status(hba, &curr_status);
66ec6d59
SRT		 5049 if (err) {
5050 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5051 __func__, err);
5052 goto out;
57d104c1
SJ		 5053 } else if (curr_status > BKOPS_STATUS_MAX) {
5054 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5055 __func__, curr_status);
5056 err = -EINVAL;
5057 goto out;
66ec6d59
SRT		 5058 }
5059
57d104c1 5060 if (curr_status >= status)
66ec6d59 5061 err = ufshcd_enable_auto_bkops(hba);
57d104c1
SJ		 5062 else
5063 err = ufshcd_disable_auto_bkops(hba);
24366c2a 5064 hba->urgent_bkops_lvl = curr_status;
66ec6d59
SRT		 5065out:
5066 return err;
5067}
5068
57d104c1
SJ		 5069/**
5070 * ufshcd_urgent_bkops - handle urgent bkops exception event
5071 * @hba: per-adapter instance
5072 *
5073 * Enable fBackgroundOpsEn flag in the device to permit background
5074 * operations.
5075 *
5076 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5077 * and negative error value for any other failure.
5078 */
5079static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5080{
afdfff59 5081 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
57d104c1
SJ		 5082}
5083
66ec6d59
SRT		 5084static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5085{
5e86ae44 5086 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
66ec6d59
SRT		 5087 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5088}
5089
afdfff59
YG		 5090static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5091{
5092 int err;
5093 u32 curr_status = 0;
5094
5095 if (hba->is_urgent_bkops_lvl_checked)
5096 goto enable_auto_bkops;
5097
5098 err = ufshcd_get_bkops_status(hba, &curr_status);
5099 if (err) {
5100 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5101 __func__, err);
5102 goto out;
5103 }
5104
5105 /*
5106 * We are seeing that some devices are raising the urgent bkops
5107 * exception events even when BKOPS status doesn't indicate performace
5108 * impacted or critical. Handle these device by determining their urgent
5109 * bkops status at runtime.
5110 */
5111 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5112 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5113 __func__, curr_status);
5114 /* update the current status as the urgent bkops level */
5115 hba->urgent_bkops_lvl = curr_status;
5116 hba->is_urgent_bkops_lvl_checked = true;
5117 }
5118
5119enable_auto_bkops:
5120 err = ufshcd_enable_auto_bkops(hba);
5121out:
5122 if (err < 0)
5123 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5124 __func__, err);
5125}
5126
66ec6d59
SRT		 5127/**
5128 * ufshcd_exception_event_handler - handle exceptions raised by device
5129 * @work: pointer to work data
5130 *
5131 * Read bExceptionEventStatus attribute from the device and handle the
5132 * exception event accordingly.
5133 */
5134static void ufshcd_exception_event_handler(struct work_struct *work)
5135{
5136 struct ufs_hba *hba;
5137 int err;
5138 u32 status = 0;
5139 hba = container_of(work, struct ufs_hba, eeh_work);
5140
62694735 5141 pm_runtime_get_sync(hba->dev);
2e3611e9 5142 scsi_block_requests(hba->host);
66ec6d59
SRT		 5143 err = ufshcd_get_ee_status(hba, &status);
5144 if (err) {
5145 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5146 __func__, err);
5147 goto out;
5148 }
5149
5150 status &= hba->ee_ctrl_mask;
afdfff59
YG		 5151
5152 if (status & MASK_EE_URGENT_BKOPS)
5153 ufshcd_bkops_exception_event_handler(hba);
5154
66ec6d59 5155out:
2e3611e9 5156 scsi_unblock_requests(hba->host);
62694735 5157 pm_runtime_put_sync(hba->dev);
66ec6d59
SRT		 5158 return;
5159}
5160
9a47ec7c
YG		 5161/* Complete requests that have door-bell cleared */
5162static void ufshcd_complete_requests(struct ufs_hba *hba)
5163{
5164 ufshcd_transfer_req_compl(hba);
5165 ufshcd_tmc_handler(hba);
5166}
5167
583fa62d
YG		 5168/**
5169 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5170 * to recover from the DL NAC errors or not.
5171 * @hba: per-adapter instance
5172 *
5173 * Returns true if error handling is required, false otherwise
5174 */
5175static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5176{
5177 unsigned long flags;
5178 bool err_handling = true;
5179
5180 spin_lock_irqsave(hba->host->host_lock, flags);
5181 /*
5182 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5183 * device fatal error and/or DL NAC & REPLAY timeout errors.
5184 */
5185 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5186 goto out;
5187
5188 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5189 ((hba->saved_err & UIC_ERROR) &&
5190 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5191 goto out;
5192
5193 if ((hba->saved_err & UIC_ERROR) &&
5194 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5195 int err;
5196 /*
5197 * wait for 50ms to see if we can get any other errors or not.
5198 */
5199 spin_unlock_irqrestore(hba->host->host_lock, flags);
5200 msleep(50);
5201 spin_lock_irqsave(hba->host->host_lock, flags);
5202
5203 /*
5204 * now check if we have got any other severe errors other than
5205 * DL NAC error?
5206 */
5207 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5208 ((hba->saved_err & UIC_ERROR) &&
5209 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5210 goto out;
5211
5212 /*
5213 * As DL NAC is the only error received so far, send out NOP
5214 * command to confirm if link is still active or not.
5215 * - If we don't get any response then do error recovery.
5216 * - If we get response then clear the DL NAC error bit.
5217 */
5218
5219 spin_unlock_irqrestore(hba->host->host_lock, flags);
5220 err = ufshcd_verify_dev_init(hba);
5221 spin_lock_irqsave(hba->host->host_lock, flags);
5222
5223 if (err)
5224 goto out;
5225
5226 /* Link seems to be alive hence ignore the DL NAC errors */
5227 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5228 hba->saved_err &= ~UIC_ERROR;
5229 /* clear NAC error */
5230 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5231 if (!hba->saved_uic_err) {
5232 err_handling = false;
5233 goto out;
5234 }
5235 }
5236out:
5237 spin_unlock_irqrestore(hba->host->host_lock, flags);
5238 return err_handling;
5239}
5240
7a3e97b0 5241/**
e8e7f271
SRT		 5242 * ufshcd_err_handler - handle UFS errors that require s/w attention
5243 * @work: pointer to work structure
7a3e97b0 5244 */
e8e7f271 5245static void ufshcd_err_handler(struct work_struct *work)
7a3e97b0
SY		 5246{
5247 struct ufs_hba *hba;
e8e7f271
SRT		 5248 unsigned long flags;
5249 u32 err_xfer = 0;
5250 u32 err_tm = 0;
5251 int err = 0;
5252 int tag;
9a47ec7c 5253 bool needs_reset = false;
e8e7f271
SRT		 5254
5255 hba = container_of(work, struct ufs_hba, eh_work);
7a3e97b0 5256
62694735 5257 pm_runtime_get_sync(hba->dev);
1ab27c9c 5258 ufshcd_hold(hba, false);
e8e7f271
SRT		 5259
5260 spin_lock_irqsave(hba->host->host_lock, flags);
9a47ec7c 5261 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
e8e7f271 5262 goto out;
e8e7f271
SRT		 5263
5264 hba->ufshcd_state = UFSHCD_STATE_RESET;
5265 ufshcd_set_eh_in_progress(hba);
5266
5267 /* Complete requests that have door-bell cleared by h/w */
9a47ec7c 5268 ufshcd_complete_requests(hba);
583fa62d
YG		 5269
5270 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5271 bool ret;
5272
5273 spin_unlock_irqrestore(hba->host->host_lock, flags);
5274 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5275 ret = ufshcd_quirk_dl_nac_errors(hba);
5276 spin_lock_irqsave(hba->host->host_lock, flags);
5277 if (!ret)
5278 goto skip_err_handling;
5279 }
9a47ec7c 5280 if ((hba->saved_err & INT_FATAL_ERRORS) ||
82174440 5281 (hba->saved_err & UFSHCD_UIC_HIBERN8_MASK) ||
9a47ec7c
YG		 5282 ((hba->saved_err & UIC_ERROR) &&
5283 (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
5284 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5285 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5286 needs_reset = true;
e8e7f271 5287
9a47ec7c
YG		 5288 /*
5289 * if host reset is required then skip clearing the pending
2df74b69
CG		 5290 * transfers forcefully because they will get cleared during
5291 * host reset and restore
9a47ec7c
YG		 5292 */
5293 if (needs_reset)
5294 goto skip_pending_xfer_clear;
5295
5296 /* release lock as clear command might sleep */
5297 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5298 /* Clear pending transfer requests */
9a47ec7c
YG		 5299 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
5300 if (ufshcd_clear_cmd(hba, tag)) {
5301 err_xfer = true;
5302 goto lock_skip_pending_xfer_clear;
5303 }
5304 }
e8e7f271
SRT		 5305
5306 /* Clear pending task management requests */
9a47ec7c
YG		 5307 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
5308 if (ufshcd_clear_tm_cmd(hba, tag)) {
5309 err_tm = true;
5310 goto lock_skip_pending_xfer_clear;
5311 }
5312 }
e8e7f271 5313
9a47ec7c 5314lock_skip_pending_xfer_clear:
e8e7f271 5315 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 5316
9a47ec7c
YG		 5317 /* Complete the requests that are cleared by s/w */
5318 ufshcd_complete_requests(hba);
5319
5320 if (err_xfer || err_tm)
5321 needs_reset = true;
5322
5323skip_pending_xfer_clear:
e8e7f271 5324 /* Fatal errors need reset */
9a47ec7c
YG		 5325 if (needs_reset) {
5326 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
5327
5328 /*
5329 * ufshcd_reset_and_restore() does the link reinitialization
5330 * which will need atleast one empty doorbell slot to send the
5331 * device management commands (NOP and query commands).
5332 * If there is no slot empty at this moment then free up last
5333 * slot forcefully.
5334 */
5335 if (hba->outstanding_reqs == max_doorbells)
5336 __ufshcd_transfer_req_compl(hba,
5337 (1UL << (hba->nutrs - 1)));
5338
5339 spin_unlock_irqrestore(hba->host->host_lock, flags);
e8e7f271 5340 err = ufshcd_reset_and_restore(hba);
9a47ec7c 5341 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271
SRT		 5342 if (err) {
5343 dev_err(hba->dev, "%s: reset and restore failed\n",
5344 __func__);
5345 hba->ufshcd_state = UFSHCD_STATE_ERROR;
5346 }
5347 /*
5348 * Inform scsi mid-layer that we did reset and allow to handle
5349 * Unit Attention properly.
5350 */
5351 scsi_report_bus_reset(hba->host, 0);
5352 hba->saved_err = 0;
5353 hba->saved_uic_err = 0;
5354 }
9a47ec7c 5355
583fa62d 5356skip_err_handling:
9a47ec7c
YG		 5357 if (!needs_reset) {
5358 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5359 if (hba->saved_err || hba->saved_uic_err)
5360 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
5361 __func__, hba->saved_err, hba->saved_uic_err);
5362 }
5363
e8e7f271
SRT		 5364 ufshcd_clear_eh_in_progress(hba);
5365
5366out:
9a47ec7c 5367 spin_unlock_irqrestore(hba->host->host_lock, flags);
38135535 5368 ufshcd_scsi_unblock_requests(hba);
1ab27c9c 5369 ufshcd_release(hba);
62694735 5370 pm_runtime_put_sync(hba->dev);
7a3e97b0
SY		 5371}
5372
5373/**
e8e7f271
SRT		 5374 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5375 * @hba: per-adapter instance
9333d775
VG		 5376 *
5377 * Returns
5378 * IRQ_HANDLED - If interrupt is valid
5379 * IRQ_NONE - If invalid interrupt
7a3e97b0 5380 */
9333d775 5381static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
7a3e97b0
SY		 5382{
5383 u32 reg;
9333d775 5384 irqreturn_t retval = IRQ_NONE;
7a3e97b0 5385
fb7b45f0
DR		 5386 /* PHY layer lane error */
5387 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5388 /* Ignore LINERESET indication, as this is not an error */
5389 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
9333d775 5390 (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)) {
fb7b45f0
DR		 5391 /*
5392 * To know whether this error is fatal or not, DB timeout
5393 * must be checked but this error is handled separately.
5394 */
5395 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", __func__);
48d5b973 5396 ufshcd_update_reg_hist(&hba->ufs_stats.pa_err, reg);
9333d775 5397 retval |= IRQ_HANDLED;
ff8e20c6 5398 }
fb7b45f0 5399
e8e7f271
SRT		 5400 /* PA_INIT_ERROR is fatal and needs UIC reset */
5401 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
9333d775
VG		 5402 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
5403 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
48d5b973 5404 ufshcd_update_reg_hist(&hba->ufs_stats.dl_err, reg);
ff8e20c6 5405
9333d775
VG		 5406 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
5407 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
5408 else if (hba->dev_quirks &
5409 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5410 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
5411 hba->uic_error |=
5412 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5413 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
5414 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
5415 }
5416 retval |= IRQ_HANDLED;
583fa62d 5417 }
e8e7f271
SRT		 5418
5419 /* UIC NL/TL/DME errors needs software retry */
5420 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
9333d775
VG		 5421 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
5422 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
48d5b973 5423 ufshcd_update_reg_hist(&hba->ufs_stats.nl_err, reg);
e8e7f271 5424 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
9333d775 5425 retval |= IRQ_HANDLED;
ff8e20c6 5426 }
e8e7f271
SRT		 5427
5428 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
9333d775
VG		 5429 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
5430 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
48d5b973 5431 ufshcd_update_reg_hist(&hba->ufs_stats.tl_err, reg);
e8e7f271 5432 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
9333d775 5433 retval |= IRQ_HANDLED;
ff8e20c6 5434 }
e8e7f271
SRT		 5435
5436 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
9333d775
VG		 5437 if ((reg & UIC_DME_ERROR) &&
5438 (reg & UIC_DME_ERROR_CODE_MASK)) {
48d5b973 5439 ufshcd_update_reg_hist(&hba->ufs_stats.dme_err, reg);
e8e7f271 5440 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
9333d775 5441 retval |= IRQ_HANDLED;
ff8e20c6 5442 }
e8e7f271
SRT		 5443
5444 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
5445 __func__, hba->uic_error);
9333d775 5446 return retval;
e8e7f271
SRT		 5447}
5448
82174440
SC		 5449static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5450 u32 intr_mask)
5451{
5452 if (!ufshcd_is_auto_hibern8_supported(hba))
5453 return false;
5454
5455 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5456 return false;
5457
5458 if (hba->active_uic_cmd &&
5459 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5460 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5461 return false;
5462
5463 return true;
5464}
5465
e8e7f271
SRT		 5466/**
5467 * ufshcd_check_errors - Check for errors that need s/w attention
5468 * @hba: per-adapter instance
9333d775
VG		 5469 *
5470 * Returns
5471 * IRQ_HANDLED - If interrupt is valid
5472 * IRQ_NONE - If invalid interrupt
e8e7f271 5473 */
9333d775 5474static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba)
e8e7f271
SRT		 5475{
5476 bool queue_eh_work = false;
9333d775 5477 irqreturn_t retval = IRQ_NONE;
e8e7f271 5478
d3c615bf
SC		 5479 if (hba->errors & INT_FATAL_ERRORS) {
5480 ufshcd_update_reg_hist(&hba->ufs_stats.fatal_err, hba->errors);
e8e7f271 5481 queue_eh_work = true;
d3c615bf 5482 }
7a3e97b0
SY		 5483
5484 if (hba->errors & UIC_ERROR) {
e8e7f271 5485 hba->uic_error = 0;
9333d775 5486 retval = ufshcd_update_uic_error(hba);
e8e7f271
SRT		 5487 if (hba->uic_error)
5488 queue_eh_work = true;
7a3e97b0 5489 }
e8e7f271 5490
82174440
SC		 5491 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
5492 dev_err(hba->dev,
5493 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
5494 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
5495 "Enter" : "Exit",
5496 hba->errors, ufshcd_get_upmcrs(hba));
d3c615bf
SC		 5497 ufshcd_update_reg_hist(&hba->ufs_stats.auto_hibern8_err,
5498 hba->errors);
82174440
SC		 5499 queue_eh_work = true;
5500 }
5501
e8e7f271 5502 if (queue_eh_work) {
9a47ec7c
YG		 5503 /*
5504 * update the transfer error masks to sticky bits, let's do this
5505 * irrespective of current ufshcd_state.
5506 */
5507 hba->saved_err |= hba->errors;
5508 hba->saved_uic_err |= hba->uic_error;
5509
e8e7f271
SRT		 5510 /* handle fatal errors only when link is functional */
5511 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
5512 /* block commands from scsi mid-layer */
38135535 5513 ufshcd_scsi_block_requests(hba);
e8e7f271 5514
141f8165 5515 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
66cc820f
DR		 5516
5517 /* dump controller state before resetting */
5518 if (hba->saved_err & (INT_FATAL_ERRORS | UIC_ERROR)) {
5519 bool pr_prdt = !!(hba->saved_err &
5520 SYSTEM_BUS_FATAL_ERROR);
5521
5522 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
5523 __func__, hba->saved_err,
5524 hba->saved_uic_err);
5525
5526 ufshcd_print_host_regs(hba);
5527 ufshcd_print_pwr_info(hba);
5528 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
5529 ufshcd_print_trs(hba, hba->outstanding_reqs,
5530 pr_prdt);
5531 }
e8e7f271
SRT		 5532 schedule_work(&hba->eh_work);
5533 }
9333d775 5534 retval |= IRQ_HANDLED;
3441da7d 5535 }
e8e7f271
SRT		 5536 /*
5537 * if (!queue_eh_work) -
5538 * Other errors are either non-fatal where host recovers
5539 * itself without s/w intervention or errors that will be
5540 * handled by the SCSI core layer.
5541 */
9333d775 5542 return retval;
7a3e97b0
SY		 5543}
5544
69a6c269
BVA		 5545struct ctm_info {
5546 struct ufs_hba *hba;
5547 unsigned long pending;
5548 unsigned int ncpl;
5549};
5550
5551static bool ufshcd_compl_tm(struct request *req, void *priv, bool reserved)
5552{
5553 struct ctm_info *const ci = priv;
5554 struct completion *c;
5555
5556 WARN_ON_ONCE(reserved);
5557 if (test_bit(req->tag, &ci->pending))
5558 return true;
5559 ci->ncpl++;
5560 c = req->end_io_data;
5561 if (c)
5562 complete(c);
5563 return true;
5564}
5565
7a3e97b0
SY		 5566/**
5567 * ufshcd_tmc_handler - handle task management function completion
5568 * @hba: per adapter instance
9333d775
VG		 5569 *
5570 * Returns
5571 * IRQ_HANDLED - If interrupt is valid
5572 * IRQ_NONE - If invalid interrupt
7a3e97b0 5573 */
9333d775 5574static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
7a3e97b0 5575{
69a6c269
BVA		 5576 struct request_queue *q = hba->tmf_queue;
5577 struct ctm_info ci = {
5578 .hba = hba,
5579 .pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL),
5580 };
7a3e97b0 5581
69a6c269
BVA		 5582 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_compl_tm, &ci);
5583 return ci.ncpl ? IRQ_HANDLED : IRQ_NONE;
7a3e97b0
SY		 5584}
5585
5586/**
5587 * ufshcd_sl_intr - Interrupt service routine
5588 * @hba: per adapter instance
5589 * @intr_status: contains interrupts generated by the controller
9333d775
VG		 5590 *
5591 * Returns
5592 * IRQ_HANDLED - If interrupt is valid
5593 * IRQ_NONE - If invalid interrupt
7a3e97b0 5594 */
9333d775 5595static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
7a3e97b0 5596{
9333d775
VG		 5597 irqreturn_t retval = IRQ_NONE;
5598
7a3e97b0 5599 hba->errors = UFSHCD_ERROR_MASK & intr_status;
82174440
SC		 5600
5601 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5602 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5603
7a3e97b0 5604 if (hba->errors)
9333d775 5605 retval |= ufshcd_check_errors(hba);
7a3e97b0 5606
53b3d9c3 5607 if (intr_status & UFSHCD_UIC_MASK)
9333d775 5608 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
7a3e97b0
SY		 5609
5610 if (intr_status & UTP_TASK_REQ_COMPL)
9333d775 5611 retval |= ufshcd_tmc_handler(hba);
7a3e97b0
SY		 5612
5613 if (intr_status & UTP_TRANSFER_REQ_COMPL)
9333d775
VG		 5614 retval |= ufshcd_transfer_req_compl(hba);
5615
5616 return retval;
7a3e97b0
SY		 5617}
5618
5619/**
5620 * ufshcd_intr - Main interrupt service routine
5621 * @irq: irq number
5622 * @__hba: pointer to adapter instance
5623 *
9333d775
VG		 5624 * Returns
5625 * IRQ_HANDLED - If interrupt is valid
5626 * IRQ_NONE - If invalid interrupt
7a3e97b0
SY		 5627 */
5628static irqreturn_t ufshcd_intr(int irq, void *__hba)
5629{
d75f7fe4 5630 u32 intr_status, enabled_intr_status;
7a3e97b0
SY		 5631 irqreturn_t retval = IRQ_NONE;
5632 struct ufs_hba *hba = __hba;
7f6ba4f1 5633 int retries = hba->nutrs;
7a3e97b0
SY		 5634
5635 spin_lock(hba->host->host_lock);
b873a275 5636 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
7a3e97b0 5637
7f6ba4f1
VG		 5638 /*
5639 * There could be max of hba->nutrs reqs in flight and in worst case
5640 * if the reqs get finished 1 by 1 after the interrupt status is
5641 * read, make sure we handle them by checking the interrupt status
5642 * again in a loop until we process all of the reqs before returning.
5643 */
5644 do {
5645 enabled_intr_status =
5646 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
5647 if (intr_status)
5648 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
9333d775
VG		 5649 if (enabled_intr_status)
5650 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
7f6ba4f1
VG		 5651
5652 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
5653 } while (intr_status && --retries);
d75f7fe4 5654
9333d775
VG		 5655 if (retval == IRQ_NONE) {
5656 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
5657 __func__, intr_status);
5658 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
5659 }
5660
7a3e97b0
SY		 5661 spin_unlock(hba->host->host_lock);
5662 return retval;
5663}
5664
e2933132
SRT		 5665static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
5666{
5667 int err = 0;
5668 u32 mask = 1 << tag;
5669 unsigned long flags;
5670
5671 if (!test_bit(tag, &hba->outstanding_tasks))
5672 goto out;
5673
5674 spin_lock_irqsave(hba->host->host_lock, flags);
1399c5b0 5675 ufshcd_utmrl_clear(hba, tag);
e2933132
SRT		 5676 spin_unlock_irqrestore(hba->host->host_lock, flags);
5677
5678 /* poll for max. 1 sec to clear door bell register by h/w */
5679 err = ufshcd_wait_for_register(hba,
5680 REG_UTP_TASK_REQ_DOOR_BELL,
596585a2 5681 mask, 0, 1000, 1000, true);
e2933132
SRT		 5682out:
5683 return err;
5684}
5685
c6049cd9
CH		 5686static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
5687 struct utp_task_req_desc *treq, u8 tm_function)
7a3e97b0 5688{
69a6c269 5689 struct request_queue *q = hba->tmf_queue;
c6049cd9 5690 struct Scsi_Host *host = hba->host;
69a6c269
BVA		 5691 DECLARE_COMPLETION_ONSTACK(wait);
5692 struct request *req;
7a3e97b0 5693 unsigned long flags;
c6049cd9 5694 int free_slot, task_tag, err;
7a3e97b0 5695
e2933132
SRT		 5696 /*
5697 * Get free slot, sleep if slots are unavailable.
5698 * Even though we use wait_event() which sleeps indefinitely,
5699 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5700 */
69a6c269
BVA		 5701 req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
5702 req->end_io_data = &wait;
5703 free_slot = req->tag;
5704 WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);
1ab27c9c 5705 ufshcd_hold(hba, false);
7a3e97b0 5706
e2933132 5707 spin_lock_irqsave(host->host_lock, flags);
e2933132 5708 task_tag = hba->nutrs + free_slot;
7a3e97b0 5709
c6049cd9
CH		 5710 treq->req_header.dword_0 |= cpu_to_be32(task_tag);
5711
5712 memcpy(hba->utmrdl_base_addr + free_slot, treq, sizeof(*treq));
d2877be4
KK		 5713 ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
5714
7a3e97b0
SY		 5715 /* send command to the controller */
5716 __set_bit(free_slot, &hba->outstanding_tasks);
897efe62
YG		 5717
5718 /* Make sure descriptors are ready before ringing the task doorbell */
5719 wmb();
5720
b873a275 5721 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
ad1a1b9c
GB		 5722 /* Make sure that doorbell is committed immediately */
5723 wmb();
7a3e97b0
SY		 5724
5725 spin_unlock_irqrestore(host->host_lock, flags);
5726
6667e6d9
OS		 5727 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_send");
5728
7a3e97b0 5729 /* wait until the task management command is completed */
69a6c269 5730 err = wait_for_completion_io_timeout(&wait,
e2933132 5731 msecs_to_jiffies(TM_CMD_TIMEOUT));
7a3e97b0 5732 if (!err) {
69a6c269
BVA		 5733 /*
5734 * Make sure that ufshcd_compl_tm() does not trigger a
5735 * use-after-free.
5736 */
5737 req->end_io_data = NULL;
6667e6d9 5738 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete_err");
e2933132
SRT		 5739 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
5740 __func__, tm_function);
5741 if (ufshcd_clear_tm_cmd(hba, free_slot))
5742 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
5743 __func__, free_slot);
5744 err = -ETIMEDOUT;
5745 } else {
c6049cd9
CH		 5746 err = 0;
5747 memcpy(treq, hba->utmrdl_base_addr + free_slot, sizeof(*treq));
5748
6667e6d9 5749 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
7a3e97b0 5750 }
e2933132 5751
b557217c
SC		 5752 spin_lock_irqsave(hba->host->host_lock, flags);
5753 __clear_bit(free_slot, &hba->outstanding_tasks);
5754 spin_unlock_irqrestore(hba->host->host_lock, flags);
5755
69a6c269 5756 blk_put_request(req);
e2933132 5757
1ab27c9c 5758 ufshcd_release(hba);
7a3e97b0
SY		 5759 return err;
5760}
5761
c6049cd9
CH		 5762/**
5763 * ufshcd_issue_tm_cmd - issues task management commands to controller
5764 * @hba: per adapter instance
5765 * @lun_id: LUN ID to which TM command is sent
5766 * @task_id: task ID to which the TM command is applicable
5767 * @tm_function: task management function opcode
5768 * @tm_response: task management service response return value
5769 *
5770 * Returns non-zero value on error, zero on success.
5771 */
5772static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
5773 u8 tm_function, u8 *tm_response)
5774{
5775 struct utp_task_req_desc treq = { { 0 }, };
5776 int ocs_value, err;
5777
5778 /* Configure task request descriptor */
5779 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
5780 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
5781
5782 /* Configure task request UPIU */
5783 treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
5784 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
5785 treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
5786
5787 /*
5788 * The host shall provide the same value for LUN field in the basic
5789 * header and for Input Parameter.
5790 */
5791 treq.input_param1 = cpu_to_be32(lun_id);
5792 treq.input_param2 = cpu_to_be32(task_id);
5793
5794 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
5795 if (err == -ETIMEDOUT)
5796 return err;
5797
5798 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
5799 if (ocs_value != OCS_SUCCESS)
5800 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
5801 __func__, ocs_value);
5802 else if (tm_response)
5803 *tm_response = be32_to_cpu(treq.output_param1) &
5804 MASK_TM_SERVICE_RESP;
5805 return err;
5806}
5807
5e0a86ee
AA		 5808/**
5809 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
5810 * @hba: per-adapter instance
5811 * @req_upiu: upiu request
5812 * @rsp_upiu: upiu reply
5e0a86ee
AA		 5813 * @desc_buff: pointer to descriptor buffer, NULL if NA
5814 * @buff_len: descriptor size, 0 if NA
d0e9760d 5815 * @cmd_type: specifies the type (NOP, Query...)
5e0a86ee
AA		 5816 * @desc_op: descriptor operation
5817 *
5818 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
5819 * Therefore, it "rides" the device management infrastructure: uses its tag and
5820 * tasks work queues.
5821 *
5822 * Since there is only one available tag for device management commands,
5823 * the caller is expected to hold the hba->dev_cmd.lock mutex.
5824 */
5825static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
5826 struct utp_upiu_req *req_upiu,
5827 struct utp_upiu_req *rsp_upiu,
5828 u8 *desc_buff, int *buff_len,
7f674c38 5829 enum dev_cmd_type cmd_type,
5e0a86ee
AA		 5830 enum query_opcode desc_op)
5831{
7252a360
BVA		 5832 struct request_queue *q = hba->cmd_queue;
5833 struct request *req;
5e0a86ee
AA		 5834 struct ufshcd_lrb *lrbp;
5835 int err = 0;
5836 int tag;
5837 struct completion wait;
5838 unsigned long flags;
5839 u32 upiu_flags;
5840
5841 down_read(&hba->clk_scaling_lock);
5842
7252a360
BVA		 5843 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
5844 if (IS_ERR(req))
5845 return PTR_ERR(req);
5846 tag = req->tag;
5847 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5e0a86ee
AA		 5848
5849 init_completion(&wait);
5850 lrbp = &hba->lrb[tag];
5851 WARN_ON(lrbp->cmd);
5852
5853 lrbp->cmd = NULL;
5854 lrbp->sense_bufflen = 0;
5855 lrbp->sense_buffer = NULL;
5856 lrbp->task_tag = tag;
5857 lrbp->lun = 0;
5858 lrbp->intr_cmd = true;
5859 hba->dev_cmd.type = cmd_type;
5860
5861 switch (hba->ufs_version) {
5862 case UFSHCI_VERSION_10:
5863 case UFSHCI_VERSION_11:
5864 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
5865 break;
5866 default:
5867 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
5868 break;
5869 }
5870
5871 /* update the task tag in the request upiu */
5872 req_upiu->header.dword_0 |= cpu_to_be32(tag);
5873
5874 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
5875
5876 /* just copy the upiu request as it is */
5877 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
5878 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
5879 /* The Data Segment Area is optional depending upon the query
5880 * function value. for WRITE DESCRIPTOR, the data segment
5881 * follows right after the tsf.
5882 */
5883 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
5884 *buff_len = 0;
5885 }
5886
5887 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5888
5889 hba->dev_cmd.complete = &wait;
5890
5891 /* Make sure descriptors are ready before ringing the doorbell */
5892 wmb();
5893 spin_lock_irqsave(hba->host->host_lock, flags);
5894 ufshcd_send_command(hba, tag);
5895 spin_unlock_irqrestore(hba->host->host_lock, flags);
5896
5897 /*
5898 * ignore the returning value here - ufshcd_check_query_response is
5899 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
5900 * read the response directly ignoring all errors.
5901 */
5902 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
5903
5904 /* just copy the upiu response as it is */
5905 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
4bbbe242
AA		 5906 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
5907 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
5908 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
5909 MASK_QUERY_DATA_SEG_LEN;
5910
5911 if (*buff_len >= resp_len) {
5912 memcpy(desc_buff, descp, resp_len);
5913 *buff_len = resp_len;
5914 } else {
3d4881d1
BH		 5915 dev_warn(hba->dev,
5916 "%s: rsp size %d is bigger than buffer size %d",
5917 __func__, resp_len, *buff_len);
4bbbe242
AA		 5918 *buff_len = 0;
5919 err = -EINVAL;
5920 }
5921 }
5e0a86ee 5922
7252a360 5923 blk_put_request(req);
5e0a86ee
AA		 5924 up_read(&hba->clk_scaling_lock);
5925 return err;
5926}
5927
5928/**
5929 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
5930 * @hba: per-adapter instance
5931 * @req_upiu: upiu request
5932 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
5933 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
5934 * @desc_buff: pointer to descriptor buffer, NULL if NA
5935 * @buff_len: descriptor size, 0 if NA
5936 * @desc_op: descriptor operation
5937 *
5938 * Supports UTP Transfer requests (nop and query), and UTP Task
5939 * Management requests.
5940 * It is up to the caller to fill the upiu conent properly, as it will
5941 * be copied without any further input validations.
5942 */
5943int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
5944 struct utp_upiu_req *req_upiu,
5945 struct utp_upiu_req *rsp_upiu,
5946 int msgcode,
5947 u8 *desc_buff, int *buff_len,
5948 enum query_opcode desc_op)
5949{
5950 int err;
7f674c38 5951 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
5e0a86ee
AA		 5952 struct utp_task_req_desc treq = { { 0 }, };
5953 int ocs_value;
5954 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
5955
5e0a86ee
AA		 5956 switch (msgcode) {
5957 case UPIU_TRANSACTION_NOP_OUT:
5958 cmd_type = DEV_CMD_TYPE_NOP;
5959 /* fall through */
5960 case UPIU_TRANSACTION_QUERY_REQ:
5961 ufshcd_hold(hba, false);
5962 mutex_lock(&hba->dev_cmd.lock);
5963 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
5964 desc_buff, buff_len,
5965 cmd_type, desc_op);
5966 mutex_unlock(&hba->dev_cmd.lock);
5967 ufshcd_release(hba);
5968
5969 break;
5970 case UPIU_TRANSACTION_TASK_REQ:
5971 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
5972 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
5973
5974 memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
5975
5976 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
5977 if (err == -ETIMEDOUT)
5978 break;
5979
5980 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
5981 if (ocs_value != OCS_SUCCESS) {
5982 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
5983 ocs_value);
5984 break;
5985 }
5986
5987 memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
5988
5989 break;
5990 default:
5991 err = -EINVAL;
5992
5993 break;
5994 }
5995
5e0a86ee
AA		 5996 return err;
5997}
5998
7a3e97b0 5999/**
3441da7d
SRT		 6000 * ufshcd_eh_device_reset_handler - device reset handler registered to
6001 * scsi layer.
7a3e97b0
SY		 6002 * @cmd: SCSI command pointer
6003 *
6004 * Returns SUCCESS/FAILED
6005 */
3441da7d 6006static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7a3e97b0
SY		 6007{
6008 struct Scsi_Host *host;
6009 struct ufs_hba *hba;
6010 unsigned int tag;
6011 u32 pos;
6012 int err;
e2933132
SRT		 6013 u8 resp = 0xF;
6014 struct ufshcd_lrb *lrbp;
3441da7d 6015 unsigned long flags;
7a3e97b0
SY		 6016
6017 host = cmd->device->host;
6018 hba = shost_priv(host);
6019 tag = cmd->request->tag;
6020
e2933132
SRT		 6021 lrbp = &hba->lrb[tag];
6022 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
6023 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
3441da7d
SRT		 6024 if (!err)
6025 err = resp;
7a3e97b0 6026 goto out;
e2933132 6027 }
7a3e97b0 6028
3441da7d
SRT		 6029 /* clear the commands that were pending for corresponding LUN */
6030 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
6031 if (hba->lrb[pos].lun == lrbp->lun) {
6032 err = ufshcd_clear_cmd(hba, pos);
6033 if (err)
6034 break;
7a3e97b0 6035 }
3441da7d
SRT		 6036 }
6037 spin_lock_irqsave(host->host_lock, flags);
6038 ufshcd_transfer_req_compl(hba);
6039 spin_unlock_irqrestore(host->host_lock, flags);
7fabb77b 6040
7a3e97b0 6041out:
7fabb77b 6042 hba->req_abort_count = 0;
8808b4e9 6043 ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, (u32)err);
3441da7d
SRT		 6044 if (!err) {
6045 err = SUCCESS;
6046 } else {
6047 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6048 err = FAILED;
6049 }
7a3e97b0
SY		 6050 return err;
6051}
6052
e0b299e3
GB		 6053static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
6054{
6055 struct ufshcd_lrb *lrbp;
6056 int tag;
6057
6058 for_each_set_bit(tag, &bitmap, hba->nutrs) {
6059 lrbp = &hba->lrb[tag];
6060 lrbp->req_abort_skip = true;
6061 }
6062}
6063
7a3e97b0
SY		 6064/**
6065 * ufshcd_abort - abort a specific command
6066 * @cmd: SCSI command pointer
6067 *
f20810d8
SRT		 6068 * Abort the pending command in device by sending UFS_ABORT_TASK task management
6069 * command, and in host controller by clearing the door-bell register. There can
6070 * be race between controller sending the command to the device while abort is
6071 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6072 * really issued and then try to abort it.
6073 *
7a3e97b0
SY		 6074 * Returns SUCCESS/FAILED
6075 */
6076static int ufshcd_abort(struct scsi_cmnd *cmd)
6077{
6078 struct Scsi_Host *host;
6079 struct ufs_hba *hba;
6080 unsigned long flags;
6081 unsigned int tag;
f20810d8
SRT		 6082 int err = 0;
6083 int poll_cnt;
e2933132
SRT		 6084 u8 resp = 0xF;
6085 struct ufshcd_lrb *lrbp;
e9d501b1 6086 u32 reg;
7a3e97b0
SY		 6087
6088 host = cmd->device->host;
6089 hba = shost_priv(host);
6090 tag = cmd->request->tag;
e7d38257 6091 lrbp = &hba->lrb[tag];
14497328
YG		 6092 if (!ufshcd_valid_tag(hba, tag)) {
6093 dev_err(hba->dev,
6094 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
6095 __func__, tag, cmd, cmd->request);
6096 BUG();
6097 }
7a3e97b0 6098
e7d38257
DR		 6099 /*
6100 * Task abort to the device W-LUN is illegal. When this command
6101 * will fail, due to spec violation, scsi err handling next step
6102 * will be to send LU reset which, again, is a spec violation.
6103 * To avoid these unnecessary/illegal step we skip to the last error
6104 * handling stage: reset and restore.
6105 */
6106 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN)
6107 return ufshcd_eh_host_reset_handler(cmd);
6108
1ab27c9c 6109 ufshcd_hold(hba, false);
14497328 6110 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
f20810d8 6111 /* If command is already aborted/completed, return SUCCESS */
14497328
YG		 6112 if (!(test_bit(tag, &hba->outstanding_reqs))) {
6113 dev_err(hba->dev,
6114 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
6115 __func__, tag, hba->outstanding_reqs, reg);
f20810d8 6116 goto out;
14497328 6117 }
7a3e97b0 6118
e9d501b1
DR		 6119 if (!(reg & (1 << tag))) {
6120 dev_err(hba->dev,
6121 "%s: cmd was completed, but without a notifying intr, tag = %d",
6122 __func__, tag);
6123 }
6124
66cc820f
DR		 6125 /* Print Transfer Request of aborted task */
6126 dev_err(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
66cc820f 6127
7fabb77b
GB		 6128 /*
6129 * Print detailed info about aborted request.
6130 * As more than one request might get aborted at the same time,
6131 * print full information only for the first aborted request in order
6132 * to reduce repeated printouts. For other aborted requests only print
6133 * basic details.
6134 */
6135 scsi_print_command(hba->lrb[tag].cmd);
6136 if (!hba->req_abort_count) {
8808b4e9 6137 ufshcd_update_reg_hist(&hba->ufs_stats.task_abort, 0);
7fabb77b 6138 ufshcd_print_host_regs(hba);
6ba65588 6139 ufshcd_print_host_state(hba);
7fabb77b
GB		 6140 ufshcd_print_pwr_info(hba);
6141 ufshcd_print_trs(hba, 1 << tag, true);
6142 } else {
6143 ufshcd_print_trs(hba, 1 << tag, false);
6144 }
6145 hba->req_abort_count++;
e0b299e3
GB		 6146
6147 /* Skip task abort in case previous aborts failed and report failure */
6148 if (lrbp->req_abort_skip) {
6149 err = -EIO;
6150 goto out;
6151 }
6152
f20810d8
SRT		 6153 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6154 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6155 UFS_QUERY_TASK, &resp);
6156 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6157 /* cmd pending in the device */
ff8e20c6
DR		 6158 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6159 __func__, tag);
f20810d8
SRT		 6160 break;
6161 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
f20810d8
SRT		 6162 /*
6163 * cmd not pending in the device, check if it is
6164 * in transition.
6165 */
ff8e20c6
DR		 6166 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6167 __func__, tag);
f20810d8
SRT		 6168 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6169 if (reg & (1 << tag)) {
6170 /* sleep for max. 200us to stabilize */
6171 usleep_range(100, 200);
6172 continue;
6173 }
6174 /* command completed already */
ff8e20c6
DR		 6175 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6176 __func__, tag);
f20810d8
SRT		 6177 goto out;
6178 } else {
ff8e20c6
DR		 6179 dev_err(hba->dev,
6180 "%s: no response from device. tag = %d, err %d\n",
6181 __func__, tag, err);
f20810d8
SRT		 6182 if (!err)
6183 err = resp; /* service response error */
6184 goto out;
6185 }
6186 }
6187
6188 if (!poll_cnt) {
6189 err = -EBUSY;
7a3e97b0
SY		 6190 goto out;
6191 }
7a3e97b0 6192
e2933132
SRT		 6193 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6194 UFS_ABORT_TASK, &resp);
6195 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
ff8e20c6 6196 if (!err) {
f20810d8 6197 err = resp; /* service response error */
ff8e20c6
DR		 6198 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6199 __func__, tag, err);
6200 }
7a3e97b0 6201 goto out;
e2933132 6202 }
7a3e97b0 6203
f20810d8 6204 err = ufshcd_clear_cmd(hba, tag);
ff8e20c6
DR		 6205 if (err) {
6206 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6207 __func__, tag, err);
f20810d8 6208 goto out;
ff8e20c6 6209 }
f20810d8 6210
7a3e97b0
SY		 6211 scsi_dma_unmap(cmd);
6212
6213 spin_lock_irqsave(host->host_lock, flags);
a48353f6 6214 ufshcd_outstanding_req_clear(hba, tag);
7a3e97b0
SY		 6215 hba->lrb[tag].cmd = NULL;
6216 spin_unlock_irqrestore(host->host_lock, flags);
5a0b0cb9 6217
7a3e97b0 6218out:
f20810d8
SRT		 6219 if (!err) {
6220 err = SUCCESS;
6221 } else {
6222 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
e0b299e3 6223 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
f20810d8
SRT		 6224 err = FAILED;
6225 }
6226
1ab27c9c
ST		 6227 /*
6228 * This ufshcd_release() corresponds to the original scsi cmd that got
6229 * aborted here (as we won't get any IRQ for it).
6230 */
6231 ufshcd_release(hba);
7a3e97b0
SY		 6232 return err;
6233}
6234
3441da7d
SRT		 6235/**
6236 * ufshcd_host_reset_and_restore - reset and restore host controller
6237 * @hba: per-adapter instance
6238 *
6239 * Note that host controller reset may issue DME_RESET to
6240 * local and remote (device) Uni-Pro stack and the attributes
6241 * are reset to default state.
6242 *
6243 * Returns zero on success, non-zero on failure
6244 */
6245static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
6246{
6247 int err;
3441da7d
SRT		 6248 unsigned long flags;
6249
2df74b69
CG		 6250 /*
6251 * Stop the host controller and complete the requests
6252 * cleared by h/w
6253 */
3441da7d 6254 spin_lock_irqsave(hba->host->host_lock, flags);
596585a2 6255 ufshcd_hba_stop(hba, false);
2df74b69
CG		 6256 hba->silence_err_logs = true;
6257 ufshcd_complete_requests(hba);
6258 hba->silence_err_logs = false;
3441da7d
SRT		 6259 spin_unlock_irqrestore(hba->host->host_lock, flags);
6260
a3cd5ec5
SJ		 6261 /* scale up clocks to max frequency before full reinitialization */
6262 ufshcd_scale_clks(hba, true);
6263
3441da7d
SRT		 6264 err = ufshcd_hba_enable(hba);
6265 if (err)
6266 goto out;
6267
6268 /* Establish the link again and restore the device */
1d337ec2
SRT		 6269 err = ufshcd_probe_hba(hba);
6270
6271 if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL))
3441da7d
SRT		 6272 err = -EIO;
6273out:
6274 if (err)
6275 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
8808b4e9 6276 ufshcd_update_reg_hist(&hba->ufs_stats.host_reset, (u32)err);
3441da7d
SRT		 6277 return err;
6278}
6279
6280/**
6281 * ufshcd_reset_and_restore - reset and re-initialize host/device
6282 * @hba: per-adapter instance
6283 *
6284 * Reset and recover device, host and re-establish link. This
6285 * is helpful to recover the communication in fatal error conditions.
6286 *
6287 * Returns zero on success, non-zero on failure
6288 */
6289static int ufshcd_reset_and_restore(struct ufs_hba *hba)
6290{
6291 int err = 0;
1d337ec2 6292 int retries = MAX_HOST_RESET_RETRIES;
3441da7d 6293
1d337ec2 6294 do {
d8d9f793
BA		 6295 /* Reset the attached device */
6296 ufshcd_vops_device_reset(hba);
6297
1d337ec2
SRT		 6298 err = ufshcd_host_reset_and_restore(hba);
6299 } while (err && --retries);
3441da7d 6300
3441da7d
SRT		 6301 return err;
6302}
6303
6304/**
6305 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
8aa29f19 6306 * @cmd: SCSI command pointer
3441da7d
SRT		 6307 *
6308 * Returns SUCCESS/FAILED
6309 */
6310static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
6311{
6312 int err;
6313 unsigned long flags;
6314 struct ufs_hba *hba;
6315
6316 hba = shost_priv(cmd->device->host);
6317
1ab27c9c 6318 ufshcd_hold(hba, false);
3441da7d
SRT		 6319 /*
6320 * Check if there is any race with fatal error handling.
6321 * If so, wait for it to complete. Even though fatal error
6322 * handling does reset and restore in some cases, don't assume
6323 * anything out of it. We are just avoiding race here.
6324 */
6325 do {
6326 spin_lock_irqsave(hba->host->host_lock, flags);
e8e7f271 6327 if (!(work_pending(&hba->eh_work) ||
8dc0da79
ZL		 6328 hba->ufshcd_state == UFSHCD_STATE_RESET ||
6329 hba->ufshcd_state == UFSHCD_STATE_EH_SCHEDULED))
3441da7d
SRT		 6330 break;
6331 spin_unlock_irqrestore(hba->host->host_lock, flags);
6332 dev_dbg(hba->dev, "%s: reset in progress\n", __func__);
e8e7f271 6333 flush_work(&hba->eh_work);
3441da7d
SRT		 6334 } while (1);
6335
6336 hba->ufshcd_state = UFSHCD_STATE_RESET;
6337 ufshcd_set_eh_in_progress(hba);
6338 spin_unlock_irqrestore(hba->host->host_lock, flags);
6339
6340 err = ufshcd_reset_and_restore(hba);
6341
6342 spin_lock_irqsave(hba->host->host_lock, flags);
6343 if (!err) {
6344 err = SUCCESS;
6345 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6346 } else {
6347 err = FAILED;
6348 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6349 }
6350 ufshcd_clear_eh_in_progress(hba);
6351 spin_unlock_irqrestore(hba->host->host_lock, flags);
6352
1ab27c9c 6353 ufshcd_release(hba);
3441da7d
SRT		 6354 return err;
6355}
6356
3a4bf06d
YG		 6357/**
6358 * ufshcd_get_max_icc_level - calculate the ICC level
6359 * @sup_curr_uA: max. current supported by the regulator
6360 * @start_scan: row at the desc table to start scan from
6361 * @buff: power descriptor buffer
6362 *
6363 * Returns calculated max ICC level for specific regulator
6364 */
6365static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
6366{
6367 int i;
6368 int curr_uA;
6369 u16 data;
6370 u16 unit;
6371
6372 for (i = start_scan; i >= 0; i--) {
d79713f9 6373 data = be16_to_cpup((__be16 *)&buff[2 * i]);
3a4bf06d
YG		 6374 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
6375 ATTR_ICC_LVL_UNIT_OFFSET;
6376 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
6377 switch (unit) {
6378 case UFSHCD_NANO_AMP:
6379 curr_uA = curr_uA / 1000;
6380 break;
6381 case UFSHCD_MILI_AMP:
6382 curr_uA = curr_uA * 1000;
6383 break;
6384 case UFSHCD_AMP:
6385 curr_uA = curr_uA * 1000 * 1000;
6386 break;
6387 case UFSHCD_MICRO_AMP:
6388 default:
6389 break;
6390 }
6391 if (sup_curr_uA >= curr_uA)
6392 break;
6393 }
6394 if (i < 0) {
6395 i = 0;
6396 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
6397 }
6398
6399 return (u32)i;
6400}
6401
6402/**
6403 * ufshcd_calc_icc_level - calculate the max ICC level
6404 * In case regulators are not initialized we'll return 0
6405 * @hba: per-adapter instance
6406 * @desc_buf: power descriptor buffer to extract ICC levels from.
6407 * @len: length of desc_buff
6408 *
6409 * Returns calculated ICC level
6410 */
6411static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
6412 u8 *desc_buf, int len)
6413{
6414 u32 icc_level = 0;
6415
6416 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
6417 !hba->vreg_info.vccq2) {
6418 dev_err(hba->dev,
6419 "%s: Regulator capability was not set, actvIccLevel=%d",
6420 __func__, icc_level);
6421 goto out;
6422 }
6423
0487fff7 6424 if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
3a4bf06d
YG		 6425 icc_level = ufshcd_get_max_icc_level(
6426 hba->vreg_info.vcc->max_uA,
6427 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
6428 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
6429
0487fff7 6430 if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
3a4bf06d
YG		 6431 icc_level = ufshcd_get_max_icc_level(
6432 hba->vreg_info.vccq->max_uA,
6433 icc_level,
6434 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
6435
0487fff7 6436 if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
3a4bf06d
YG		 6437 icc_level = ufshcd_get_max_icc_level(
6438 hba->vreg_info.vccq2->max_uA,
6439 icc_level,
6440 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
6441out:
6442 return icc_level;
6443}
6444
6445static void ufshcd_init_icc_levels(struct ufs_hba *hba)
6446{
6447 int ret;
a4b0e8a4 6448 int buff_len = hba->desc_size.pwr_desc;
bbe21d7a
KC		 6449 u8 *desc_buf;
6450
6451 desc_buf = kmalloc(buff_len, GFP_KERNEL);
6452 if (!desc_buf)
6453 return;
3a4bf06d
YG		 6454
6455 ret = ufshcd_read_power_desc(hba, desc_buf, buff_len);
6456 if (ret) {
6457 dev_err(hba->dev,
6458 "%s: Failed reading power descriptor.len = %d ret = %d",
6459 __func__, buff_len, ret);
bbe21d7a 6460 goto out;
3a4bf06d
YG		 6461 }
6462
6463 hba->init_prefetch_data.icc_level =
6464 ufshcd_find_max_sup_active_icc_level(hba,
6465 desc_buf, buff_len);
6466 dev_dbg(hba->dev, "%s: setting icc_level 0x%x",
6467 __func__, hba->init_prefetch_data.icc_level);
6468
dbd34a61
SM		 6469 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
6470 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0,
6471 &hba->init_prefetch_data.icc_level);
3a4bf06d
YG		 6472
6473 if (ret)
6474 dev_err(hba->dev,
6475 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
6476 __func__, hba->init_prefetch_data.icc_level , ret);
6477
bbe21d7a
KC		 6478out:
6479 kfree(desc_buf);
3a4bf06d
YG		 6480}
6481
2a8fa600
SJ		 6482/**
6483 * ufshcd_scsi_add_wlus - Adds required W-LUs
6484 * @hba: per-adapter instance
6485 *
6486 * UFS device specification requires the UFS devices to support 4 well known
6487 * logical units:
6488 * "REPORT_LUNS" (address: 01h)
6489 * "UFS Device" (address: 50h)
6490 * "RPMB" (address: 44h)
6491 * "BOOT" (address: 30h)
6492 * UFS device's power management needs to be controlled by "POWER CONDITION"
6493 * field of SSU (START STOP UNIT) command. But this "power condition" field
6494 * will take effect only when its sent to "UFS device" well known logical unit
6495 * hence we require the scsi_device instance to represent this logical unit in
6496 * order for the UFS host driver to send the SSU command for power management.
8aa29f19 6497 *
2a8fa600
SJ		 6498 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
6499 * Block) LU so user space process can control this LU. User space may also
6500 * want to have access to BOOT LU.
8aa29f19 6501 *
2a8fa600
SJ		 6502 * This function adds scsi device instances for each of all well known LUs
6503 * (except "REPORT LUNS" LU).
6504 *
6505 * Returns zero on success (all required W-LUs are added successfully),
6506 * non-zero error value on failure (if failed to add any of the required W-LU).
6507 */
6508static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
6509{
6510 int ret = 0;
7c48bfd0
AM		 6511 struct scsi_device *sdev_rpmb;
6512 struct scsi_device *sdev_boot;
2a8fa600
SJ		 6513
6514 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
6515 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
6516 if (IS_ERR(hba->sdev_ufs_device)) {
6517 ret = PTR_ERR(hba->sdev_ufs_device);
6518 hba->sdev_ufs_device = NULL;
6519 goto out;
6520 }
7c48bfd0 6521 scsi_device_put(hba->sdev_ufs_device);
2a8fa600 6522
7c48bfd0 6523 sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
2a8fa600 6524 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7c48bfd0
AM		 6525 if (IS_ERR(sdev_rpmb)) {
6526 ret = PTR_ERR(sdev_rpmb);
3d21fbde 6527 goto remove_sdev_ufs_device;
2a8fa600 6528 }
7c48bfd0 6529 scsi_device_put(sdev_rpmb);
3d21fbde
HK		 6530
6531 sdev_boot = __scsi_add_device(hba->host, 0, 0,
6532 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
6533 if (IS_ERR(sdev_boot))
6534 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
6535 else
6536 scsi_device_put(sdev_boot);
2a8fa600
SJ		 6537 goto out;
6538
2a8fa600
SJ		 6539remove_sdev_ufs_device:
6540 scsi_remove_device(hba->sdev_ufs_device);
6541out:
6542 return ret;
6543}
6544
93fdd5ac
TW		 6545static int ufs_get_device_desc(struct ufs_hba *hba,
6546 struct ufs_dev_desc *dev_desc)
c58ab7aa
YG		 6547{
6548 int err;
bbe21d7a 6549 size_t buff_len;
c58ab7aa 6550 u8 model_index;
bbe21d7a
KC		 6551 u8 *desc_buf;
6552
4b828fe1
TW		 6553 if (!dev_desc)
6554 return -EINVAL;
6555
bbe21d7a
KC		 6556 buff_len = max_t(size_t, hba->desc_size.dev_desc,
6557 QUERY_DESC_MAX_SIZE + 1);
6558 desc_buf = kmalloc(buff_len, GFP_KERNEL);
6559 if (!desc_buf) {
6560 err = -ENOMEM;
6561 goto out;
6562 }
c58ab7aa 6563
a4b0e8a4 6564 err = ufshcd_read_device_desc(hba, desc_buf, hba->desc_size.dev_desc);
c58ab7aa
YG		 6565 if (err) {
6566 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
6567 __func__, err);
6568 goto out;
6569 }
6570
6571 /*
6572 * getting vendor (manufacturerID) and Bank Index in big endian
6573 * format
6574 */
93fdd5ac 6575 dev_desc->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
c58ab7aa
YG		 6576 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
6577
6578 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
4b828fe1
TW		 6579 err = ufshcd_read_string_desc(hba, model_index,
6580 &dev_desc->model, SD_ASCII_STD);
6581 if (err < 0) {
c58ab7aa
YG		 6582 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
6583 __func__, err);
6584 goto out;
6585 }
6586
4b828fe1
TW		 6587 /*
6588 * ufshcd_read_string_desc returns size of the string
6589 * reset the error value
6590 */
6591 err = 0;
c58ab7aa
YG		 6592
6593out:
bbe21d7a 6594 kfree(desc_buf);
c58ab7aa
YG		 6595 return err;
6596}
6597
4b828fe1
TW		 6598static void ufs_put_device_desc(struct ufs_dev_desc *dev_desc)
6599{
6600 kfree(dev_desc->model);
6601 dev_desc->model = NULL;
6602}
6603
93fdd5ac
TW		 6604static void ufs_fixup_device_setup(struct ufs_hba *hba,
6605 struct ufs_dev_desc *dev_desc)
c58ab7aa 6606{
c58ab7aa 6607 struct ufs_dev_fix *f;
c58ab7aa
YG		 6608
6609 for (f = ufs_fixups; f->quirk; f++) {
93fdd5ac
TW		 6610 if ((f->card.wmanufacturerid == dev_desc->wmanufacturerid ||
6611 f->card.wmanufacturerid == UFS_ANY_VENDOR) &&
4b828fe1
TW		 6612 ((dev_desc->model &&
6613 STR_PRFX_EQUAL(f->card.model, dev_desc->model)) ||
6614 !strcmp(f->card.model, UFS_ANY_MODEL)))
c58ab7aa
YG		 6615 hba->dev_quirks |= f->quirk;
6616 }
6617}
6618
37113106
YG		 6619/**
6620 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6621 * @hba: per-adapter instance
6622 *
6623 * PA_TActivate parameter can be tuned manually if UniPro version is less than
6624 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6625 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6626 * the hibern8 exit latency.
6627 *
6628 * Returns zero on success, non-zero error value on failure.
6629 */
6630static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
6631{
6632 int ret = 0;
6633 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
6634
6635 ret = ufshcd_dme_peer_get(hba,
6636 UIC_ARG_MIB_SEL(
6637 RX_MIN_ACTIVATETIME_CAPABILITY,
6638 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6639 &peer_rx_min_activatetime);
6640 if (ret)
6641 goto out;
6642
6643 /* make sure proper unit conversion is applied */
6644 tuned_pa_tactivate =
6645 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
6646 / PA_TACTIVATE_TIME_UNIT_US);
6647 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6648 tuned_pa_tactivate);
6649
6650out:
6651 return ret;
6652}
6653
6654/**
6655 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6656 * @hba: per-adapter instance
6657 *
6658 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6659 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6660 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6661 * This optimal value can help reduce the hibern8 exit latency.
6662 *
6663 * Returns zero on success, non-zero error value on failure.
6664 */
6665static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
6666{
6667 int ret = 0;
6668 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
6669 u32 max_hibern8_time, tuned_pa_hibern8time;
6670
6671 ret = ufshcd_dme_get(hba,
6672 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
6673 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
6674 &local_tx_hibern8_time_cap);
6675 if (ret)
6676 goto out;
6677
6678 ret = ufshcd_dme_peer_get(hba,
6679 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
6680 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6681 &peer_rx_hibern8_time_cap);
6682 if (ret)
6683 goto out;
6684
6685 max_hibern8_time = max(local_tx_hibern8_time_cap,
6686 peer_rx_hibern8_time_cap);
6687 /* make sure proper unit conversion is applied */
6688 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
6689 / PA_HIBERN8_TIME_UNIT_US);
6690 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
6691 tuned_pa_hibern8time);
6692out:
6693 return ret;
6694}
6695
c6a6db43
SJ		 6696/**
6697 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6698 * less than device PA_TACTIVATE time.
6699 * @hba: per-adapter instance
6700 *
6701 * Some UFS devices require host PA_TACTIVATE to be lower than device
6702 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6703 * for such devices.
6704 *
6705 * Returns zero on success, non-zero error value on failure.
6706 */
6707static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
6708{
6709 int ret = 0;
6710 u32 granularity, peer_granularity;
6711 u32 pa_tactivate, peer_pa_tactivate;
6712 u32 pa_tactivate_us, peer_pa_tactivate_us;
6713 u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
6714
6715 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6716 &granularity);
6717 if (ret)
6718 goto out;
6719
6720 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6721 &peer_granularity);
6722 if (ret)
6723 goto out;
6724
6725 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
6726 (granularity > PA_GRANULARITY_MAX_VAL)) {
6727 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
6728 __func__, granularity);
6729 return -EINVAL;
6730 }
6731
6732 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
6733 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
6734 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
6735 __func__, peer_granularity);
6736 return -EINVAL;
6737 }
6738
6739 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
6740 if (ret)
6741 goto out;
6742
6743 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
6744 &peer_pa_tactivate);
6745 if (ret)
6746 goto out;
6747
6748 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
6749 peer_pa_tactivate_us = peer_pa_tactivate *
6750 gran_to_us_table[peer_granularity - 1];
6751
6752 if (pa_tactivate_us > peer_pa_tactivate_us) {
6753 u32 new_peer_pa_tactivate;
6754
6755 new_peer_pa_tactivate = pa_tactivate_us /
6756 gran_to_us_table[peer_granularity - 1];
6757 new_peer_pa_tactivate++;
6758 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6759 new_peer_pa_tactivate);
6760 }
6761
6762out:
6763 return ret;
6764}
6765
37113106
YG		 6766static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
6767{
6768 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
6769 ufshcd_tune_pa_tactivate(hba);
6770 ufshcd_tune_pa_hibern8time(hba);
6771 }
6772
6773 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
6774 /* set 1ms timeout for PA_TACTIVATE */
6775 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
c6a6db43
SJ		 6776
6777 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
6778 ufshcd_quirk_tune_host_pa_tactivate(hba);
56d4a186
SJ		 6779
6780 ufshcd_vops_apply_dev_quirks(hba);
37113106
YG		 6781}
6782
ff8e20c6
DR		 6783static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
6784{
ff8e20c6
DR		 6785 hba->ufs_stats.hibern8_exit_cnt = 0;
6786 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7fabb77b 6787 hba->req_abort_count = 0;
ff8e20c6
DR		 6788}
6789
a4b0e8a4
PM		 6790static void ufshcd_init_desc_sizes(struct ufs_hba *hba)
6791{
6792 int err;
6793
6794 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0,
6795 &hba->desc_size.dev_desc);
6796 if (err)
6797 hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
6798
6799 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0,
6800 &hba->desc_size.pwr_desc);
6801 if (err)
6802 hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
6803
6804 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0,
6805 &hba->desc_size.interc_desc);
6806 if (err)
6807 hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
6808
6809 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0,
6810 &hba->desc_size.conf_desc);
6811 if (err)
6812 hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
6813
6814 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0,
6815 &hba->desc_size.unit_desc);
6816 if (err)
6817 hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
6818
6819 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0,
6820 &hba->desc_size.geom_desc);
6821 if (err)
6822 hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
059efd84 6823
c648c2d2
SN		 6824 err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0,
6825 &hba->desc_size.hlth_desc);
6826 if (err)
6827 hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
a4b0e8a4
PM		 6828}
6829
9e1e8a75
SJ		 6830static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
6831 {19200000, REF_CLK_FREQ_19_2_MHZ},
6832 {26000000, REF_CLK_FREQ_26_MHZ},
6833 {38400000, REF_CLK_FREQ_38_4_MHZ},
6834 {52000000, REF_CLK_FREQ_52_MHZ},
6835 {0, REF_CLK_FREQ_INVAL},
6836};
6837
6838static enum ufs_ref_clk_freq
6839ufs_get_bref_clk_from_hz(unsigned long freq)
6840{
6841 int i;
6842
6843 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
6844 if (ufs_ref_clk_freqs[i].freq_hz == freq)
6845 return ufs_ref_clk_freqs[i].val;
6846
6847 return REF_CLK_FREQ_INVAL;
6848}
6849
6850void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
6851{
6852 unsigned long freq;
6853
6854 freq = clk_get_rate(refclk);
6855
6856 hba->dev_ref_clk_freq =
6857 ufs_get_bref_clk_from_hz(freq);
6858
6859 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
6860 dev_err(hba->dev,
6861 "invalid ref_clk setting = %ld\n", freq);
6862}
6863
6864static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
6865{
6866 int err;
6867 u32 ref_clk;
6868 u32 freq = hba->dev_ref_clk_freq;
6869
6870 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6871 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
6872
6873 if (err) {
6874 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
6875 err);
6876 goto out;
6877 }
6878
6879 if (ref_clk == freq)
6880 goto out; /* nothing to update */
6881
6882 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
6883 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
6884
6885 if (err) {
6886 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
6887 ufs_ref_clk_freqs[freq].freq_hz);
6888 goto out;
6889 }
6890
6891 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
6892 ufs_ref_clk_freqs[freq].freq_hz);
6893
6894out:
6895 return err;
6896}
6897
6ccf44fe 6898/**
1d337ec2
SRT		 6899 * ufshcd_probe_hba - probe hba to detect device and initialize
6900 * @hba: per-adapter instance
6901 *
6902 * Execute link-startup and verify device initialization
6ccf44fe 6903 */
1d337ec2 6904static int ufshcd_probe_hba(struct ufs_hba *hba)
6ccf44fe 6905{
93fdd5ac 6906 struct ufs_dev_desc card = {0};
6ccf44fe 6907 int ret;
7ff5ab47 6908 ktime_t start = ktime_get();
6ccf44fe
SJ		 6909
6910 ret = ufshcd_link_startup(hba);
5a0b0cb9
SRT		 6911 if (ret)
6912 goto out;
6913
afdfff59
YG		 6914 /* set the default level for urgent bkops */
6915 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
6916 hba->is_urgent_bkops_lvl_checked = false;
6917
ff8e20c6
DR		 6918 /* Debug counters initialization */
6919 ufshcd_clear_dbg_ufs_stats(hba);
6920
57d104c1
SJ		 6921 /* UniPro link is active now */
6922 ufshcd_set_link_active(hba);
d3e89bac 6923
5a0b0cb9
SRT		 6924 ret = ufshcd_verify_dev_init(hba);
6925 if (ret)
6926 goto out;
68078d5c
DR		 6927
6928 ret = ufshcd_complete_dev_init(hba);
6929 if (ret)
6930 goto out;
5a0b0cb9 6931
a4b0e8a4
PM		 6932 /* Init check for device descriptor sizes */
6933 ufshcd_init_desc_sizes(hba);
6934
93fdd5ac
TW		 6935 ret = ufs_get_device_desc(hba, &card);
6936 if (ret) {
6937 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
6938 __func__, ret);
6939 goto out;
6940 }
6941
6942 ufs_fixup_device_setup(hba, &card);
4b828fe1
TW		 6943 ufs_put_device_desc(&card);
6944
37113106 6945 ufshcd_tune_unipro_params(hba);
60f01870 6946
57d104c1
SJ		 6947 /* UFS device is also active now */
6948 ufshcd_set_ufs_dev_active(hba);
66ec6d59 6949 ufshcd_force_reset_auto_bkops(hba);
57d104c1
SJ		 6950 hba->wlun_dev_clr_ua = true;
6951
7eb584db
DR		 6952 if (ufshcd_get_max_pwr_mode(hba)) {
6953 dev_err(hba->dev,
6954 "%s: Failed getting max supported power mode\n",
6955 __func__);
6956 } else {
9e1e8a75
SJ		 6957 /*
6958 * Set the right value to bRefClkFreq before attempting to
6959 * switch to HS gears.
6960 */
6961 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
6962 ufshcd_set_dev_ref_clk(hba);
7eb584db 6963 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8643ae66 6964 if (ret) {
7eb584db
DR		 6965 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
6966 __func__, ret);
8643ae66
DL		 6967 goto out;
6968 }
7eb584db 6969 }
57d104c1 6970
53c12d0e
YG		 6971 /* set the state as operational after switching to desired gear */
6972 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
a4b0e8a4 6973
71d848b8
CG		 6974 /* Enable Auto-Hibernate if configured */
6975 ufshcd_auto_hibern8_enable(hba);
6976
57d104c1
SJ		 6977 /*
6978 * If we are in error handling context or in power management callbacks
6979 * context, no need to scan the host
6980 */
6981 if (!ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
6982 bool flag;
6983
6984 /* clear any previous UFS device information */
6985 memset(&hba->dev_info, 0, sizeof(hba->dev_info));
dc3c8d3a
YG		 6986 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
6987 QUERY_FLAG_IDN_PWR_ON_WPE, &flag))
57d104c1 6988 hba->dev_info.f_power_on_wp_en = flag;
3441da7d 6989
3a4bf06d
YG		 6990 if (!hba->is_init_prefetch)
6991 ufshcd_init_icc_levels(hba);
6992
2a8fa600
SJ		 6993 /* Add required well known logical units to scsi mid layer */
6994 if (ufshcd_scsi_add_wlus(hba))
6995 goto out;
6996
0701e49d
SJ		 6997 /* Initialize devfreq after UFS device is detected */
6998 if (ufshcd_is_clkscaling_supported(hba)) {
6999 memcpy(&hba->clk_scaling.saved_pwr_info.info,
7000 &hba->pwr_info,
7001 sizeof(struct ufs_pa_layer_attr));
7002 hba->clk_scaling.saved_pwr_info.is_valid = true;
7003 if (!hba->devfreq) {
deac444f
BA		 7004 ret = ufshcd_devfreq_init(hba);
7005 if (ret)
0701e49d 7006 goto out;
0701e49d
SJ		 7007 }
7008 hba->clk_scaling.is_allowed = true;
7009 }
7010
df032bf2
AA		 7011 ufs_bsg_probe(hba);
7012
3441da7d
SRT		 7013 scsi_scan_host(hba->host);
7014 pm_runtime_put_sync(hba->dev);
7015 }
3a4bf06d
YG		 7016
7017 if (!hba->is_init_prefetch)
7018 hba->is_init_prefetch = true;
7019
5a0b0cb9 7020out:
1d337ec2
SRT		 7021 /*
7022 * If we failed to initialize the device or the device is not
7023 * present, turn off the power/clocks etc.
7024 */
57d104c1
SJ		 7025 if (ret && !ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
7026 pm_runtime_put_sync(hba->dev);
eebcc196 7027 ufshcd_exit_clk_scaling(hba);
1d337ec2 7028 ufshcd_hba_exit(hba);
57d104c1 7029 }
1d337ec2 7030
7ff5ab47
SJ		 7031 trace_ufshcd_init(dev_name(hba->dev), ret,
7032 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 7033 hba->curr_dev_pwr_mode, hba->uic_link_state);
1d337ec2
SRT		 7034 return ret;
7035}
7036
7037/**
7038 * ufshcd_async_scan - asynchronous execution for probing hba
7039 * @data: data pointer to pass to this function
7040 * @cookie: cookie data
7041 */
7042static void ufshcd_async_scan(void *data, async_cookie_t cookie)
7043{
7044 struct ufs_hba *hba = (struct ufs_hba *)data;
7045
7046 ufshcd_probe_hba(hba);
6ccf44fe
SJ		 7047}
7048
f550c65b
YG		 7049static enum blk_eh_timer_return ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
7050{
7051 unsigned long flags;
7052 struct Scsi_Host *host;
7053 struct ufs_hba *hba;
7054 int index;
7055 bool found = false;
7056
7057 if (!scmd || !scmd->device || !scmd->device->host)
6600593c 7058 return BLK_EH_DONE;
f550c65b
YG		 7059
7060 host = scmd->device->host;
7061 hba = shost_priv(host);
7062 if (!hba)
6600593c 7063 return BLK_EH_DONE;
f550c65b
YG		 7064
7065 spin_lock_irqsave(host->host_lock, flags);
7066
7067 for_each_set_bit(index, &hba->outstanding_reqs, hba->nutrs) {
7068 if (hba->lrb[index].cmd == scmd) {
7069 found = true;
7070 break;
7071 }
7072 }
7073
7074 spin_unlock_irqrestore(host->host_lock, flags);
7075
7076 /*
7077 * Bypass SCSI error handling and reset the block layer timer if this
7078 * SCSI command was not actually dispatched to UFS driver, otherwise
7079 * let SCSI layer handle the error as usual.
7080 */
6600593c 7081 return found ? BLK_EH_DONE : BLK_EH_RESET_TIMER;
f550c65b
YG		 7082}
7083
d829fc8a
SN		 7084static const struct attribute_group *ufshcd_driver_groups[] = {
7085 &ufs_sysfs_unit_descriptor_group,
ec92b59c 7086 &ufs_sysfs_lun_attributes_group,
d829fc8a
SN		 7087 NULL,
7088};
7089
7a3e97b0
SY		 7090static struct scsi_host_template ufshcd_driver_template = {
7091 .module = THIS_MODULE,
7092 .name = UFSHCD,
7093 .proc_name = UFSHCD,
7094 .queuecommand = ufshcd_queuecommand,
7095 .slave_alloc = ufshcd_slave_alloc,
eeda4749 7096 .slave_configure = ufshcd_slave_configure,
7a3e97b0 7097 .slave_destroy = ufshcd_slave_destroy,
4264fd61 7098 .change_queue_depth = ufshcd_change_queue_depth,
7a3e97b0 7099 .eh_abort_handler = ufshcd_abort,
3441da7d
SRT		 7100 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
7101 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
f550c65b 7102 .eh_timed_out = ufshcd_eh_timed_out,
7a3e97b0
SY		 7103 .this_id = -1,
7104 .sg_tablesize = SG_ALL,
7105 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
7106 .can_queue = UFSHCD_CAN_QUEUE,
552a990c 7107 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
1ab27c9c 7108 .max_host_blocked = 1,
c40ecc12 7109 .track_queue_depth = 1,
d829fc8a 7110 .sdev_groups = ufshcd_driver_groups,
4af14d11 7111 .dma_boundary = PAGE_SIZE - 1,
49615ba1 7112 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS,
7a3e97b0
SY		 7113};
7114
57d104c1
SJ		 7115static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
7116 int ua)
7117{
7b16a07c 7118 int ret;
57d104c1 7119
7b16a07c
BA		 7120 if (!vreg)
7121 return 0;
57d104c1 7122
0487fff7
SC		 7123 /*
7124 * "set_load" operation shall be required on those regulators
7125 * which specifically configured current limitation. Otherwise
7126 * zero max_uA may cause unexpected behavior when regulator is
7127 * enabled or set as high power mode.
7128 */
7129 if (!vreg->max_uA)
7130 return 0;
7131
7b16a07c
BA		 7132 ret = regulator_set_load(vreg->reg, ua);
7133 if (ret < 0) {
7134 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
7135 __func__, vreg->name, ua, ret);
57d104c1
SJ		 7136 }
7137
7138 return ret;
7139}
7140
7141static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
7142 struct ufs_vreg *vreg)
7143{
73067981 7144 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
57d104c1
SJ		 7145}
7146
7147static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
7148 struct ufs_vreg *vreg)
7149{
7c7cfdcf
AH		 7150 if (!vreg)
7151 return 0;
7152
73067981 7153 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
57d104c1
SJ		 7154}
7155
aa497613
SRT		 7156static int ufshcd_config_vreg(struct device *dev,
7157 struct ufs_vreg *vreg, bool on)
7158{
7159 int ret = 0;
72753590
GS		 7160 struct regulator *reg;
7161 const char *name;
aa497613
SRT		 7162 int min_uV, uA_load;
7163
7164 BUG_ON(!vreg);
7165
72753590
GS		 7166 reg = vreg->reg;
7167 name = vreg->name;
7168
aa497613 7169 if (regulator_count_voltages(reg) > 0) {
3b141e8c
SC		 7170 if (vreg->min_uV && vreg->max_uV) {
7171 min_uV = on ? vreg->min_uV : 0;
7172 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
7173 if (ret) {
7174 dev_err(dev,
7175 "%s: %s set voltage failed, err=%d\n",
aa497613 7176 __func__, name, ret);
3b141e8c
SC		 7177 goto out;
7178 }
aa497613
SRT		 7179 }
7180
7181 uA_load = on ? vreg->max_uA : 0;
57d104c1
SJ		 7182 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
7183 if (ret)
aa497613 7184 goto out;
aa497613
SRT		 7185 }
7186out:
7187 return ret;
7188}
7189
7190static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
7191{
7192 int ret = 0;
7193
73067981 7194 if (!vreg || vreg->enabled)
aa497613
SRT		 7195 goto out;
7196
7197 ret = ufshcd_config_vreg(dev, vreg, true);
7198 if (!ret)
7199 ret = regulator_enable(vreg->reg);
7200
7201 if (!ret)
7202 vreg->enabled = true;
7203 else
7204 dev_err(dev, "%s: %s enable failed, err=%d\n",
7205 __func__, vreg->name, ret);
7206out:
7207 return ret;
7208}
7209
7210static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
7211{
7212 int ret = 0;
7213
73067981 7214 if (!vreg || !vreg->enabled)
aa497613
SRT		 7215 goto out;
7216
7217 ret = regulator_disable(vreg->reg);
7218
7219 if (!ret) {
7220 /* ignore errors on applying disable config */
7221 ufshcd_config_vreg(dev, vreg, false);
7222 vreg->enabled = false;
7223 } else {
7224 dev_err(dev, "%s: %s disable failed, err=%d\n",
7225 __func__, vreg->name, ret);
7226 }
7227out:
7228 return ret;
7229}
7230
7231static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
7232{
7233 int ret = 0;
7234 struct device *dev = hba->dev;
7235 struct ufs_vreg_info *info = &hba->vreg_info;
7236
aa497613
SRT		 7237 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
7238 if (ret)
7239 goto out;
7240
7241 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
7242 if (ret)
7243 goto out;
7244
7245 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
7246 if (ret)
7247 goto out;
7248
7249out:
7250 if (ret) {
7251 ufshcd_toggle_vreg(dev, info->vccq2, false);
7252 ufshcd_toggle_vreg(dev, info->vccq, false);
7253 ufshcd_toggle_vreg(dev, info->vcc, false);
7254 }
7255 return ret;
7256}
7257
6a771a65
RS		 7258static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
7259{
7260 struct ufs_vreg_info *info = &hba->vreg_info;
7261
60b7b823 7262 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
6a771a65
RS		 7263}
7264
aa497613
SRT		 7265static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
7266{
7267 int ret = 0;
7268
7269 if (!vreg)
7270 goto out;
7271
7272 vreg->reg = devm_regulator_get(dev, vreg->name);
7273 if (IS_ERR(vreg->reg)) {
7274 ret = PTR_ERR(vreg->reg);
7275 dev_err(dev, "%s: %s get failed, err=%d\n",
7276 __func__, vreg->name, ret);
7277 }
7278out:
7279 return ret;
7280}
7281
7282static int ufshcd_init_vreg(struct ufs_hba *hba)
7283{
7284 int ret = 0;
7285 struct device *dev = hba->dev;
7286 struct ufs_vreg_info *info = &hba->vreg_info;
7287
aa497613
SRT		 7288 ret = ufshcd_get_vreg(dev, info->vcc);
7289 if (ret)
7290 goto out;
7291
7292 ret = ufshcd_get_vreg(dev, info->vccq);
7293 if (ret)
7294 goto out;
7295
7296 ret = ufshcd_get_vreg(dev, info->vccq2);
7297out:
7298 return ret;
7299}
7300
6a771a65
RS		 7301static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
7302{
7303 struct ufs_vreg_info *info = &hba->vreg_info;
7304
7305 if (info)
7306 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
7307
7308 return 0;
7309}
7310
57d104c1
SJ		 7311static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
7312 bool skip_ref_clk)
c6e79dac
SRT		 7313{
7314 int ret = 0;
7315 struct ufs_clk_info *clki;
7316 struct list_head *head = &hba->clk_list_head;
1ab27c9c 7317 unsigned long flags;
911a0771
SJ		 7318 ktime_t start = ktime_get();
7319 bool clk_state_changed = false;
c6e79dac 7320
566ec9ad 7321 if (list_empty(head))
c6e79dac
SRT		 7322 goto out;
7323
b334456e
SJ		 7324 /*
7325 * vendor specific setup_clocks ops may depend on clocks managed by
7326 * this standard driver hence call the vendor specific setup_clocks
7327 * before disabling the clocks managed here.
7328 */
7329 if (!on) {
7330 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
7331 if (ret)
7332 return ret;
7333 }
1e879e8f 7334
c6e79dac
SRT		 7335 list_for_each_entry(clki, head, list) {
7336 if (!IS_ERR_OR_NULL(clki->clk)) {
57d104c1
SJ		 7337 if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
7338 continue;
7339
911a0771 7340 clk_state_changed = on ^ clki->enabled;
c6e79dac
SRT		 7341 if (on && !clki->enabled) {
7342 ret = clk_prepare_enable(clki->clk);
7343 if (ret) {
7344 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
7345 __func__, clki->name, ret);
7346 goto out;
7347 }
7348 } else if (!on && clki->enabled) {
7349 clk_disable_unprepare(clki->clk);
7350 }
7351 clki->enabled = on;
7352 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
7353 clki->name, on ? "en" : "dis");
7354 }
7355 }
1ab27c9c 7356
b334456e
SJ		 7357 /*
7358 * vendor specific setup_clocks ops may depend on clocks managed by
7359 * this standard driver hence call the vendor specific setup_clocks
7360 * after enabling the clocks managed here.
7361 */
7362 if (on) {
7363 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
7364 if (ret)
7365 return ret;
7366 }
1e879e8f 7367
c6e79dac
SRT		 7368out:
7369 if (ret) {
7370 list_for_each_entry(clki, head, list) {
7371 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
7372 clk_disable_unprepare(clki->clk);
7373 }
7ff5ab47 7374 } else if (!ret && on) {
1ab27c9c
ST		 7375 spin_lock_irqsave(hba->host->host_lock, flags);
7376 hba->clk_gating.state = CLKS_ON;
7ff5ab47
SJ		 7377 trace_ufshcd_clk_gating(dev_name(hba->dev),
7378 hba->clk_gating.state);
1ab27c9c 7379 spin_unlock_irqrestore(hba->host->host_lock, flags);
c6e79dac 7380 }
7ff5ab47 7381
911a0771
SJ		 7382 if (clk_state_changed)
7383 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
7384 (on ? "on" : "off"),
7385 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
c6e79dac
SRT		 7386 return ret;
7387}
7388
57d104c1
SJ		 7389static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
7390{
7391 return __ufshcd_setup_clocks(hba, on, false);
7392}
7393
c6e79dac
SRT		 7394static int ufshcd_init_clocks(struct ufs_hba *hba)
7395{
7396 int ret = 0;
7397 struct ufs_clk_info *clki;
7398 struct device *dev = hba->dev;
7399 struct list_head *head = &hba->clk_list_head;
7400
566ec9ad 7401 if (list_empty(head))
c6e79dac
SRT		 7402 goto out;
7403
7404 list_for_each_entry(clki, head, list) {
7405 if (!clki->name)
7406 continue;
7407
7408 clki->clk = devm_clk_get(dev, clki->name);
7409 if (IS_ERR(clki->clk)) {
7410 ret = PTR_ERR(clki->clk);
7411 dev_err(dev, "%s: %s clk get failed, %d\n",
7412 __func__, clki->name, ret);
7413 goto out;
7414 }
7415
9e1e8a75
SJ		 7416 /*
7417 * Parse device ref clk freq as per device tree "ref_clk".
7418 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
7419 * in ufshcd_alloc_host().
7420 */
7421 if (!strcmp(clki->name, "ref_clk"))
7422 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
7423
c6e79dac
SRT		 7424 if (clki->max_freq) {
7425 ret = clk_set_rate(clki->clk, clki->max_freq);
7426 if (ret) {
7427 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
7428 __func__, clki->name,
7429 clki->max_freq, ret);
7430 goto out;
7431 }
856b3483 7432 clki->curr_freq = clki->max_freq;
c6e79dac
SRT		 7433 }
7434 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
7435 clki->name, clk_get_rate(clki->clk));
7436 }
7437out:
7438 return ret;
7439}
7440
5c0c28a8
SRT		 7441static int ufshcd_variant_hba_init(struct ufs_hba *hba)
7442{
7443 int err = 0;
7444
7445 if (!hba->vops)
7446 goto out;
7447
0263bcd0
YG		 7448 err = ufshcd_vops_init(hba);
7449 if (err)
7450 goto out;
5c0c28a8 7451
0263bcd0
YG		 7452 err = ufshcd_vops_setup_regulators(hba, true);
7453 if (err)
7454 goto out_exit;
5c0c28a8
SRT		 7455
7456 goto out;
7457
5c0c28a8 7458out_exit:
0263bcd0 7459 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 7460out:
7461 if (err)
7462 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
0263bcd0 7463 __func__, ufshcd_get_var_name(hba), err);
5c0c28a8
SRT		 7464 return err;
7465}
7466
7467static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
7468{
7469 if (!hba->vops)
7470 return;
7471
0263bcd0 7472 ufshcd_vops_setup_regulators(hba, false);
5c0c28a8 7473
0263bcd0 7474 ufshcd_vops_exit(hba);
5c0c28a8
SRT		 7475}
7476
aa497613
SRT		 7477static int ufshcd_hba_init(struct ufs_hba *hba)
7478{
7479 int err;
7480
6a771a65
RS		 7481 /*
7482 * Handle host controller power separately from the UFS device power
7483 * rails as it will help controlling the UFS host controller power
7484 * collapse easily which is different than UFS device power collapse.
7485 * Also, enable the host controller power before we go ahead with rest
7486 * of the initialization here.
7487 */
7488 err = ufshcd_init_hba_vreg(hba);
aa497613
SRT		 7489 if (err)
7490 goto out;
7491
6a771a65 7492 err = ufshcd_setup_hba_vreg(hba, true);
aa497613
SRT		 7493 if (err)
7494 goto out;
7495
6a771a65
RS		 7496 err = ufshcd_init_clocks(hba);
7497 if (err)
7498 goto out_disable_hba_vreg;
7499
7500 err = ufshcd_setup_clocks(hba, true);
7501 if (err)
7502 goto out_disable_hba_vreg;
7503
c6e79dac
SRT		 7504 err = ufshcd_init_vreg(hba);
7505 if (err)
7506 goto out_disable_clks;
7507
7508 err = ufshcd_setup_vreg(hba, true);
7509 if (err)
7510 goto out_disable_clks;
7511
aa497613
SRT		 7512 err = ufshcd_variant_hba_init(hba);
7513 if (err)
7514 goto out_disable_vreg;
7515
1d337ec2 7516 hba->is_powered = true;
aa497613
SRT		 7517 goto out;
7518
7519out_disable_vreg:
7520 ufshcd_setup_vreg(hba, false);
c6e79dac
SRT		 7521out_disable_clks:
7522 ufshcd_setup_clocks(hba, false);
6a771a65
RS		 7523out_disable_hba_vreg:
7524 ufshcd_setup_hba_vreg(hba, false);
aa497613
SRT		 7525out:
7526 return err;
7527}
7528
7529static void ufshcd_hba_exit(struct ufs_hba *hba)
7530{
1d337ec2
SRT		 7531 if (hba->is_powered) {
7532 ufshcd_variant_hba_exit(hba);
7533 ufshcd_setup_vreg(hba, false);
a508253d 7534 ufshcd_suspend_clkscaling(hba);
eebcc196 7535 if (ufshcd_is_clkscaling_supported(hba))
0701e49d
SJ		 7536 if (hba->devfreq)
7537 ufshcd_suspend_clkscaling(hba);
1d337ec2
SRT		 7538 ufshcd_setup_clocks(hba, false);
7539 ufshcd_setup_hba_vreg(hba, false);
7540 hba->is_powered = false;
7541 }
aa497613
SRT		 7542}
7543
57d104c1
SJ		 7544static int
7545ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
7546{
7547 unsigned char cmd[6] = {REQUEST_SENSE,
7548 0,
7549 0,
7550 0,
09a5a24f 7551 UFS_SENSE_SIZE,
57d104c1
SJ		 7552 0};
7553 char *buffer;
7554 int ret;
7555
09a5a24f 7556 buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
57d104c1
SJ		 7557 if (!buffer) {
7558 ret = -ENOMEM;
7559 goto out;
7560 }
7561
fcbfffe2 7562 ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
09a5a24f 7563 UFS_SENSE_SIZE, NULL, NULL,
fcbfffe2 7564 msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
57d104c1
SJ		 7565 if (ret)
7566 pr_err("%s: failed with err %d\n", __func__, ret);
7567
7568 kfree(buffer);
7569out:
7570 return ret;
7571}
7572
7573/**
7574 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
7575 * power mode
7576 * @hba: per adapter instance
7577 * @pwr_mode: device power mode to set
7578 *
7579 * Returns 0 if requested power mode is set successfully
7580 * Returns non-zero if failed to set the requested power mode
7581 */
7582static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
7583 enum ufs_dev_pwr_mode pwr_mode)
7584{
7585 unsigned char cmd[6] = { START_STOP };
7586 struct scsi_sense_hdr sshdr;
7c48bfd0
AM		 7587 struct scsi_device *sdp;
7588 unsigned long flags;
57d104c1
SJ		 7589 int ret;
7590
7c48bfd0
AM		 7591 spin_lock_irqsave(hba->host->host_lock, flags);
7592 sdp = hba->sdev_ufs_device;
7593 if (sdp) {
7594 ret = scsi_device_get(sdp);
7595 if (!ret && !scsi_device_online(sdp)) {
7596 ret = -ENODEV;
7597 scsi_device_put(sdp);
7598 }
7599 } else {
7600 ret = -ENODEV;
7601 }
7602 spin_unlock_irqrestore(hba->host->host_lock, flags);
7603
7604 if (ret)
7605 return ret;
57d104c1
SJ		 7606
7607 /*
7608 * If scsi commands fail, the scsi mid-layer schedules scsi error-
7609 * handling, which would wait for host to be resumed. Since we know
7610 * we are functional while we are here, skip host resume in error
7611 * handling context.
7612 */
7613 hba->host->eh_noresume = 1;
7614 if (hba->wlun_dev_clr_ua) {
7615 ret = ufshcd_send_request_sense(hba, sdp);
7616 if (ret)
7617 goto out;
7618 /* Unit attention condition is cleared now */
7619 hba->wlun_dev_clr_ua = false;
7620 }
7621
7622 cmd[4] = pwr_mode << 4;
7623
7624 /*
7625 * Current function would be generally called from the power management
e8064021 7626 * callbacks hence set the RQF_PM flag so that it doesn't resume the
57d104c1
SJ		 7627 * already suspended childs.
7628 */
fcbfffe2
CH		 7629 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
7630 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
57d104c1
SJ		 7631 if (ret) {
7632 sdev_printk(KERN_WARNING, sdp,
ef61329d
HR		 7633 "START_STOP failed for power mode: %d, result %x\n",
7634 pwr_mode, ret);
c65be1a6 7635 if (driver_byte(ret) == DRIVER_SENSE)
21045519 7636 scsi_print_sense_hdr(sdp, NULL, &sshdr);
57d104c1
SJ		 7637 }
7638
7639 if (!ret)
7640 hba->curr_dev_pwr_mode = pwr_mode;
7641out:
7c48bfd0 7642 scsi_device_put(sdp);
57d104c1
SJ		 7643 hba->host->eh_noresume = 0;
7644 return ret;
7645}
7646
7647static int ufshcd_link_state_transition(struct ufs_hba *hba,
7648 enum uic_link_state req_link_state,
7649 int check_for_bkops)
7650{
7651 int ret = 0;
7652
7653 if (req_link_state == hba->uic_link_state)
7654 return 0;
7655
7656 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
7657 ret = ufshcd_uic_hibern8_enter(hba);
7658 if (!ret)
7659 ufshcd_set_link_hibern8(hba);
7660 else
7661 goto out;
7662 }
7663 /*
7664 * If autobkops is enabled, link can't be turned off because
7665 * turning off the link would also turn off the device.
7666 */
7667 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
7668 (!check_for_bkops || (check_for_bkops &&
7669 !hba->auto_bkops_enabled))) {
f3099fbd
YG		 7670 /*
7671 * Let's make sure that link is in low power mode, we are doing
7672 * this currently by putting the link in Hibern8. Otherway to
7673 * put the link in low power mode is to send the DME end point
7674 * to device and then send the DME reset command to local
7675 * unipro. But putting the link in hibern8 is much faster.
7676 */
7677 ret = ufshcd_uic_hibern8_enter(hba);
7678 if (ret)
7679 goto out;
57d104c1
SJ		 7680 /*
7681 * Change controller state to "reset state" which
7682 * should also put the link in off/reset state
7683 */
596585a2 7684 ufshcd_hba_stop(hba, true);
57d104c1
SJ		 7685 /*
7686 * TODO: Check if we need any delay to make sure that
7687 * controller is reset
7688 */
7689 ufshcd_set_link_off(hba);
7690 }
7691
7692out:
7693 return ret;
7694}
7695
7696static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
7697{
b799fdf7
YG		 7698 /*
7699 * It seems some UFS devices may keep drawing more than sleep current
7700 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7701 * To avoid this situation, add 2ms delay before putting these UFS
7702 * rails in LPM mode.
7703 */
7704 if (!ufshcd_is_link_active(hba) &&
7705 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
7706 usleep_range(2000, 2100);
7707
57d104c1
SJ		 7708 /*
7709 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7710 * power.
7711 *
7712 * If UFS device and link is in OFF state, all power supplies (VCC,
7713 * VCCQ, VCCQ2) can be turned off if power on write protect is not
7714 * required. If UFS link is inactive (Hibern8 or OFF state) and device
7715 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7716 *
7717 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7718 * in low power state which would save some power.
7719 */
7720 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7721 !hba->dev_info.is_lu_power_on_wp) {
7722 ufshcd_setup_vreg(hba, false);
7723 } else if (!ufshcd_is_ufs_dev_active(hba)) {
7724 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7725 if (!ufshcd_is_link_active(hba)) {
7726 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7727 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
7728 }
7729 }
7730}
7731
7732static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
7733{
7734 int ret = 0;
7735
7736 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7737 !hba->dev_info.is_lu_power_on_wp) {
7738 ret = ufshcd_setup_vreg(hba, true);
7739 } else if (!ufshcd_is_ufs_dev_active(hba)) {
57d104c1
SJ		 7740 if (!ret && !ufshcd_is_link_active(hba)) {
7741 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
7742 if (ret)
7743 goto vcc_disable;
7744 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
7745 if (ret)
7746 goto vccq_lpm;
7747 }
69d72ac8 7748 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
57d104c1
SJ		 7749 }
7750 goto out;
7751
7752vccq_lpm:
7753 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7754vcc_disable:
7755 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7756out:
7757 return ret;
7758}
7759
7760static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
7761{
7762 if (ufshcd_is_link_off(hba))
7763 ufshcd_setup_hba_vreg(hba, false);
7764}
7765
7766static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
7767{
7768 if (ufshcd_is_link_off(hba))
7769 ufshcd_setup_hba_vreg(hba, true);
7770}
7771
7a3e97b0 7772/**
57d104c1 7773 * ufshcd_suspend - helper function for suspend operations
3b1d0580 7774 * @hba: per adapter instance
57d104c1
SJ		 7775 * @pm_op: desired low power operation type
7776 *
7777 * This function will try to put the UFS device and link into low power
7778 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7779 * (System PM level).
7780 *
7781 * If this function is called during shutdown, it will make sure that
7782 * both UFS device and UFS link is powered off.
7a3e97b0 7783 *
57d104c1
SJ		 7784 * NOTE: UFS device & link must be active before we enter in this function.
7785 *
7786 * Returns 0 for success and non-zero for failure
7a3e97b0 7787 */
57d104c1 7788static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 7789{
57d104c1
SJ		 7790 int ret = 0;
7791 enum ufs_pm_level pm_lvl;
7792 enum ufs_dev_pwr_mode req_dev_pwr_mode;
7793 enum uic_link_state req_link_state;
7794
7795 hba->pm_op_in_progress = 1;
7796 if (!ufshcd_is_shutdown_pm(pm_op)) {
7797 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
7798 hba->rpm_lvl : hba->spm_lvl;
7799 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
7800 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
7801 } else {
7802 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
7803 req_link_state = UIC_LINK_OFF_STATE;
7804 }
7805
7a3e97b0 7806 /*
57d104c1
SJ		 7807 * If we can't transition into any of the low power modes
7808 * just gate the clocks.
7a3e97b0 7809 */
1ab27c9c
ST		 7810 ufshcd_hold(hba, false);
7811 hba->clk_gating.is_suspended = true;
7812
401f1e44
SJ		 7813 if (hba->clk_scaling.is_allowed) {
7814 cancel_work_sync(&hba->clk_scaling.suspend_work);
7815 cancel_work_sync(&hba->clk_scaling.resume_work);
7816 ufshcd_suspend_clkscaling(hba);
7817 }
d6fcf81a 7818
57d104c1
SJ		 7819 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
7820 req_link_state == UIC_LINK_ACTIVE_STATE) {
7821 goto disable_clks;
7822 }
7a3e97b0 7823
57d104c1
SJ		 7824 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
7825 (req_link_state == hba->uic_link_state))
d6fcf81a 7826 goto enable_gating;
57d104c1
SJ		 7827
7828 /* UFS device & link must be active before we enter in this function */
7829 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
7830 ret = -EINVAL;
d6fcf81a 7831 goto enable_gating;
57d104c1
SJ		 7832 }
7833
7834 if (ufshcd_is_runtime_pm(pm_op)) {
374a246e
SJ		 7835 if (ufshcd_can_autobkops_during_suspend(hba)) {
7836 /*
7837 * The device is idle with no requests in the queue,
7838 * allow background operations if bkops status shows
7839 * that performance might be impacted.
7840 */
7841 ret = ufshcd_urgent_bkops(hba);
7842 if (ret)
7843 goto enable_gating;
7844 } else {
7845 /* make sure that auto bkops is disabled */
7846 ufshcd_disable_auto_bkops(hba);
7847 }
57d104c1
SJ		 7848 }
7849
7850 if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) &&
7851 ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
7852 !ufshcd_is_runtime_pm(pm_op))) {
7853 /* ensure that bkops is disabled */
7854 ufshcd_disable_auto_bkops(hba);
7855 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
7856 if (ret)
1ab27c9c 7857 goto enable_gating;
57d104c1
SJ		 7858 }
7859
7860 ret = ufshcd_link_state_transition(hba, req_link_state, 1);
7861 if (ret)
7862 goto set_dev_active;
7863
7864 ufshcd_vreg_set_lpm(hba);
7865
7866disable_clks:
7867 /*
7868 * Call vendor specific suspend callback. As these callbacks may access
7869 * vendor specific host controller register space call them before the
7870 * host clocks are ON.
7871 */
0263bcd0
YG		 7872 ret = ufshcd_vops_suspend(hba, pm_op);
7873 if (ret)
7874 goto set_link_active;
57d104c1 7875
57d104c1
SJ		 7876 if (!ufshcd_is_link_active(hba))
7877 ufshcd_setup_clocks(hba, false);
7878 else
7879 /* If link is active, device ref_clk can't be switched off */
7880 __ufshcd_setup_clocks(hba, false, true);
7881
1ab27c9c 7882 hba->clk_gating.state = CLKS_OFF;
7ff5ab47 7883 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
57d104c1
SJ		 7884 /*
7885 * Disable the host irq as host controller as there won't be any
0263bcd0 7886 * host controller transaction expected till resume.
57d104c1
SJ		 7887 */
7888 ufshcd_disable_irq(hba);
7889 /* Put the host controller in low power mode if possible */
7890 ufshcd_hba_vreg_set_lpm(hba);
7891 goto out;
7892
57d104c1 7893set_link_active:
401f1e44
SJ		 7894 if (hba->clk_scaling.is_allowed)
7895 ufshcd_resume_clkscaling(hba);
57d104c1
SJ		 7896 ufshcd_vreg_set_hpm(hba);
7897 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
7898 ufshcd_set_link_active(hba);
7899 else if (ufshcd_is_link_off(hba))
7900 ufshcd_host_reset_and_restore(hba);
7901set_dev_active:
7902 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
7903 ufshcd_disable_auto_bkops(hba);
1ab27c9c 7904enable_gating:
401f1e44
SJ		 7905 if (hba->clk_scaling.is_allowed)
7906 ufshcd_resume_clkscaling(hba);
1ab27c9c
ST		 7907 hba->clk_gating.is_suspended = false;
7908 ufshcd_release(hba);
57d104c1
SJ		 7909out:
7910 hba->pm_op_in_progress = 0;
8808b4e9
SC		 7911 if (ret)
7912 ufshcd_update_reg_hist(&hba->ufs_stats.suspend_err, (u32)ret);
57d104c1 7913 return ret;
7a3e97b0
SY		 7914}
7915
7916/**
57d104c1 7917 * ufshcd_resume - helper function for resume operations
3b1d0580 7918 * @hba: per adapter instance
57d104c1 7919 * @pm_op: runtime PM or system PM
7a3e97b0 7920 *
57d104c1
SJ		 7921 * This function basically brings the UFS device, UniPro link and controller
7922 * to active state.
7923 *
7924 * Returns 0 for success and non-zero for failure
7a3e97b0 7925 */
57d104c1 7926static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7a3e97b0 7927{
57d104c1
SJ		 7928 int ret;
7929 enum uic_link_state old_link_state;
7930
7931 hba->pm_op_in_progress = 1;
7932 old_link_state = hba->uic_link_state;
7933
7934 ufshcd_hba_vreg_set_hpm(hba);
7935 /* Make sure clocks are enabled before accessing controller */
7936 ret = ufshcd_setup_clocks(hba, true);
7937 if (ret)
7938 goto out;
7939
57d104c1
SJ		 7940 /* enable the host irq as host controller would be active soon */
7941 ret = ufshcd_enable_irq(hba);
7942 if (ret)
7943 goto disable_irq_and_vops_clks;
7944
7945 ret = ufshcd_vreg_set_hpm(hba);
7946 if (ret)
7947 goto disable_irq_and_vops_clks;
7948
7a3e97b0 7949 /*
57d104c1
SJ		 7950 * Call vendor specific resume callback. As these callbacks may access
7951 * vendor specific host controller register space call them when the
7952 * host clocks are ON.
7a3e97b0 7953 */
0263bcd0
YG		 7954 ret = ufshcd_vops_resume(hba, pm_op);
7955 if (ret)
7956 goto disable_vreg;
57d104c1
SJ		 7957
7958 if (ufshcd_is_link_hibern8(hba)) {
7959 ret = ufshcd_uic_hibern8_exit(hba);
7960 if (!ret)
7961 ufshcd_set_link_active(hba);
7962 else
7963 goto vendor_suspend;
7964 } else if (ufshcd_is_link_off(hba)) {
7965 ret = ufshcd_host_reset_and_restore(hba);
7966 /*
7967 * ufshcd_host_reset_and_restore() should have already
7968 * set the link state as active
7969 */
7970 if (ret || !ufshcd_is_link_active(hba))
7971 goto vendor_suspend;
7972 }
7973
7974 if (!ufshcd_is_ufs_dev_active(hba)) {
7975 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
7976 if (ret)
7977 goto set_old_link_state;
7978 }
7979
4e768e76
SJ		 7980 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
7981 ufshcd_enable_auto_bkops(hba);
7982 else
7983 /*
7984 * If BKOPs operations are urgently needed at this moment then
7985 * keep auto-bkops enabled or else disable it.
7986 */
7987 ufshcd_urgent_bkops(hba);
7988
1ab27c9c
ST		 7989 hba->clk_gating.is_suspended = false;
7990
fcb0c4b0
ST		 7991 if (hba->clk_scaling.is_allowed)
7992 ufshcd_resume_clkscaling(hba);
856b3483 7993
ad448378
AH		 7994 /* Enable Auto-Hibernate if configured */
7995 ufshcd_auto_hibern8_enable(hba);
7996
71d848b8
CG		 7997 /* Schedule clock gating in case of no access to UFS device yet */
7998 ufshcd_release(hba);
7999
57d104c1
SJ		 8000 goto out;
8001
8002set_old_link_state:
8003 ufshcd_link_state_transition(hba, old_link_state, 0);
8004vendor_suspend:
0263bcd0 8005 ufshcd_vops_suspend(hba, pm_op);
57d104c1
SJ		 8006disable_vreg:
8007 ufshcd_vreg_set_lpm(hba);
8008disable_irq_and_vops_clks:
8009 ufshcd_disable_irq(hba);
401f1e44
SJ		 8010 if (hba->clk_scaling.is_allowed)
8011 ufshcd_suspend_clkscaling(hba);
57d104c1
SJ		 8012 ufshcd_setup_clocks(hba, false);
8013out:
8014 hba->pm_op_in_progress = 0;
8808b4e9
SC		 8015 if (ret)
8016 ufshcd_update_reg_hist(&hba->ufs_stats.resume_err, (u32)ret);
57d104c1
SJ		 8017 return ret;
8018}
8019
8020/**
8021 * ufshcd_system_suspend - system suspend routine
8022 * @hba: per adapter instance
57d104c1
SJ		 8023 *
8024 * Check the description of ufshcd_suspend() function for more details.
8025 *
8026 * Returns 0 for success and non-zero for failure
8027 */
8028int ufshcd_system_suspend(struct ufs_hba *hba)
8029{
8030 int ret = 0;
7ff5ab47 8031 ktime_t start = ktime_get();
57d104c1
SJ		 8032
8033 if (!hba || !hba->is_powered)
233b594b 8034 return 0;
57d104c1 8035
0b257734
SJ		 8036 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
8037 hba->curr_dev_pwr_mode) &&
8038 (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
8039 hba->uic_link_state))
8040 goto out;
57d104c1 8041
0b257734 8042 if (pm_runtime_suspended(hba->dev)) {
57d104c1
SJ		 8043 /*
8044 * UFS device and/or UFS link low power states during runtime
8045 * suspend seems to be different than what is expected during
8046 * system suspend. Hence runtime resume the devic & link and
8047 * let the system suspend low power states to take effect.
8048 * TODO: If resume takes longer time, we might have optimize
8049 * it in future by not resuming everything if possible.
8050 */
8051 ret = ufshcd_runtime_resume(hba);
8052 if (ret)
8053 goto out;
8054 }
8055
8056 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
8057out:
7ff5ab47
SJ		 8058 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
8059 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8060 hba->curr_dev_pwr_mode, hba->uic_link_state);
e785060e
DR		 8061 if (!ret)
8062 hba->is_sys_suspended = true;
57d104c1
SJ		 8063 return ret;
8064}
8065EXPORT_SYMBOL(ufshcd_system_suspend);
8066
8067/**
8068 * ufshcd_system_resume - system resume routine
8069 * @hba: per adapter instance
8070 *
8071 * Returns 0 for success and non-zero for failure
8072 */
7a3e97b0 8073
57d104c1
SJ		 8074int ufshcd_system_resume(struct ufs_hba *hba)
8075{
7ff5ab47
SJ		 8076 int ret = 0;
8077 ktime_t start = ktime_get();
8078
e3ce73d6
YG		 8079 if (!hba)
8080 return -EINVAL;
8081
8082 if (!hba->is_powered || pm_runtime_suspended(hba->dev))
57d104c1
SJ		 8083 /*
8084 * Let the runtime resume take care of resuming
8085 * if runtime suspended.
8086 */
7ff5ab47
SJ		 8087 goto out;
8088 else
8089 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
8090out:
8091 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
8092 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8093 hba->curr_dev_pwr_mode, hba->uic_link_state);
ce9e7bce
SC		 8094 if (!ret)
8095 hba->is_sys_suspended = false;
7ff5ab47 8096 return ret;
7a3e97b0 8097}
57d104c1 8098EXPORT_SYMBOL(ufshcd_system_resume);
3b1d0580 8099
57d104c1
SJ		 8100/**
8101 * ufshcd_runtime_suspend - runtime suspend routine
8102 * @hba: per adapter instance
8103 *
8104 * Check the description of ufshcd_suspend() function for more details.
8105 *
8106 * Returns 0 for success and non-zero for failure
8107 */
66ec6d59
SRT		 8108int ufshcd_runtime_suspend(struct ufs_hba *hba)
8109{
7ff5ab47
SJ		 8110 int ret = 0;
8111 ktime_t start = ktime_get();
8112
e3ce73d6
YG		 8113 if (!hba)
8114 return -EINVAL;
8115
8116 if (!hba->is_powered)
7ff5ab47
SJ		 8117 goto out;
8118 else
8119 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
8120out:
8121 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
8122 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8123 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8124 return ret;
66ec6d59
SRT		 8125}
8126EXPORT_SYMBOL(ufshcd_runtime_suspend);
8127
57d104c1
SJ		 8128/**
8129 * ufshcd_runtime_resume - runtime resume routine
8130 * @hba: per adapter instance
8131 *
8132 * This function basically brings the UFS device, UniPro link and controller
8133 * to active state. Following operations are done in this function:
8134 *
8135 * 1. Turn on all the controller related clocks
8136 * 2. Bring the UniPro link out of Hibernate state
8137 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
8138 * to active state.
8139 * 4. If auto-bkops is enabled on the device, disable it.
8140 *
8141 * So following would be the possible power state after this function return
8142 * successfully:
8143 * S1: UFS device in Active state with VCC rail ON
8144 * UniPro link in Active state
8145 * All the UFS/UniPro controller clocks are ON
8146 *
8147 * Returns 0 for success and non-zero for failure
8148 */
66ec6d59
SRT		 8149int ufshcd_runtime_resume(struct ufs_hba *hba)
8150{
7ff5ab47
SJ		 8151 int ret = 0;
8152 ktime_t start = ktime_get();
8153
e3ce73d6
YG		 8154 if (!hba)
8155 return -EINVAL;
8156
8157 if (!hba->is_powered)
7ff5ab47
SJ		 8158 goto out;
8159 else
8160 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
8161out:
8162 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
8163 ktime_to_us(ktime_sub(ktime_get(), start)),
73eba2be 8164 hba->curr_dev_pwr_mode, hba->uic_link_state);
7ff5ab47 8165 return ret;
66ec6d59
SRT		 8166}
8167EXPORT_SYMBOL(ufshcd_runtime_resume);
8168
8169int ufshcd_runtime_idle(struct ufs_hba *hba)
8170{
8171 return 0;
8172}
8173EXPORT_SYMBOL(ufshcd_runtime_idle);
8174
57d104c1
SJ		 8175/**
8176 * ufshcd_shutdown - shutdown routine
8177 * @hba: per adapter instance
8178 *
8179 * This function would power off both UFS device and UFS link.
8180 *
8181 * Returns 0 always to allow force shutdown even in case of errors.
8182 */
8183int ufshcd_shutdown(struct ufs_hba *hba)
8184{
8185 int ret = 0;
8186
f51913ee
SC		 8187 if (!hba->is_powered)
8188 goto out;
8189
57d104c1
SJ		 8190 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
8191 goto out;
8192
8193 if (pm_runtime_suspended(hba->dev)) {
8194 ret = ufshcd_runtime_resume(hba);
8195 if (ret)
8196 goto out;
8197 }
8198
8199 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
8200out:
8201 if (ret)
8202 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
8203 /* allow force shutdown even in case of errors */
8204 return 0;
8205}
8206EXPORT_SYMBOL(ufshcd_shutdown);
8207
7a3e97b0 8208/**
3b1d0580 8209 * ufshcd_remove - de-allocate SCSI host and host memory space
7a3e97b0 8210 * data structure memory
8aa29f19 8211 * @hba: per adapter instance
7a3e97b0 8212 */
3b1d0580 8213void ufshcd_remove(struct ufs_hba *hba)
7a3e97b0 8214{
df032bf2 8215 ufs_bsg_remove(hba);
cbb6813e 8216 ufs_sysfs_remove_nodes(hba->dev);
69a6c269
BVA		 8217 blk_cleanup_queue(hba->tmf_queue);
8218 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360 8219 blk_cleanup_queue(hba->cmd_queue);
cfdf9c91 8220 scsi_remove_host(hba->host);
7a3e97b0 8221 /* disable interrupts */
2fbd009b 8222 ufshcd_disable_intr(hba, hba->intr_mask);
596585a2 8223 ufshcd_hba_stop(hba, true);
7a3e97b0 8224
eebcc196 8225 ufshcd_exit_clk_scaling(hba);
1ab27c9c 8226 ufshcd_exit_clk_gating(hba);
fcb0c4b0
ST		 8227 if (ufshcd_is_clkscaling_supported(hba))
8228 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
aa497613 8229 ufshcd_hba_exit(hba);
3b1d0580
VH		 8230}
8231EXPORT_SYMBOL_GPL(ufshcd_remove);
8232
47555a5c
YG		 8233/**
8234 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
8235 * @hba: pointer to Host Bus Adapter (HBA)
8236 */
8237void ufshcd_dealloc_host(struct ufs_hba *hba)
8238{
8239 scsi_host_put(hba->host);
8240}
8241EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
8242
ca3d7bf9
AM		 8243/**
8244 * ufshcd_set_dma_mask - Set dma mask based on the controller
8245 * addressing capability
8246 * @hba: per adapter instance
8247 *
8248 * Returns 0 for success, non-zero for failure
8249 */
8250static int ufshcd_set_dma_mask(struct ufs_hba *hba)
8251{
8252 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
8253 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
8254 return 0;
8255 }
8256 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
8257}
8258
7a3e97b0 8259/**
5c0c28a8 8260 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
3b1d0580
VH		 8261 * @dev: pointer to device handle
8262 * @hba_handle: driver private handle
7a3e97b0
SY		 8263 * Returns 0 on success, non-zero value on failure
8264 */
5c0c28a8 8265int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
7a3e97b0
SY		 8266{
8267 struct Scsi_Host *host;
8268 struct ufs_hba *hba;
5c0c28a8 8269 int err = 0;
7a3e97b0 8270
3b1d0580
VH		 8271 if (!dev) {
8272 dev_err(dev,
8273 "Invalid memory reference for dev is NULL\n");
8274 err = -ENODEV;
7a3e97b0
SY		 8275 goto out_error;
8276 }
8277
7a3e97b0
SY		 8278 host = scsi_host_alloc(&ufshcd_driver_template,
8279 sizeof(struct ufs_hba));
8280 if (!host) {
3b1d0580 8281 dev_err(dev, "scsi_host_alloc failed\n");
7a3e97b0 8282 err = -ENOMEM;
3b1d0580 8283 goto out_error;
7a3e97b0
SY		 8284 }
8285 hba = shost_priv(host);
7a3e97b0 8286 hba->host = host;
3b1d0580 8287 hba->dev = dev;
5c0c28a8 8288 *hba_handle = hba;
9e1e8a75 8289 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
5c0c28a8 8290
566ec9ad
SM		 8291 INIT_LIST_HEAD(&hba->clk_list_head);
8292
5c0c28a8
SRT		 8293out_error:
8294 return err;
8295}
8296EXPORT_SYMBOL(ufshcd_alloc_host);
8297
69a6c269
BVA		 8298/* This function exists because blk_mq_alloc_tag_set() requires this. */
8299static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
8300 const struct blk_mq_queue_data *qd)
8301{
8302 WARN_ON_ONCE(true);
8303 return BLK_STS_NOTSUPP;
8304}
8305
8306static const struct blk_mq_ops ufshcd_tmf_ops = {
8307 .queue_rq = ufshcd_queue_tmf,
8308};
8309
5c0c28a8
SRT		 8310/**
8311 * ufshcd_init - Driver initialization routine
8312 * @hba: per-adapter instance
8313 * @mmio_base: base register address
8314 * @irq: Interrupt line of device
8315 * Returns 0 on success, non-zero value on failure
8316 */
8317int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
8318{
8319 int err;
8320 struct Scsi_Host *host = hba->host;
8321 struct device *dev = hba->dev;
8322
8323 if (!mmio_base) {
8324 dev_err(hba->dev,
8325 "Invalid memory reference for mmio_base is NULL\n");
8326 err = -ENODEV;
8327 goto out_error;
8328 }
8329
3b1d0580
VH		 8330 hba->mmio_base = mmio_base;
8331 hba->irq = irq;
7a3e97b0 8332
aa497613 8333 err = ufshcd_hba_init(hba);
5c0c28a8
SRT		 8334 if (err)
8335 goto out_error;
8336
7a3e97b0
SY		 8337 /* Read capabilities registers */
8338 ufshcd_hba_capabilities(hba);
8339
8340 /* Get UFS version supported by the controller */
8341 hba->ufs_version = ufshcd_get_ufs_version(hba);
8342
c01848c6
YG		 8343 if ((hba->ufs_version != UFSHCI_VERSION_10) &&
8344 (hba->ufs_version != UFSHCI_VERSION_11) &&
8345 (hba->ufs_version != UFSHCI_VERSION_20) &&
8346 (hba->ufs_version != UFSHCI_VERSION_21))
8347 dev_err(hba->dev, "invalid UFS version 0x%x\n",
8348 hba->ufs_version);
8349
2fbd009b
SJ		 8350 /* Get Interrupt bit mask per version */
8351 hba->intr_mask = ufshcd_get_intr_mask(hba);
8352
ca3d7bf9
AM		 8353 err = ufshcd_set_dma_mask(hba);
8354 if (err) {
8355 dev_err(hba->dev, "set dma mask failed\n");
8356 goto out_disable;
8357 }
8358
7a3e97b0
SY		 8359 /* Allocate memory for host memory space */
8360 err = ufshcd_memory_alloc(hba);
8361 if (err) {
3b1d0580
VH		 8362 dev_err(hba->dev, "Memory allocation failed\n");
8363 goto out_disable;
7a3e97b0
SY		 8364 }
8365
8366 /* Configure LRB */
8367 ufshcd_host_memory_configure(hba);
8368
8369 host->can_queue = hba->nutrs;
8370 host->cmd_per_lun = hba->nutrs;
8371 host->max_id = UFSHCD_MAX_ID;
0ce147d4 8372 host->max_lun = UFS_MAX_LUNS;
7a3e97b0
SY		 8373 host->max_channel = UFSHCD_MAX_CHANNEL;
8374 host->unique_id = host->host_no;
a851b2bd 8375 host->max_cmd_len = UFS_CDB_SIZE;
7a3e97b0 8376
7eb584db
DR		 8377 hba->max_pwr_info.is_valid = false;
8378
7a3e97b0 8379 /* Initialize work queues */
e8e7f271 8380 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
66ec6d59 8381 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
7a3e97b0 8382
6ccf44fe
SJ		 8383 /* Initialize UIC command mutex */
8384 mutex_init(&hba->uic_cmd_mutex);
8385
5a0b0cb9
SRT		 8386 /* Initialize mutex for device management commands */
8387 mutex_init(&hba->dev_cmd.lock);
8388
a3cd5ec5
SJ		 8389 init_rwsem(&hba->clk_scaling_lock);
8390
1ab27c9c 8391 ufshcd_init_clk_gating(hba);
199ef13c 8392
eebcc196
VG		 8393 ufshcd_init_clk_scaling(hba);
8394
199ef13c
YG		 8395 /*
8396 * In order to avoid any spurious interrupt immediately after
8397 * registering UFS controller interrupt handler, clear any pending UFS
8398 * interrupt status and disable all the UFS interrupts.
8399 */
8400 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
8401 REG_INTERRUPT_STATUS);
8402 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
8403 /*
8404 * Make sure that UFS interrupts are disabled and any pending interrupt
8405 * status is cleared before registering UFS interrupt handler.
8406 */
8407 mb();
8408
7a3e97b0 8409 /* IRQ registration */
2953f850 8410 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
7a3e97b0 8411 if (err) {
3b1d0580 8412 dev_err(hba->dev, "request irq failed\n");
1ab27c9c 8413 goto exit_gating;
57d104c1
SJ		 8414 } else {
8415 hba->is_irq_enabled = true;
7a3e97b0
SY		 8416 }
8417
3b1d0580 8418 err = scsi_add_host(host, hba->dev);
7a3e97b0 8419 if (err) {
3b1d0580 8420 dev_err(hba->dev, "scsi_add_host failed\n");
1ab27c9c 8421 goto exit_gating;
7a3e97b0
SY		 8422 }
8423
7252a360
BVA		 8424 hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set);
8425 if (IS_ERR(hba->cmd_queue)) {
8426 err = PTR_ERR(hba->cmd_queue);
8427 goto out_remove_scsi_host;
8428 }
8429
69a6c269
BVA		 8430 hba->tmf_tag_set = (struct blk_mq_tag_set) {
8431 .nr_hw_queues = 1,
8432 .queue_depth = hba->nutmrs,
8433 .ops = &ufshcd_tmf_ops,
8434 .flags = BLK_MQ_F_NO_SCHED,
8435 };
8436 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
8437 if (err < 0)
8438 goto free_cmd_queue;
8439 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
8440 if (IS_ERR(hba->tmf_queue)) {
8441 err = PTR_ERR(hba->tmf_queue);
8442 goto free_tmf_tag_set;
8443 }
8444
d8d9f793
BA		 8445 /* Reset the attached device */
8446 ufshcd_vops_device_reset(hba);
8447
6ccf44fe
SJ		 8448 /* Host controller enable */
8449 err = ufshcd_hba_enable(hba);
7a3e97b0 8450 if (err) {
6ccf44fe 8451 dev_err(hba->dev, "Host controller enable failed\n");
66cc820f 8452 ufshcd_print_host_regs(hba);
6ba65588 8453 ufshcd_print_host_state(hba);
69a6c269 8454 goto free_tmf_queue;
7a3e97b0 8455 }
6ccf44fe 8456
0c8f7586
SJ		 8457 /*
8458 * Set the default power management level for runtime and system PM.
8459 * Default power saving mode is to keep UFS link in Hibern8 state
8460 * and UFS device in sleep state.
8461 */
8462 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8463 UFS_SLEEP_PWR_MODE,
8464 UIC_LINK_HIBERN8_STATE);
8465 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8466 UFS_SLEEP_PWR_MODE,
8467 UIC_LINK_HIBERN8_STATE);
8468
ad448378 8469 /* Set the default auto-hiberate idle timer value to 150 ms */
f571b377 8470 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
ad448378
AH		 8471 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
8472 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
8473 }
8474
62694735
SRT		 8475 /* Hold auto suspend until async scan completes */
8476 pm_runtime_get_sync(dev);
38135535 8477 atomic_set(&hba->scsi_block_reqs_cnt, 0);
57d104c1 8478 /*
7caf489b
SJ		 8479 * We are assuming that device wasn't put in sleep/power-down
8480 * state exclusively during the boot stage before kernel.
8481 * This assumption helps avoid doing link startup twice during
8482 * ufshcd_probe_hba().
57d104c1 8483 */
7caf489b 8484 ufshcd_set_ufs_dev_active(hba);
57d104c1 8485
6ccf44fe 8486 async_schedule(ufshcd_async_scan, hba);
cbb6813e 8487 ufs_sysfs_add_nodes(hba->dev);
6ccf44fe 8488
7a3e97b0
SY		 8489 return 0;
8490
69a6c269
BVA		 8491free_tmf_queue:
8492 blk_cleanup_queue(hba->tmf_queue);
8493free_tmf_tag_set:
8494 blk_mq_free_tag_set(&hba->tmf_tag_set);
7252a360
BVA		 8495free_cmd_queue:
8496 blk_cleanup_queue(hba->cmd_queue);
3b1d0580
VH		 8497out_remove_scsi_host:
8498 scsi_remove_host(hba->host);
1ab27c9c 8499exit_gating:
eebcc196 8500 ufshcd_exit_clk_scaling(hba);
1ab27c9c 8501 ufshcd_exit_clk_gating(hba);
3b1d0580 8502out_disable:
57d104c1 8503 hba->is_irq_enabled = false;
aa497613 8504 ufshcd_hba_exit(hba);
3b1d0580
VH		 8505out_error:
8506 return err;
8507}
8508EXPORT_SYMBOL_GPL(ufshcd_init);
8509
3b1d0580
VH		 8510MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
8511MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
e0eca63e 8512MODULE_DESCRIPTION("Generic UFS host controller driver Core");
7a3e97b0
SY		 8513MODULE_LICENSE("GPL");
8514MODULE_VERSION(UFSHCD_DRIVER_VERSION);
Ubuntu Jammy Linux kernel mirror