2 * Universal Flash Storage Host controller driver Core
4 * This code is based on drivers/scsi/ufs/ufshcd.c
5 * Copyright (C) 2011-2013 Samsung India Software Operations
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation; either version 2
14 * of the License, or (at your option) any later version.
15 * See the COPYING file in the top-level directory or visit
16 * <http://www.gnu.org/licenses/gpl-2.0.html>
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
23 * This program is provided "AS IS" and "WITH ALL FAULTS" and
24 * without warranty of any kind. You are solely responsible for
25 * determining the appropriateness of using and distributing
26 * the program and assume all risks associated with your exercise
27 * of rights with respect to the program, including but not limited
28 * to infringement of third party rights, the risks and costs of
29 * program errors, damage to or loss of data, programs or equipment,
30 * and unavailability or interruption of operations. Under no
31 * circumstances will the contributor of this Program be liable for
32 * any damages of any kind arising from your use or distribution of
36 #include <linux/async.h>
40 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
43 /* UIC command timeout, unit: ms */
44 #define UIC_CMD_TIMEOUT 500
46 /* NOP OUT retries waiting for NOP IN response */
47 #define NOP_OUT_RETRIES 10
48 /* Timeout after 30 msecs if NOP OUT hangs without response */
49 #define NOP_OUT_TIMEOUT 30 /* msecs */
51 /* Query request retries */
52 #define QUERY_REQ_RETRIES 10
53 /* Query request timeout */
54 #define QUERY_REQ_TIMEOUT 30 /* msec */
56 /* Expose the flag value from utp_upiu_query.value */
57 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
60 UFSHCD_MAX_CHANNEL
= 0,
63 UFSHCD_CMD_PER_LUN
= 32,
64 UFSHCD_CAN_QUEUE
= 32,
69 UFSHCD_STATE_OPERATIONAL
,
74 /* Interrupt configuration options */
81 /* Interrupt aggregation options */
88 * ufshcd_wait_for_register - wait for register value to change
89 * @hba - per-adapter interface
90 * @reg - mmio register offset
91 * @mask - mask to apply to read register value
92 * @val - wait condition
93 * @interval_us - polling interval in microsecs
94 * @timeout_ms - timeout in millisecs
96 * Returns -ETIMEDOUT on error, zero on success
98 static int ufshcd_wait_for_register(struct ufs_hba
*hba
, u32 reg
, u32 mask
,
99 u32 val
, unsigned long interval_us
, unsigned long timeout_ms
)
102 unsigned long timeout
= jiffies
+ msecs_to_jiffies(timeout_ms
);
104 /* ignore bits that we don't intend to wait on */
107 while ((ufshcd_readl(hba
, reg
) & mask
) != val
) {
108 /* wakeup within 50us of expiry */
109 usleep_range(interval_us
, interval_us
+ 50);
111 if (time_after(jiffies
, timeout
)) {
112 if ((ufshcd_readl(hba
, reg
) & mask
) != val
)
122 * ufshcd_get_intr_mask - Get the interrupt bit mask
123 * @hba - Pointer to adapter instance
125 * Returns interrupt bit mask per version
127 static inline u32
ufshcd_get_intr_mask(struct ufs_hba
*hba
)
129 if (hba
->ufs_version
== UFSHCI_VERSION_10
)
130 return INTERRUPT_MASK_ALL_VER_10
;
132 return INTERRUPT_MASK_ALL_VER_11
;
136 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
137 * @hba - Pointer to adapter instance
139 * Returns UFSHCI version supported by the controller
141 static inline u32
ufshcd_get_ufs_version(struct ufs_hba
*hba
)
143 return ufshcd_readl(hba
, REG_UFS_VERSION
);
147 * ufshcd_is_device_present - Check if any device connected to
148 * the host controller
149 * @reg_hcs - host controller status register value
151 * Returns 1 if device present, 0 if no device detected
153 static inline int ufshcd_is_device_present(u32 reg_hcs
)
155 return (DEVICE_PRESENT
& reg_hcs
) ? 1 : 0;
159 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
160 * @lrb: pointer to local command reference block
162 * This function is used to get the OCS field from UTRD
163 * Returns the OCS field in the UTRD
165 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb
*lrbp
)
167 return lrbp
->utr_descriptor_ptr
->header
.dword_2
& MASK_OCS
;
171 * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
172 * @task_req_descp: pointer to utp_task_req_desc structure
174 * This function is used to get the OCS field from UTMRD
175 * Returns the OCS field in the UTMRD
178 ufshcd_get_tmr_ocs(struct utp_task_req_desc
*task_req_descp
)
180 return task_req_descp
->header
.dword_2
& MASK_OCS
;
184 * ufshcd_get_tm_free_slot - get a free slot for task management request
185 * @hba: per adapter instance
187 * Returns maximum number of task management request slots in case of
188 * task management queue full or returns the free slot number
190 static inline int ufshcd_get_tm_free_slot(struct ufs_hba
*hba
)
192 return find_first_zero_bit(&hba
->outstanding_tasks
, hba
->nutmrs
);
196 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
197 * @hba: per adapter instance
198 * @pos: position of the bit to be cleared
200 static inline void ufshcd_utrl_clear(struct ufs_hba
*hba
, u32 pos
)
202 ufshcd_writel(hba
, ~(1 << pos
), REG_UTP_TRANSFER_REQ_LIST_CLEAR
);
206 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
207 * @reg: Register value of host controller status
209 * Returns integer, 0 on Success and positive value if failed
211 static inline int ufshcd_get_lists_status(u32 reg
)
214 * The mask 0xFF is for the following HCS register bits
224 return (((reg
) & (0xFF)) >> 1) ^ (0x07);
228 * ufshcd_get_uic_cmd_result - Get the UIC command result
229 * @hba: Pointer to adapter instance
231 * This function gets the result of UIC command completion
232 * Returns 0 on success, non zero value on error
234 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba
*hba
)
236 return ufshcd_readl(hba
, REG_UIC_COMMAND_ARG_2
) &
237 MASK_UIC_COMMAND_RESULT
;
241 * ufshcd_get_req_rsp - returns the TR response transaction type
242 * @ucd_rsp_ptr: pointer to response UPIU
245 ufshcd_get_req_rsp(struct utp_upiu_rsp
*ucd_rsp_ptr
)
247 return be32_to_cpu(ucd_rsp_ptr
->header
.dword_0
) >> 24;
251 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
252 * @ucd_rsp_ptr: pointer to response UPIU
254 * This function gets the response status and scsi_status from response UPIU
255 * Returns the response result code.
258 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp
*ucd_rsp_ptr
)
260 return be32_to_cpu(ucd_rsp_ptr
->header
.dword_1
) & MASK_RSP_UPIU_RESULT
;
264 * ufshcd_is_exception_event - Check if the device raised an exception event
265 * @ucd_rsp_ptr: pointer to response UPIU
267 * The function checks if the device raised an exception event indicated in
268 * the Device Information field of response UPIU.
270 * Returns true if exception is raised, false otherwise.
272 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp
*ucd_rsp_ptr
)
274 return be32_to_cpu(ucd_rsp_ptr
->header
.dword_2
) &
275 MASK_RSP_EXCEPTION_EVENT
? true : false;
279 * ufshcd_config_int_aggr - Configure interrupt aggregation values.
280 * Currently there is no use case where we want to configure
281 * interrupt aggregation dynamically. So to configure interrupt
282 * aggregation, #define INT_AGGR_COUNTER_THRESHOLD_VALUE and
283 * INT_AGGR_TIMEOUT_VALUE are used.
284 * @hba: per adapter instance
285 * @option: Interrupt aggregation option
288 ufshcd_config_int_aggr(struct ufs_hba
*hba
, int option
)
292 ufshcd_writel(hba
, INT_AGGR_ENABLE
|
293 INT_AGGR_COUNTER_AND_TIMER_RESET
,
294 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL
);
296 case INT_AGGR_CONFIG
:
297 ufshcd_writel(hba
, INT_AGGR_ENABLE
| INT_AGGR_PARAM_WRITE
|
298 INT_AGGR_COUNTER_THRESHOLD_VALUE
|
299 INT_AGGR_TIMEOUT_VALUE
,
300 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL
);
306 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
307 * When run-stop registers are set to 1, it indicates the
308 * host controller that it can process the requests
309 * @hba: per adapter instance
311 static void ufshcd_enable_run_stop_reg(struct ufs_hba
*hba
)
313 ufshcd_writel(hba
, UTP_TASK_REQ_LIST_RUN_STOP_BIT
,
314 REG_UTP_TASK_REQ_LIST_RUN_STOP
);
315 ufshcd_writel(hba
, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT
,
316 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP
);
320 * ufshcd_hba_start - Start controller initialization sequence
321 * @hba: per adapter instance
323 static inline void ufshcd_hba_start(struct ufs_hba
*hba
)
325 ufshcd_writel(hba
, CONTROLLER_ENABLE
, REG_CONTROLLER_ENABLE
);
329 * ufshcd_is_hba_active - Get controller state
330 * @hba: per adapter instance
332 * Returns zero if controller is active, 1 otherwise
334 static inline int ufshcd_is_hba_active(struct ufs_hba
*hba
)
336 return (ufshcd_readl(hba
, REG_CONTROLLER_ENABLE
) & 0x1) ? 0 : 1;
340 * ufshcd_send_command - Send SCSI or device management commands
341 * @hba: per adapter instance
342 * @task_tag: Task tag of the command
345 void ufshcd_send_command(struct ufs_hba
*hba
, unsigned int task_tag
)
347 __set_bit(task_tag
, &hba
->outstanding_reqs
);
348 ufshcd_writel(hba
, 1 << task_tag
, REG_UTP_TRANSFER_REQ_DOOR_BELL
);
352 * ufshcd_copy_sense_data - Copy sense data in case of check condition
353 * @lrb - pointer to local reference block
355 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb
*lrbp
)
358 if (lrbp
->sense_buffer
) {
359 len
= be16_to_cpu(lrbp
->ucd_rsp_ptr
->sr
.sense_data_len
);
360 memcpy(lrbp
->sense_buffer
,
361 lrbp
->ucd_rsp_ptr
->sr
.sense_data
,
362 min_t(int, len
, SCSI_SENSE_BUFFERSIZE
));
367 * ufshcd_query_to_cpu() - formats the buffer to native cpu endian
368 * @response: upiu query response to convert
370 static inline void ufshcd_query_to_cpu(struct utp_upiu_query
*response
)
372 response
->length
= be16_to_cpu(response
->length
);
373 response
->value
= be32_to_cpu(response
->value
);
377 * ufshcd_query_to_be() - formats the buffer to big endian
378 * @request: upiu query request to convert
380 static inline void ufshcd_query_to_be(struct utp_upiu_query
*request
)
382 request
->length
= cpu_to_be16(request
->length
);
383 request
->value
= cpu_to_be32(request
->value
);
387 * ufshcd_copy_query_response() - Copy the Query Response and the data
389 * @hba: per adapter instance
390 * @lrb - pointer to local reference block
393 void ufshcd_copy_query_response(struct ufs_hba
*hba
, struct ufshcd_lrb
*lrbp
)
395 struct ufs_query_res
*query_res
= &hba
->dev_cmd
.query
.response
;
397 /* Get the UPIU response */
398 query_res
->response
= ufshcd_get_rsp_upiu_result(lrbp
->ucd_rsp_ptr
) >>
399 UPIU_RSP_CODE_OFFSET
;
401 memcpy(&query_res
->upiu_res
, &lrbp
->ucd_rsp_ptr
->qr
, QUERY_OSF_SIZE
);
402 ufshcd_query_to_cpu(&query_res
->upiu_res
);
405 /* Get the descriptor */
406 if (lrbp
->ucd_rsp_ptr
->qr
.opcode
== UPIU_QUERY_OPCODE_READ_DESC
) {
407 u8
*descp
= (u8
*)&lrbp
->ucd_rsp_ptr
+
408 GENERAL_UPIU_REQUEST_SIZE
;
411 /* data segment length */
412 len
= be32_to_cpu(lrbp
->ucd_rsp_ptr
->header
.dword_2
) &
413 MASK_QUERY_DATA_SEG_LEN
;
415 memcpy(hba
->dev_cmd
.query
.descriptor
, descp
,
416 min_t(u16
, len
, QUERY_DESC_MAX_SIZE
));
421 * ufshcd_hba_capabilities - Read controller capabilities
422 * @hba: per adapter instance
424 static inline void ufshcd_hba_capabilities(struct ufs_hba
*hba
)
426 hba
->capabilities
= ufshcd_readl(hba
, REG_CONTROLLER_CAPABILITIES
);
428 /* nutrs and nutmrs are 0 based values */
429 hba
->nutrs
= (hba
->capabilities
& MASK_TRANSFER_REQUESTS_SLOTS
) + 1;
431 ((hba
->capabilities
& MASK_TASK_MANAGEMENT_REQUEST_SLOTS
) >> 16) + 1;
435 * ufshcd_ready_for_uic_cmd - Check if controller is ready
436 * to accept UIC commands
437 * @hba: per adapter instance
438 * Return true on success, else false
440 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba
*hba
)
442 if (ufshcd_readl(hba
, REG_CONTROLLER_STATUS
) & UIC_COMMAND_READY
)
449 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
450 * @hba: per adapter instance
451 * @uic_cmd: UIC command
453 * Mutex must be held.
456 ufshcd_dispatch_uic_cmd(struct ufs_hba
*hba
, struct uic_command
*uic_cmd
)
458 WARN_ON(hba
->active_uic_cmd
);
460 hba
->active_uic_cmd
= uic_cmd
;
463 ufshcd_writel(hba
, uic_cmd
->argument1
, REG_UIC_COMMAND_ARG_1
);
464 ufshcd_writel(hba
, uic_cmd
->argument2
, REG_UIC_COMMAND_ARG_2
);
465 ufshcd_writel(hba
, uic_cmd
->argument3
, REG_UIC_COMMAND_ARG_3
);
468 ufshcd_writel(hba
, uic_cmd
->command
& COMMAND_OPCODE_MASK
,
473 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
474 * @hba: per adapter instance
475 * @uic_command: UIC command
477 * Must be called with mutex held.
478 * Returns 0 only if success.
481 ufshcd_wait_for_uic_cmd(struct ufs_hba
*hba
, struct uic_command
*uic_cmd
)
486 if (wait_for_completion_timeout(&uic_cmd
->done
,
487 msecs_to_jiffies(UIC_CMD_TIMEOUT
)))
488 ret
= uic_cmd
->argument2
& MASK_UIC_COMMAND_RESULT
;
492 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
493 hba
->active_uic_cmd
= NULL
;
494 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
500 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
501 * @hba: per adapter instance
502 * @uic_cmd: UIC command
504 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
506 * Returns 0 only if success.
509 __ufshcd_send_uic_cmd(struct ufs_hba
*hba
, struct uic_command
*uic_cmd
)
514 if (!ufshcd_ready_for_uic_cmd(hba
)) {
516 "Controller not ready to accept UIC commands\n");
520 init_completion(&uic_cmd
->done
);
522 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
523 ufshcd_dispatch_uic_cmd(hba
, uic_cmd
);
524 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
526 ret
= ufshcd_wait_for_uic_cmd(hba
, uic_cmd
);
532 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
533 * @hba: per adapter instance
534 * @uic_cmd: UIC command
536 * Returns 0 only if success.
539 ufshcd_send_uic_cmd(struct ufs_hba
*hba
, struct uic_command
*uic_cmd
)
543 mutex_lock(&hba
->uic_cmd_mutex
);
544 ret
= __ufshcd_send_uic_cmd(hba
, uic_cmd
);
545 mutex_unlock(&hba
->uic_cmd_mutex
);
551 * ufshcd_map_sg - Map scatter-gather list to prdt
552 * @lrbp - pointer to local reference block
554 * Returns 0 in case of success, non-zero value in case of failure
556 static int ufshcd_map_sg(struct ufshcd_lrb
*lrbp
)
558 struct ufshcd_sg_entry
*prd_table
;
559 struct scatterlist
*sg
;
560 struct scsi_cmnd
*cmd
;
565 sg_segments
= scsi_dma_map(cmd
);
570 lrbp
->utr_descriptor_ptr
->prd_table_length
=
571 cpu_to_le16((u16
) (sg_segments
));
573 prd_table
= (struct ufshcd_sg_entry
*)lrbp
->ucd_prdt_ptr
;
575 scsi_for_each_sg(cmd
, sg
, sg_segments
, i
) {
577 cpu_to_le32(((u32
) sg_dma_len(sg
))-1);
578 prd_table
[i
].base_addr
=
579 cpu_to_le32(lower_32_bits(sg
->dma_address
));
580 prd_table
[i
].upper_addr
=
581 cpu_to_le32(upper_32_bits(sg
->dma_address
));
584 lrbp
->utr_descriptor_ptr
->prd_table_length
= 0;
591 * ufshcd_enable_intr - enable interrupts
592 * @hba: per adapter instance
593 * @intrs: interrupt bits
595 static void ufshcd_enable_intr(struct ufs_hba
*hba
, u32 intrs
)
597 u32 set
= ufshcd_readl(hba
, REG_INTERRUPT_ENABLE
);
599 if (hba
->ufs_version
== UFSHCI_VERSION_10
) {
601 rw
= set
& INTERRUPT_MASK_RW_VER_10
;
602 set
= rw
| ((set
^ intrs
) & intrs
);
607 ufshcd_writel(hba
, set
, REG_INTERRUPT_ENABLE
);
611 * ufshcd_disable_intr - disable interrupts
612 * @hba: per adapter instance
613 * @intrs: interrupt bits
615 static void ufshcd_disable_intr(struct ufs_hba
*hba
, u32 intrs
)
617 u32 set
= ufshcd_readl(hba
, REG_INTERRUPT_ENABLE
);
619 if (hba
->ufs_version
== UFSHCI_VERSION_10
) {
621 rw
= (set
& INTERRUPT_MASK_RW_VER_10
) &
622 ~(intrs
& INTERRUPT_MASK_RW_VER_10
);
623 set
= rw
| ((set
& intrs
) & ~INTERRUPT_MASK_RW_VER_10
);
629 ufshcd_writel(hba
, set
, REG_INTERRUPT_ENABLE
);
633 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
634 * descriptor according to request
635 * @lrbp: pointer to local reference block
636 * @upiu_flags: flags required in the header
637 * @cmd_dir: requests data direction
639 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb
*lrbp
,
640 u32
*upiu_flags
, enum dma_data_direction cmd_dir
)
642 struct utp_transfer_req_desc
*req_desc
= lrbp
->utr_descriptor_ptr
;
646 if (cmd_dir
== DMA_FROM_DEVICE
) {
647 data_direction
= UTP_DEVICE_TO_HOST
;
648 *upiu_flags
= UPIU_CMD_FLAGS_READ
;
649 } else if (cmd_dir
== DMA_TO_DEVICE
) {
650 data_direction
= UTP_HOST_TO_DEVICE
;
651 *upiu_flags
= UPIU_CMD_FLAGS_WRITE
;
653 data_direction
= UTP_NO_DATA_TRANSFER
;
654 *upiu_flags
= UPIU_CMD_FLAGS_NONE
;
657 dword_0
= data_direction
| (lrbp
->command_type
658 << UPIU_COMMAND_TYPE_OFFSET
);
660 dword_0
|= UTP_REQ_DESC_INT_CMD
;
662 /* Transfer request descriptor header fields */
663 req_desc
->header
.dword_0
= cpu_to_le32(dword_0
);
666 * assigning invalid value for command status. Controller
667 * updates OCS on command completion, with the command
670 req_desc
->header
.dword_2
=
671 cpu_to_le32(OCS_INVALID_COMMAND_STATUS
);
675 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
677 * @lrbp - local reference block pointer
678 * @upiu_flags - flags
681 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb
*lrbp
, u32 upiu_flags
)
683 struct utp_upiu_req
*ucd_req_ptr
= lrbp
->ucd_req_ptr
;
685 /* command descriptor fields */
686 ucd_req_ptr
->header
.dword_0
= UPIU_HEADER_DWORD(
687 UPIU_TRANSACTION_COMMAND
, upiu_flags
,
688 lrbp
->lun
, lrbp
->task_tag
);
689 ucd_req_ptr
->header
.dword_1
= UPIU_HEADER_DWORD(
690 UPIU_COMMAND_SET_TYPE_SCSI
, 0, 0, 0);
692 /* Total EHS length and Data segment length will be zero */
693 ucd_req_ptr
->header
.dword_2
= 0;
695 ucd_req_ptr
->sc
.exp_data_transfer_len
=
696 cpu_to_be32(lrbp
->cmd
->sdb
.length
);
698 memcpy(ucd_req_ptr
->sc
.cdb
, lrbp
->cmd
->cmnd
,
699 (min_t(unsigned short, lrbp
->cmd
->cmd_len
, MAX_CDB_SIZE
)));
703 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
706 * @lrbp: local reference block pointer
709 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba
*hba
,
710 struct ufshcd_lrb
*lrbp
, u32 upiu_flags
)
712 struct utp_upiu_req
*ucd_req_ptr
= lrbp
->ucd_req_ptr
;
713 struct ufs_query
*query
= &hba
->dev_cmd
.query
;
714 u16 len
= query
->request
.upiu_req
.length
;
715 u8
*descp
= (u8
*)lrbp
->ucd_req_ptr
+ GENERAL_UPIU_REQUEST_SIZE
;
717 /* Query request header */
718 ucd_req_ptr
->header
.dword_0
= UPIU_HEADER_DWORD(
719 UPIU_TRANSACTION_QUERY_REQ
, upiu_flags
,
720 lrbp
->lun
, lrbp
->task_tag
);
721 ucd_req_ptr
->header
.dword_1
= UPIU_HEADER_DWORD(
722 0, query
->request
.query_func
, 0, 0);
724 /* Data segment length */
725 ucd_req_ptr
->header
.dword_2
= UPIU_HEADER_DWORD(
726 0, 0, len
>> 8, (u8
)len
);
728 /* Copy the Query Request buffer as is */
729 memcpy(&ucd_req_ptr
->qr
, &query
->request
.upiu_req
,
731 ufshcd_query_to_be(&ucd_req_ptr
->qr
);
733 /* Copy the Descriptor */
734 if ((len
> 0) && (query
->request
.upiu_req
.opcode
==
735 UPIU_QUERY_OPCODE_WRITE_DESC
)) {
736 memcpy(descp
, query
->descriptor
,
737 min_t(u16
, len
, QUERY_DESC_MAX_SIZE
));
741 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb
*lrbp
)
743 struct utp_upiu_req
*ucd_req_ptr
= lrbp
->ucd_req_ptr
;
745 memset(ucd_req_ptr
, 0, sizeof(struct utp_upiu_req
));
747 /* command descriptor fields */
748 ucd_req_ptr
->header
.dword_0
=
750 UPIU_TRANSACTION_NOP_OUT
, 0, 0, lrbp
->task_tag
);
754 * ufshcd_compose_upiu - form UFS Protocol Information Unit(UPIU)
755 * @hba - per adapter instance
756 * @lrb - pointer to local reference block
758 static int ufshcd_compose_upiu(struct ufs_hba
*hba
, struct ufshcd_lrb
*lrbp
)
763 switch (lrbp
->command_type
) {
764 case UTP_CMD_TYPE_SCSI
:
765 if (likely(lrbp
->cmd
)) {
766 ufshcd_prepare_req_desc_hdr(lrbp
, &upiu_flags
,
767 lrbp
->cmd
->sc_data_direction
);
768 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp
, upiu_flags
);
773 case UTP_CMD_TYPE_DEV_MANAGE
:
774 ufshcd_prepare_req_desc_hdr(lrbp
, &upiu_flags
, DMA_NONE
);
775 if (hba
->dev_cmd
.type
== DEV_CMD_TYPE_QUERY
)
776 ufshcd_prepare_utp_query_req_upiu(
777 hba
, lrbp
, upiu_flags
);
778 else if (hba
->dev_cmd
.type
== DEV_CMD_TYPE_NOP
)
779 ufshcd_prepare_utp_nop_upiu(lrbp
);
783 case UTP_CMD_TYPE_UFS
:
784 /* For UFS native command implementation */
786 dev_err(hba
->dev
, "%s: UFS native command are not supported\n",
791 dev_err(hba
->dev
, "%s: unknown command type: 0x%x\n",
792 __func__
, lrbp
->command_type
);
794 } /* end of switch */
800 * ufshcd_queuecommand - main entry point for SCSI requests
801 * @cmd: command from SCSI Midlayer
802 * @done: call back function
804 * Returns 0 for success, non-zero in case of failure
806 static int ufshcd_queuecommand(struct Scsi_Host
*host
, struct scsi_cmnd
*cmd
)
808 struct ufshcd_lrb
*lrbp
;
814 hba
= shost_priv(host
);
816 tag
= cmd
->request
->tag
;
818 if (hba
->ufshcd_state
!= UFSHCD_STATE_OPERATIONAL
) {
819 err
= SCSI_MLQUEUE_HOST_BUSY
;
823 /* acquire the tag to make sure device cmds don't use it */
824 if (test_and_set_bit_lock(tag
, &hba
->lrb_in_use
)) {
826 * Dev manage command in progress, requeue the command.
827 * Requeuing the command helps in cases where the request *may*
828 * find different tag instead of waiting for dev manage command
831 err
= SCSI_MLQUEUE_HOST_BUSY
;
835 lrbp
= &hba
->lrb
[tag
];
839 lrbp
->sense_bufflen
= SCSI_SENSE_BUFFERSIZE
;
840 lrbp
->sense_buffer
= cmd
->sense_buffer
;
841 lrbp
->task_tag
= tag
;
842 lrbp
->lun
= cmd
->device
->lun
;
843 lrbp
->intr_cmd
= false;
844 lrbp
->command_type
= UTP_CMD_TYPE_SCSI
;
846 /* form UPIU before issuing the command */
847 ufshcd_compose_upiu(hba
, lrbp
);
848 err
= ufshcd_map_sg(lrbp
);
851 clear_bit_unlock(tag
, &hba
->lrb_in_use
);
855 /* issue command to the controller */
856 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
857 ufshcd_send_command(hba
, tag
);
858 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
863 static int ufshcd_compose_dev_cmd(struct ufs_hba
*hba
,
864 struct ufshcd_lrb
*lrbp
, enum dev_cmd_type cmd_type
, int tag
)
867 lrbp
->sense_bufflen
= 0;
868 lrbp
->sense_buffer
= NULL
;
869 lrbp
->task_tag
= tag
;
870 lrbp
->lun
= 0; /* device management cmd is not specific to any LUN */
871 lrbp
->command_type
= UTP_CMD_TYPE_DEV_MANAGE
;
872 lrbp
->intr_cmd
= true; /* No interrupt aggregation */
873 hba
->dev_cmd
.type
= cmd_type
;
875 return ufshcd_compose_upiu(hba
, lrbp
);
879 ufshcd_clear_cmd(struct ufs_hba
*hba
, int tag
)
885 /* clear outstanding transaction before retry */
886 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
887 ufshcd_utrl_clear(hba
, tag
);
888 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
891 * wait for for h/w to clear corresponding bit in door-bell.
892 * max. wait is 1 sec.
894 err
= ufshcd_wait_for_register(hba
,
895 REG_UTP_TRANSFER_REQ_DOOR_BELL
,
896 mask
, ~mask
, 1000, 1000);
902 * ufshcd_dev_cmd_completion() - handles device management command responses
903 * @hba: per adapter instance
904 * @lrbp: pointer to local reference block
907 ufshcd_dev_cmd_completion(struct ufs_hba
*hba
, struct ufshcd_lrb
*lrbp
)
912 resp
= ufshcd_get_req_rsp(lrbp
->ucd_rsp_ptr
);
915 case UPIU_TRANSACTION_NOP_IN
:
916 if (hba
->dev_cmd
.type
!= DEV_CMD_TYPE_NOP
) {
918 dev_err(hba
->dev
, "%s: unexpected response %x\n",
922 case UPIU_TRANSACTION_QUERY_RSP
:
923 ufshcd_copy_query_response(hba
, lrbp
);
925 case UPIU_TRANSACTION_REJECT_UPIU
:
926 /* TODO: handle Reject UPIU Response */
928 dev_err(hba
->dev
, "%s: Reject UPIU not fully implemented\n",
933 dev_err(hba
->dev
, "%s: Invalid device management cmd response: %x\n",
941 static int ufshcd_wait_for_dev_cmd(struct ufs_hba
*hba
,
942 struct ufshcd_lrb
*lrbp
, int max_timeout
)
945 unsigned long time_left
;
948 time_left
= wait_for_completion_timeout(hba
->dev_cmd
.complete
,
949 msecs_to_jiffies(max_timeout
));
951 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
952 hba
->dev_cmd
.complete
= NULL
;
953 if (likely(time_left
)) {
954 err
= ufshcd_get_tr_ocs(lrbp
);
956 err
= ufshcd_dev_cmd_completion(hba
, lrbp
);
958 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
962 if (!ufshcd_clear_cmd(hba
, lrbp
->task_tag
))
963 /* sucessfully cleared the command, retry if needed */
971 * ufshcd_get_dev_cmd_tag - Get device management command tag
972 * @hba: per-adapter instance
973 * @tag: pointer to variable with available slot value
975 * Get a free slot and lock it until device management command
978 * Returns false if free slot is unavailable for locking, else
979 * return true with tag value in @tag.
981 static bool ufshcd_get_dev_cmd_tag(struct ufs_hba
*hba
, int *tag_out
)
991 tmp
= ~hba
->lrb_in_use
;
992 tag
= find_last_bit(&tmp
, hba
->nutrs
);
993 if (tag
>= hba
->nutrs
)
995 } while (test_and_set_bit_lock(tag
, &hba
->lrb_in_use
));
1003 static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba
*hba
, int tag
)
1005 clear_bit_unlock(tag
, &hba
->lrb_in_use
);
1009 * ufshcd_exec_dev_cmd - API for sending device management requests
1011 * @cmd_type - specifies the type (NOP, Query...)
1012 * @timeout - time in seconds
1014 * NOTE: Since there is only one available tag for device management commands,
1015 * it is expected you hold the hba->dev_cmd.lock mutex.
1017 static int ufshcd_exec_dev_cmd(struct ufs_hba
*hba
,
1018 enum dev_cmd_type cmd_type
, int timeout
)
1020 struct ufshcd_lrb
*lrbp
;
1023 struct completion wait
;
1024 unsigned long flags
;
1027 * Get free slot, sleep if slots are unavailable.
1028 * Even though we use wait_event() which sleeps indefinitely,
1029 * the maximum wait time is bounded by SCSI request timeout.
1031 wait_event(hba
->dev_cmd
.tag_wq
, ufshcd_get_dev_cmd_tag(hba
, &tag
));
1033 init_completion(&wait
);
1034 lrbp
= &hba
->lrb
[tag
];
1036 err
= ufshcd_compose_dev_cmd(hba
, lrbp
, cmd_type
, tag
);
1040 hba
->dev_cmd
.complete
= &wait
;
1042 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
1043 ufshcd_send_command(hba
, tag
);
1044 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
1046 err
= ufshcd_wait_for_dev_cmd(hba
, lrbp
, timeout
);
1049 ufshcd_put_dev_cmd_tag(hba
, tag
);
1050 wake_up(&hba
->dev_cmd
.tag_wq
);
1055 * ufshcd_query_flag() - API function for sending flag query requests
1056 * hba: per-adapter instance
1057 * query_opcode: flag query to perform
1058 * idn: flag idn to access
1059 * flag_res: the flag value after the query request completes
1061 * Returns 0 for success, non-zero in case of failure
1063 static int ufshcd_query_flag(struct ufs_hba
*hba
, enum query_opcode opcode
,
1064 enum flag_idn idn
, bool *flag_res
)
1066 struct ufs_query_req
*request
;
1067 struct ufs_query_res
*response
;
1072 mutex_lock(&hba
->dev_cmd
.lock
);
1073 request
= &hba
->dev_cmd
.query
.request
;
1074 response
= &hba
->dev_cmd
.query
.response
;
1075 memset(request
, 0, sizeof(struct ufs_query_req
));
1076 memset(response
, 0, sizeof(struct ufs_query_res
));
1079 case UPIU_QUERY_OPCODE_SET_FLAG
:
1080 case UPIU_QUERY_OPCODE_CLEAR_FLAG
:
1081 case UPIU_QUERY_OPCODE_TOGGLE_FLAG
:
1082 request
->query_func
= UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST
;
1084 case UPIU_QUERY_OPCODE_READ_FLAG
:
1085 request
->query_func
= UPIU_QUERY_FUNC_STANDARD_READ_REQUEST
;
1087 /* No dummy reads */
1088 dev_err(hba
->dev
, "%s: Invalid argument for read request\n",
1096 "%s: Expected query flag opcode but got = %d\n",
1101 request
->upiu_req
.opcode
= opcode
;
1102 request
->upiu_req
.idn
= idn
;
1104 /* Send query request */
1105 err
= ufshcd_exec_dev_cmd(hba
, DEV_CMD_TYPE_QUERY
,
1110 "%s: Sending flag query for idn %d failed, err = %d\n",
1111 __func__
, idn
, err
);
1116 *flag_res
= (response
->upiu_res
.value
&
1117 MASK_QUERY_UPIU_FLAG_LOC
) & 0x1;
1120 mutex_unlock(&hba
->dev_cmd
.lock
);
1125 * ufshcd_query_attr - API function for sending attribute requests
1126 * hba: per-adapter instance
1127 * opcode: attribute opcode
1128 * idn: attribute idn to access
1129 * index: index field
1130 * selector: selector field
1131 * attr_val: the attribute value after the query request completes
1133 * Returns 0 for success, non-zero in case of failure
1135 int ufshcd_query_attr(struct ufs_hba
*hba
, enum query_opcode opcode
,
1136 enum attr_idn idn
, u8 index
, u8 selector
, u32
*attr_val
)
1138 struct ufs_query_req
*request
;
1139 struct ufs_query_res
*response
;
1145 dev_err(hba
->dev
, "%s: attribute value required for opcode 0x%x\n",
1151 mutex_lock(&hba
->dev_cmd
.lock
);
1152 request
= &hba
->dev_cmd
.query
.request
;
1153 response
= &hba
->dev_cmd
.query
.response
;
1154 memset(request
, 0, sizeof(struct ufs_query_req
));
1155 memset(response
, 0, sizeof(struct ufs_query_res
));
1158 case UPIU_QUERY_OPCODE_WRITE_ATTR
:
1159 request
->query_func
= UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST
;
1160 request
->upiu_req
.value
= *attr_val
;
1162 case UPIU_QUERY_OPCODE_READ_ATTR
:
1163 request
->query_func
= UPIU_QUERY_FUNC_STANDARD_READ_REQUEST
;
1166 dev_err(hba
->dev
, "%s: Expected query attr opcode but got = 0x%.2x\n",
1172 request
->upiu_req
.opcode
= opcode
;
1173 request
->upiu_req
.idn
= idn
;
1174 request
->upiu_req
.index
= index
;
1175 request
->upiu_req
.selector
= selector
;
1177 /* Send query request */
1178 err
= ufshcd_exec_dev_cmd(hba
, DEV_CMD_TYPE_QUERY
,
1182 dev_err(hba
->dev
, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
1183 __func__
, opcode
, idn
, err
);
1187 *attr_val
= response
->upiu_res
.value
;
1190 mutex_unlock(&hba
->dev_cmd
.lock
);
1196 * ufshcd_memory_alloc - allocate memory for host memory space data structures
1197 * @hba: per adapter instance
1199 * 1. Allocate DMA memory for Command Descriptor array
1200 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
1201 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
1202 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
1204 * 4. Allocate memory for local reference block(lrb).
1206 * Returns 0 for success, non-zero in case of failure
1208 static int ufshcd_memory_alloc(struct ufs_hba
*hba
)
1210 size_t utmrdl_size
, utrdl_size
, ucdl_size
;
1212 /* Allocate memory for UTP command descriptors */
1213 ucdl_size
= (sizeof(struct utp_transfer_cmd_desc
) * hba
->nutrs
);
1214 hba
->ucdl_base_addr
= dmam_alloc_coherent(hba
->dev
,
1216 &hba
->ucdl_dma_addr
,
1220 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
1221 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
1222 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
1223 * be aligned to 128 bytes as well
1225 if (!hba
->ucdl_base_addr
||
1226 WARN_ON(hba
->ucdl_dma_addr
& (PAGE_SIZE
- 1))) {
1228 "Command Descriptor Memory allocation failed\n");
1233 * Allocate memory for UTP Transfer descriptors
1234 * UFSHCI requires 1024 byte alignment of UTRD
1236 utrdl_size
= (sizeof(struct utp_transfer_req_desc
) * hba
->nutrs
);
1237 hba
->utrdl_base_addr
= dmam_alloc_coherent(hba
->dev
,
1239 &hba
->utrdl_dma_addr
,
1241 if (!hba
->utrdl_base_addr
||
1242 WARN_ON(hba
->utrdl_dma_addr
& (PAGE_SIZE
- 1))) {
1244 "Transfer Descriptor Memory allocation failed\n");
1249 * Allocate memory for UTP Task Management descriptors
1250 * UFSHCI requires 1024 byte alignment of UTMRD
1252 utmrdl_size
= sizeof(struct utp_task_req_desc
) * hba
->nutmrs
;
1253 hba
->utmrdl_base_addr
= dmam_alloc_coherent(hba
->dev
,
1255 &hba
->utmrdl_dma_addr
,
1257 if (!hba
->utmrdl_base_addr
||
1258 WARN_ON(hba
->utmrdl_dma_addr
& (PAGE_SIZE
- 1))) {
1260 "Task Management Descriptor Memory allocation failed\n");
1264 /* Allocate memory for local reference block */
1265 hba
->lrb
= devm_kzalloc(hba
->dev
,
1266 hba
->nutrs
* sizeof(struct ufshcd_lrb
),
1269 dev_err(hba
->dev
, "LRB Memory allocation failed\n");
1278 * ufshcd_host_memory_configure - configure local reference block with
1280 * @hba: per adapter instance
1282 * Configure Host memory space
1283 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
1285 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
1287 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
1288 * into local reference block.
1290 static void ufshcd_host_memory_configure(struct ufs_hba
*hba
)
1292 struct utp_transfer_cmd_desc
*cmd_descp
;
1293 struct utp_transfer_req_desc
*utrdlp
;
1294 dma_addr_t cmd_desc_dma_addr
;
1295 dma_addr_t cmd_desc_element_addr
;
1296 u16 response_offset
;
1301 utrdlp
= hba
->utrdl_base_addr
;
1302 cmd_descp
= hba
->ucdl_base_addr
;
1305 offsetof(struct utp_transfer_cmd_desc
, response_upiu
);
1307 offsetof(struct utp_transfer_cmd_desc
, prd_table
);
1309 cmd_desc_size
= sizeof(struct utp_transfer_cmd_desc
);
1310 cmd_desc_dma_addr
= hba
->ucdl_dma_addr
;
1312 for (i
= 0; i
< hba
->nutrs
; i
++) {
1313 /* Configure UTRD with command descriptor base address */
1314 cmd_desc_element_addr
=
1315 (cmd_desc_dma_addr
+ (cmd_desc_size
* i
));
1316 utrdlp
[i
].command_desc_base_addr_lo
=
1317 cpu_to_le32(lower_32_bits(cmd_desc_element_addr
));
1318 utrdlp
[i
].command_desc_base_addr_hi
=
1319 cpu_to_le32(upper_32_bits(cmd_desc_element_addr
));
1321 /* Response upiu and prdt offset should be in double words */
1322 utrdlp
[i
].response_upiu_offset
=
1323 cpu_to_le16((response_offset
>> 2));
1324 utrdlp
[i
].prd_table_offset
=
1325 cpu_to_le16((prdt_offset
>> 2));
1326 utrdlp
[i
].response_upiu_length
=
1327 cpu_to_le16(ALIGNED_UPIU_SIZE
>> 2);
1329 hba
->lrb
[i
].utr_descriptor_ptr
= (utrdlp
+ i
);
1330 hba
->lrb
[i
].ucd_req_ptr
=
1331 (struct utp_upiu_req
*)(cmd_descp
+ i
);
1332 hba
->lrb
[i
].ucd_rsp_ptr
=
1333 (struct utp_upiu_rsp
*)cmd_descp
[i
].response_upiu
;
1334 hba
->lrb
[i
].ucd_prdt_ptr
=
1335 (struct ufshcd_sg_entry
*)cmd_descp
[i
].prd_table
;
1340 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
1341 * @hba: per adapter instance
1343 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
1344 * in order to initialize the Unipro link startup procedure.
1345 * Once the Unipro links are up, the device connected to the controller
1348 * Returns 0 on success, non-zero value on failure
1350 static int ufshcd_dme_link_startup(struct ufs_hba
*hba
)
1352 struct uic_command uic_cmd
= {0};
1355 uic_cmd
.command
= UIC_CMD_DME_LINK_STARTUP
;
1357 ret
= ufshcd_send_uic_cmd(hba
, &uic_cmd
);
1360 "dme-link-startup: error code %d\n", ret
);
1365 * ufshcd_complete_dev_init() - checks device readiness
1366 * hba: per-adapter instance
1368 * Set fDeviceInit flag and poll until device toggles it.
1370 static int ufshcd_complete_dev_init(struct ufs_hba
*hba
)
1372 int i
, retries
, err
= 0;
1375 for (retries
= QUERY_REQ_RETRIES
; retries
> 0; retries
--) {
1376 /* Set the fDeviceInit flag */
1377 err
= ufshcd_query_flag(hba
, UPIU_QUERY_OPCODE_SET_FLAG
,
1378 QUERY_FLAG_IDN_FDEVICEINIT
, NULL
);
1379 if (!err
|| err
== -ETIMEDOUT
)
1381 dev_dbg(hba
->dev
, "%s: error %d retrying\n", __func__
, err
);
1385 "%s setting fDeviceInit flag failed with error %d\n",
1390 /* poll for max. 100 iterations for fDeviceInit flag to clear */
1391 for (i
= 0; i
< 100 && !err
&& flag_res
; i
++) {
1392 for (retries
= QUERY_REQ_RETRIES
; retries
> 0; retries
--) {
1393 err
= ufshcd_query_flag(hba
,
1394 UPIU_QUERY_OPCODE_READ_FLAG
,
1395 QUERY_FLAG_IDN_FDEVICEINIT
, &flag_res
);
1396 if (!err
|| err
== -ETIMEDOUT
)
1398 dev_dbg(hba
->dev
, "%s: error %d retrying\n", __func__
,
1404 "%s reading fDeviceInit flag failed with error %d\n",
1408 "%s fDeviceInit was not cleared by the device\n",
1416 * ufshcd_make_hba_operational - Make UFS controller operational
1417 * @hba: per adapter instance
1419 * To bring UFS host controller to operational state,
1420 * 1. Check if device is present
1421 * 2. Enable required interrupts
1422 * 3. Configure interrupt aggregation
1423 * 4. Program UTRL and UTMRL base addres
1424 * 5. Configure run-stop-registers
1426 * Returns 0 on success, non-zero value on failure
1428 static int ufshcd_make_hba_operational(struct ufs_hba
*hba
)
1433 /* check if device present */
1434 reg
= ufshcd_readl(hba
, REG_CONTROLLER_STATUS
);
1435 if (!ufshcd_is_device_present(reg
)) {
1436 dev_err(hba
->dev
, "cc: Device not present\n");
1441 /* Enable required interrupts */
1442 ufshcd_enable_intr(hba
, UFSHCD_ENABLE_INTRS
);
1444 /* Configure interrupt aggregation */
1445 ufshcd_config_int_aggr(hba
, INT_AGGR_CONFIG
);
1447 /* Configure UTRL and UTMRL base address registers */
1448 ufshcd_writel(hba
, lower_32_bits(hba
->utrdl_dma_addr
),
1449 REG_UTP_TRANSFER_REQ_LIST_BASE_L
);
1450 ufshcd_writel(hba
, upper_32_bits(hba
->utrdl_dma_addr
),
1451 REG_UTP_TRANSFER_REQ_LIST_BASE_H
);
1452 ufshcd_writel(hba
, lower_32_bits(hba
->utmrdl_dma_addr
),
1453 REG_UTP_TASK_REQ_LIST_BASE_L
);
1454 ufshcd_writel(hba
, upper_32_bits(hba
->utmrdl_dma_addr
),
1455 REG_UTP_TASK_REQ_LIST_BASE_H
);
1458 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
1459 * DEI, HEI bits must be 0
1461 if (!(ufshcd_get_lists_status(reg
))) {
1462 ufshcd_enable_run_stop_reg(hba
);
1465 "Host controller not ready to process requests");
1470 if (hba
->ufshcd_state
== UFSHCD_STATE_RESET
)
1471 scsi_unblock_requests(hba
->host
);
1473 hba
->ufshcd_state
= UFSHCD_STATE_OPERATIONAL
;
1480 * ufshcd_hba_enable - initialize the controller
1481 * @hba: per adapter instance
1483 * The controller resets itself and controller firmware initialization
1484 * sequence kicks off. When controller is ready it will set
1485 * the Host Controller Enable bit to 1.
1487 * Returns 0 on success, non-zero value on failure
1489 static int ufshcd_hba_enable(struct ufs_hba
*hba
)
1494 * msleep of 1 and 5 used in this function might result in msleep(20),
1495 * but it was necessary to send the UFS FPGA to reset mode during
1496 * development and testing of this driver. msleep can be changed to
1497 * mdelay and retry count can be reduced based on the controller.
1499 if (!ufshcd_is_hba_active(hba
)) {
1501 /* change controller state to "reset state" */
1502 ufshcd_hba_stop(hba
);
1505 * This delay is based on the testing done with UFS host
1506 * controller FPGA. The delay can be changed based on the
1507 * host controller used.
1512 /* start controller initialization sequence */
1513 ufshcd_hba_start(hba
);
1516 * To initialize a UFS host controller HCE bit must be set to 1.
1517 * During initialization the HCE bit value changes from 1->0->1.
1518 * When the host controller completes initialization sequence
1519 * it sets the value of HCE bit to 1. The same HCE bit is read back
1520 * to check if the controller has completed initialization sequence.
1521 * So without this delay the value HCE = 1, set in the previous
1522 * instruction might be read back.
1523 * This delay can be changed based on the controller.
1527 /* wait for the host controller to complete initialization */
1529 while (ufshcd_is_hba_active(hba
)) {
1534 "Controller enable failed\n");
1543 * ufshcd_link_startup - Initialize unipro link startup
1544 * @hba: per adapter instance
1546 * Returns 0 for success, non-zero in case of failure
1548 static int ufshcd_link_startup(struct ufs_hba
*hba
)
1552 /* enable UIC related interrupts */
1553 ufshcd_enable_intr(hba
, UIC_COMMAND_COMPL
);
1555 ret
= ufshcd_dme_link_startup(hba
);
1559 ret
= ufshcd_make_hba_operational(hba
);
1563 dev_err(hba
->dev
, "link startup failed %d\n", ret
);
1568 * ufshcd_verify_dev_init() - Verify device initialization
1569 * @hba: per-adapter instance
1571 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
1572 * device Transport Protocol (UTP) layer is ready after a reset.
1573 * If the UTP layer at the device side is not initialized, it may
1574 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
1575 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
1577 static int ufshcd_verify_dev_init(struct ufs_hba
*hba
)
1582 mutex_lock(&hba
->dev_cmd
.lock
);
1583 for (retries
= NOP_OUT_RETRIES
; retries
> 0; retries
--) {
1584 err
= ufshcd_exec_dev_cmd(hba
, DEV_CMD_TYPE_NOP
,
1587 if (!err
|| err
== -ETIMEDOUT
)
1590 dev_dbg(hba
->dev
, "%s: error %d retrying\n", __func__
, err
);
1592 mutex_unlock(&hba
->dev_cmd
.lock
);
1595 dev_err(hba
->dev
, "%s: NOP OUT failed %d\n", __func__
, err
);
1600 * ufshcd_do_reset - reset the host controller
1601 * @hba: per adapter instance
1603 * Returns SUCCESS/FAILED
1605 static int ufshcd_do_reset(struct ufs_hba
*hba
)
1607 struct ufshcd_lrb
*lrbp
;
1608 unsigned long flags
;
1611 /* block commands from midlayer */
1612 scsi_block_requests(hba
->host
);
1614 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
1615 hba
->ufshcd_state
= UFSHCD_STATE_RESET
;
1617 /* send controller to reset state */
1618 ufshcd_hba_stop(hba
);
1619 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
1621 /* abort outstanding commands */
1622 for (tag
= 0; tag
< hba
->nutrs
; tag
++) {
1623 if (test_bit(tag
, &hba
->outstanding_reqs
)) {
1624 lrbp
= &hba
->lrb
[tag
];
1626 scsi_dma_unmap(lrbp
->cmd
);
1627 lrbp
->cmd
->result
= DID_RESET
<< 16;
1628 lrbp
->cmd
->scsi_done(lrbp
->cmd
);
1630 clear_bit_unlock(tag
, &hba
->lrb_in_use
);
1635 /* complete device management command */
1636 if (hba
->dev_cmd
.complete
)
1637 complete(hba
->dev_cmd
.complete
);
1639 /* clear outstanding request/task bit maps */
1640 hba
->outstanding_reqs
= 0;
1641 hba
->outstanding_tasks
= 0;
1643 /* Host controller enable */
1644 if (ufshcd_hba_enable(hba
)) {
1646 "Reset: Controller initialization failed\n");
1650 if (ufshcd_link_startup(hba
)) {
1652 "Reset: Link start-up failed\n");
1660 * ufshcd_slave_alloc - handle initial SCSI device configurations
1661 * @sdev: pointer to SCSI device
1665 static int ufshcd_slave_alloc(struct scsi_device
*sdev
)
1667 struct ufs_hba
*hba
;
1669 hba
= shost_priv(sdev
->host
);
1670 sdev
->tagged_supported
= 1;
1672 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
1673 sdev
->use_10_for_ms
= 1;
1674 scsi_set_tag_type(sdev
, MSG_SIMPLE_TAG
);
1677 * Inform SCSI Midlayer that the LUN queue depth is same as the
1678 * controller queue depth. If a LUN queue depth is less than the
1679 * controller queue depth and if the LUN reports
1680 * SAM_STAT_TASK_SET_FULL, the LUN queue depth will be adjusted
1681 * with scsi_adjust_queue_depth.
1683 scsi_activate_tcq(sdev
, hba
->nutrs
);
1688 * ufshcd_slave_destroy - remove SCSI device configurations
1689 * @sdev: pointer to SCSI device
1691 static void ufshcd_slave_destroy(struct scsi_device
*sdev
)
1693 struct ufs_hba
*hba
;
1695 hba
= shost_priv(sdev
->host
);
1696 scsi_deactivate_tcq(sdev
, hba
->nutrs
);
1700 * ufshcd_task_req_compl - handle task management request completion
1701 * @hba: per adapter instance
1702 * @index: index of the completed request
1704 * Returns SUCCESS/FAILED
1706 static int ufshcd_task_req_compl(struct ufs_hba
*hba
, u32 index
)
1708 struct utp_task_req_desc
*task_req_descp
;
1709 struct utp_upiu_task_rsp
*task_rsp_upiup
;
1710 unsigned long flags
;
1714 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
1716 /* Clear completed tasks from outstanding_tasks */
1717 __clear_bit(index
, &hba
->outstanding_tasks
);
1719 task_req_descp
= hba
->utmrdl_base_addr
;
1720 ocs_value
= ufshcd_get_tmr_ocs(&task_req_descp
[index
]);
1722 if (ocs_value
== OCS_SUCCESS
) {
1723 task_rsp_upiup
= (struct utp_upiu_task_rsp
*)
1724 task_req_descp
[index
].task_rsp_upiu
;
1725 task_result
= be32_to_cpu(task_rsp_upiup
->header
.dword_1
);
1726 task_result
= ((task_result
& MASK_TASK_RESPONSE
) >> 8);
1728 if (task_result
!= UPIU_TASK_MANAGEMENT_FUNC_COMPL
&&
1729 task_result
!= UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED
)
1730 task_result
= FAILED
;
1732 task_result
= SUCCESS
;
1734 task_result
= FAILED
;
1736 "trc: Invalid ocs = %x\n", ocs_value
);
1738 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
1743 * ufshcd_adjust_lun_qdepth - Update LUN queue depth if device responds with
1744 * SAM_STAT_TASK_SET_FULL SCSI command status.
1745 * @cmd: pointer to SCSI command
1747 static void ufshcd_adjust_lun_qdepth(struct scsi_cmnd
*cmd
)
1749 struct ufs_hba
*hba
;
1753 hba
= shost_priv(cmd
->device
->host
);
1756 * LUN queue depth can be obtained by counting outstanding commands
1759 for (i
= 0; i
< hba
->nutrs
; i
++) {
1760 if (test_bit(i
, &hba
->outstanding_reqs
)) {
1763 * Check if the outstanding command belongs
1764 * to the LUN which reported SAM_STAT_TASK_SET_FULL.
1766 if (cmd
->device
->lun
== hba
->lrb
[i
].lun
)
1772 * LUN queue depth will be total outstanding commands, except the
1773 * command for which the LUN reported SAM_STAT_TASK_SET_FULL.
1775 scsi_adjust_queue_depth(cmd
->device
, MSG_SIMPLE_TAG
, lun_qdepth
- 1);
1779 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
1780 * @lrb: pointer to local reference block of completed command
1781 * @scsi_status: SCSI command status
1783 * Returns value base on SCSI command status
1786 ufshcd_scsi_cmd_status(struct ufshcd_lrb
*lrbp
, int scsi_status
)
1790 switch (scsi_status
) {
1792 result
|= DID_OK
<< 16 |
1793 COMMAND_COMPLETE
<< 8 |
1796 case SAM_STAT_CHECK_CONDITION
:
1797 result
|= DID_OK
<< 16 |
1798 COMMAND_COMPLETE
<< 8 |
1799 SAM_STAT_CHECK_CONDITION
;
1800 ufshcd_copy_sense_data(lrbp
);
1803 result
|= SAM_STAT_BUSY
;
1805 case SAM_STAT_TASK_SET_FULL
:
1808 * If a LUN reports SAM_STAT_TASK_SET_FULL, then the LUN queue
1809 * depth needs to be adjusted to the exact number of
1810 * outstanding commands the LUN can handle at any given time.
1812 ufshcd_adjust_lun_qdepth(lrbp
->cmd
);
1813 result
|= SAM_STAT_TASK_SET_FULL
;
1815 case SAM_STAT_TASK_ABORTED
:
1816 result
|= SAM_STAT_TASK_ABORTED
;
1819 result
|= DID_ERROR
<< 16;
1821 } /* end of switch */
1827 * ufshcd_transfer_rsp_status - Get overall status of the response
1828 * @hba: per adapter instance
1829 * @lrb: pointer to local reference block of completed command
1831 * Returns result of the command to notify SCSI midlayer
1834 ufshcd_transfer_rsp_status(struct ufs_hba
*hba
, struct ufshcd_lrb
*lrbp
)
1840 /* overall command status of utrd */
1841 ocs
= ufshcd_get_tr_ocs(lrbp
);
1845 result
= ufshcd_get_req_rsp(lrbp
->ucd_rsp_ptr
);
1848 case UPIU_TRANSACTION_RESPONSE
:
1850 * get the response UPIU result to extract
1851 * the SCSI command status
1853 result
= ufshcd_get_rsp_upiu_result(lrbp
->ucd_rsp_ptr
);
1856 * get the result based on SCSI status response
1857 * to notify the SCSI midlayer of the command status
1859 scsi_status
= result
& MASK_SCSI_STATUS
;
1860 result
= ufshcd_scsi_cmd_status(lrbp
, scsi_status
);
1862 if (ufshcd_is_exception_event(lrbp
->ucd_rsp_ptr
))
1863 schedule_work(&hba
->eeh_work
);
1865 case UPIU_TRANSACTION_REJECT_UPIU
:
1866 /* TODO: handle Reject UPIU Response */
1867 result
= DID_ERROR
<< 16;
1869 "Reject UPIU not fully implemented\n");
1872 result
= DID_ERROR
<< 16;
1874 "Unexpected request response code = %x\n",
1880 result
|= DID_ABORT
<< 16;
1882 case OCS_INVALID_CMD_TABLE_ATTR
:
1883 case OCS_INVALID_PRDT_ATTR
:
1884 case OCS_MISMATCH_DATA_BUF_SIZE
:
1885 case OCS_MISMATCH_RESP_UPIU_SIZE
:
1886 case OCS_PEER_COMM_FAILURE
:
1887 case OCS_FATAL_ERROR
:
1889 result
|= DID_ERROR
<< 16;
1891 "OCS error from controller = %x\n", ocs
);
1893 } /* end of switch */
1899 * ufshcd_uic_cmd_compl - handle completion of uic command
1900 * @hba: per adapter instance
1902 static void ufshcd_uic_cmd_compl(struct ufs_hba
*hba
)
1904 if (hba
->active_uic_cmd
) {
1905 hba
->active_uic_cmd
->argument2
|=
1906 ufshcd_get_uic_cmd_result(hba
);
1907 complete(&hba
->active_uic_cmd
->done
);
1912 * ufshcd_transfer_req_compl - handle SCSI and query command completion
1913 * @hba: per adapter instance
1915 static void ufshcd_transfer_req_compl(struct ufs_hba
*hba
)
1917 struct ufshcd_lrb
*lrbp
;
1918 struct scsi_cmnd
*cmd
;
1919 unsigned long completed_reqs
;
1923 bool int_aggr_reset
= false;
1925 tr_doorbell
= ufshcd_readl(hba
, REG_UTP_TRANSFER_REQ_DOOR_BELL
);
1926 completed_reqs
= tr_doorbell
^ hba
->outstanding_reqs
;
1928 for (index
= 0; index
< hba
->nutrs
; index
++) {
1929 if (test_bit(index
, &completed_reqs
)) {
1930 lrbp
= &hba
->lrb
[index
];
1933 * Don't skip resetting interrupt aggregation counters
1934 * if a regular command is present.
1936 int_aggr_reset
|= !lrbp
->intr_cmd
;
1939 result
= ufshcd_transfer_rsp_status(hba
, lrbp
);
1940 scsi_dma_unmap(cmd
);
1941 cmd
->result
= result
;
1942 /* Mark completed command as NULL in LRB */
1944 clear_bit_unlock(index
, &hba
->lrb_in_use
);
1945 /* Do not touch lrbp after scsi done */
1946 cmd
->scsi_done(cmd
);
1947 } else if (lrbp
->command_type
==
1948 UTP_CMD_TYPE_DEV_MANAGE
) {
1949 if (hba
->dev_cmd
.complete
)
1950 complete(hba
->dev_cmd
.complete
);
1955 /* clear corresponding bits of completed commands */
1956 hba
->outstanding_reqs
^= completed_reqs
;
1958 /* we might have free'd some tags above */
1959 wake_up(&hba
->dev_cmd
.tag_wq
);
1961 /* Reset interrupt aggregation counters */
1963 ufshcd_config_int_aggr(hba
, INT_AGGR_RESET
);
1967 * ufshcd_disable_ee - disable exception event
1968 * @hba: per-adapter instance
1969 * @mask: exception event to disable
1971 * Disables exception event in the device so that the EVENT_ALERT
1974 * Returns zero on success, non-zero error value on failure.
1976 static int ufshcd_disable_ee(struct ufs_hba
*hba
, u16 mask
)
1981 if (!(hba
->ee_ctrl_mask
& mask
))
1984 val
= hba
->ee_ctrl_mask
& ~mask
;
1985 val
&= 0xFFFF; /* 2 bytes */
1986 err
= ufshcd_query_attr(hba
, UPIU_QUERY_OPCODE_WRITE_ATTR
,
1987 QUERY_ATTR_IDN_EE_CONTROL
, 0, 0, &val
);
1989 hba
->ee_ctrl_mask
&= ~mask
;
1995 * ufshcd_enable_ee - enable exception event
1996 * @hba: per-adapter instance
1997 * @mask: exception event to enable
1999 * Enable corresponding exception event in the device to allow
2000 * device to alert host in critical scenarios.
2002 * Returns zero on success, non-zero error value on failure.
2004 static int ufshcd_enable_ee(struct ufs_hba
*hba
, u16 mask
)
2009 if (hba
->ee_ctrl_mask
& mask
)
2012 val
= hba
->ee_ctrl_mask
| mask
;
2013 val
&= 0xFFFF; /* 2 bytes */
2014 err
= ufshcd_query_attr(hba
, UPIU_QUERY_OPCODE_WRITE_ATTR
,
2015 QUERY_ATTR_IDN_EE_CONTROL
, 0, 0, &val
);
2017 hba
->ee_ctrl_mask
|= mask
;
2023 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
2024 * @hba: per-adapter instance
2026 * Allow device to manage background operations on its own. Enabling
2027 * this might lead to inconsistent latencies during normal data transfers
2028 * as the device is allowed to manage its own way of handling background
2031 * Returns zero on success, non-zero on failure.
2033 static int ufshcd_enable_auto_bkops(struct ufs_hba
*hba
)
2037 if (hba
->auto_bkops_enabled
)
2040 err
= ufshcd_query_flag(hba
, UPIU_QUERY_OPCODE_SET_FLAG
,
2041 QUERY_FLAG_IDN_BKOPS_EN
, NULL
);
2043 dev_err(hba
->dev
, "%s: failed to enable bkops %d\n",
2048 hba
->auto_bkops_enabled
= true;
2050 /* No need of URGENT_BKOPS exception from the device */
2051 err
= ufshcd_disable_ee(hba
, MASK_EE_URGENT_BKOPS
);
2053 dev_err(hba
->dev
, "%s: failed to disable exception event %d\n",
2060 * ufshcd_disable_auto_bkops - block device in doing background operations
2061 * @hba: per-adapter instance
2063 * Disabling background operations improves command response latency but
2064 * has drawback of device moving into critical state where the device is
2065 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
2066 * host is idle so that BKOPS are managed effectively without any negative
2069 * Returns zero on success, non-zero on failure.
2071 static int ufshcd_disable_auto_bkops(struct ufs_hba
*hba
)
2075 if (!hba
->auto_bkops_enabled
)
2079 * If host assisted BKOPs is to be enabled, make sure
2080 * urgent bkops exception is allowed.
2082 err
= ufshcd_enable_ee(hba
, MASK_EE_URGENT_BKOPS
);
2084 dev_err(hba
->dev
, "%s: failed to enable exception event %d\n",
2089 err
= ufshcd_query_flag(hba
, UPIU_QUERY_OPCODE_CLEAR_FLAG
,
2090 QUERY_FLAG_IDN_BKOPS_EN
, NULL
);
2092 dev_err(hba
->dev
, "%s: failed to disable bkops %d\n",
2094 ufshcd_disable_ee(hba
, MASK_EE_URGENT_BKOPS
);
2098 hba
->auto_bkops_enabled
= false;
2104 * ufshcd_force_reset_auto_bkops - force enable of auto bkops
2105 * @hba: per adapter instance
2107 * After a device reset the device may toggle the BKOPS_EN flag
2108 * to default value. The s/w tracking variables should be updated
2109 * as well. Do this by forcing enable of auto bkops.
2111 static void ufshcd_force_reset_auto_bkops(struct ufs_hba
*hba
)
2113 hba
->auto_bkops_enabled
= false;
2114 hba
->ee_ctrl_mask
|= MASK_EE_URGENT_BKOPS
;
2115 ufshcd_enable_auto_bkops(hba
);
2118 static inline int ufshcd_get_bkops_status(struct ufs_hba
*hba
, u32
*status
)
2120 return ufshcd_query_attr(hba
, UPIU_QUERY_OPCODE_READ_ATTR
,
2121 QUERY_ATTR_IDN_BKOPS_STATUS
, 0, 0, status
);
2125 * ufshcd_urgent_bkops - handle urgent bkops exception event
2126 * @hba: per-adapter instance
2128 * Enable fBackgroundOpsEn flag in the device to permit background
2131 static int ufshcd_urgent_bkops(struct ufs_hba
*hba
)
2136 err
= ufshcd_get_bkops_status(hba
, &status
);
2138 dev_err(hba
->dev
, "%s: failed to get BKOPS status %d\n",
2143 status
= status
& 0xF;
2145 /* handle only if status indicates performance impact or critical */
2146 if (status
>= BKOPS_STATUS_PERF_IMPACT
)
2147 err
= ufshcd_enable_auto_bkops(hba
);
2152 static inline int ufshcd_get_ee_status(struct ufs_hba
*hba
, u32
*status
)
2154 return ufshcd_query_attr(hba
, UPIU_QUERY_OPCODE_READ_ATTR
,
2155 QUERY_ATTR_IDN_EE_STATUS
, 0, 0, status
);
2159 * ufshcd_exception_event_handler - handle exceptions raised by device
2160 * @work: pointer to work data
2162 * Read bExceptionEventStatus attribute from the device and handle the
2163 * exception event accordingly.
2165 static void ufshcd_exception_event_handler(struct work_struct
*work
)
2167 struct ufs_hba
*hba
;
2170 hba
= container_of(work
, struct ufs_hba
, eeh_work
);
2172 pm_runtime_get_sync(hba
->dev
);
2173 err
= ufshcd_get_ee_status(hba
, &status
);
2175 dev_err(hba
->dev
, "%s: failed to get exception status %d\n",
2180 status
&= hba
->ee_ctrl_mask
;
2181 if (status
& MASK_EE_URGENT_BKOPS
) {
2182 err
= ufshcd_urgent_bkops(hba
);
2184 dev_err(hba
->dev
, "%s: failed to handle urgent bkops %d\n",
2188 pm_runtime_put_sync(hba
->dev
);
2193 * ufshcd_fatal_err_handler - handle fatal errors
2194 * @hba: per adapter instance
2196 static void ufshcd_fatal_err_handler(struct work_struct
*work
)
2198 struct ufs_hba
*hba
;
2199 hba
= container_of(work
, struct ufs_hba
, feh_workq
);
2201 pm_runtime_get_sync(hba
->dev
);
2202 /* check if reset is already in progress */
2203 if (hba
->ufshcd_state
!= UFSHCD_STATE_RESET
)
2204 ufshcd_do_reset(hba
);
2205 pm_runtime_put_sync(hba
->dev
);
2209 * ufshcd_err_handler - Check for fatal errors
2210 * @work: pointer to a work queue structure
2212 static void ufshcd_err_handler(struct ufs_hba
*hba
)
2216 if (hba
->errors
& INT_FATAL_ERRORS
)
2219 if (hba
->errors
& UIC_ERROR
) {
2220 reg
= ufshcd_readl(hba
, REG_UIC_ERROR_CODE_DATA_LINK_LAYER
);
2221 if (reg
& UIC_DATA_LINK_LAYER_ERROR_PA_INIT
)
2226 hba
->ufshcd_state
= UFSHCD_STATE_ERROR
;
2227 schedule_work(&hba
->feh_workq
);
2231 * ufshcd_tmc_handler - handle task management function completion
2232 * @hba: per adapter instance
2234 static void ufshcd_tmc_handler(struct ufs_hba
*hba
)
2238 tm_doorbell
= ufshcd_readl(hba
, REG_UTP_TASK_REQ_DOOR_BELL
);
2239 hba
->tm_condition
= tm_doorbell
^ hba
->outstanding_tasks
;
2240 wake_up_interruptible(&hba
->ufshcd_tm_wait_queue
);
2244 * ufshcd_sl_intr - Interrupt service routine
2245 * @hba: per adapter instance
2246 * @intr_status: contains interrupts generated by the controller
2248 static void ufshcd_sl_intr(struct ufs_hba
*hba
, u32 intr_status
)
2250 hba
->errors
= UFSHCD_ERROR_MASK
& intr_status
;
2252 ufshcd_err_handler(hba
);
2254 if (intr_status
& UIC_COMMAND_COMPL
)
2255 ufshcd_uic_cmd_compl(hba
);
2257 if (intr_status
& UTP_TASK_REQ_COMPL
)
2258 ufshcd_tmc_handler(hba
);
2260 if (intr_status
& UTP_TRANSFER_REQ_COMPL
)
2261 ufshcd_transfer_req_compl(hba
);
2265 * ufshcd_intr - Main interrupt service routine
2267 * @__hba: pointer to adapter instance
2269 * Returns IRQ_HANDLED - If interrupt is valid
2270 * IRQ_NONE - If invalid interrupt
2272 static irqreturn_t
ufshcd_intr(int irq
, void *__hba
)
2275 irqreturn_t retval
= IRQ_NONE
;
2276 struct ufs_hba
*hba
= __hba
;
2278 spin_lock(hba
->host
->host_lock
);
2279 intr_status
= ufshcd_readl(hba
, REG_INTERRUPT_STATUS
);
2282 ufshcd_writel(hba
, intr_status
, REG_INTERRUPT_STATUS
);
2283 ufshcd_sl_intr(hba
, intr_status
);
2284 retval
= IRQ_HANDLED
;
2286 spin_unlock(hba
->host
->host_lock
);
2291 * ufshcd_issue_tm_cmd - issues task management commands to controller
2292 * @hba: per adapter instance
2293 * @lrbp: pointer to local reference block
2295 * Returns SUCCESS/FAILED
2298 ufshcd_issue_tm_cmd(struct ufs_hba
*hba
,
2299 struct ufshcd_lrb
*lrbp
,
2302 struct utp_task_req_desc
*task_req_descp
;
2303 struct utp_upiu_task_req
*task_req_upiup
;
2304 struct Scsi_Host
*host
;
2305 unsigned long flags
;
2311 spin_lock_irqsave(host
->host_lock
, flags
);
2313 /* If task management queue is full */
2314 free_slot
= ufshcd_get_tm_free_slot(hba
);
2315 if (free_slot
>= hba
->nutmrs
) {
2316 spin_unlock_irqrestore(host
->host_lock
, flags
);
2317 dev_err(hba
->dev
, "Task management queue full\n");
2322 task_req_descp
= hba
->utmrdl_base_addr
;
2323 task_req_descp
+= free_slot
;
2325 /* Configure task request descriptor */
2326 task_req_descp
->header
.dword_0
= cpu_to_le32(UTP_REQ_DESC_INT_CMD
);
2327 task_req_descp
->header
.dword_2
=
2328 cpu_to_le32(OCS_INVALID_COMMAND_STATUS
);
2330 /* Configure task request UPIU */
2332 (struct utp_upiu_task_req
*) task_req_descp
->task_req_upiu
;
2333 task_req_upiup
->header
.dword_0
=
2334 UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ
, 0,
2335 lrbp
->lun
, lrbp
->task_tag
);
2336 task_req_upiup
->header
.dword_1
=
2337 UPIU_HEADER_DWORD(0, tm_function
, 0, 0);
2339 task_req_upiup
->input_param1
= lrbp
->lun
;
2340 task_req_upiup
->input_param1
=
2341 cpu_to_be32(task_req_upiup
->input_param1
);
2342 task_req_upiup
->input_param2
= lrbp
->task_tag
;
2343 task_req_upiup
->input_param2
=
2344 cpu_to_be32(task_req_upiup
->input_param2
);
2346 /* send command to the controller */
2347 __set_bit(free_slot
, &hba
->outstanding_tasks
);
2348 ufshcd_writel(hba
, 1 << free_slot
, REG_UTP_TASK_REQ_DOOR_BELL
);
2350 spin_unlock_irqrestore(host
->host_lock
, flags
);
2352 /* wait until the task management command is completed */
2354 wait_event_interruptible_timeout(hba
->ufshcd_tm_wait_queue
,
2355 (test_bit(free_slot
,
2356 &hba
->tm_condition
) != 0),
2360 "Task management command timed-out\n");
2364 clear_bit(free_slot
, &hba
->tm_condition
);
2365 err
= ufshcd_task_req_compl(hba
, free_slot
);
2371 * ufshcd_device_reset - reset device and abort all the pending commands
2372 * @cmd: SCSI command pointer
2374 * Returns SUCCESS/FAILED
2376 static int ufshcd_device_reset(struct scsi_cmnd
*cmd
)
2378 struct Scsi_Host
*host
;
2379 struct ufs_hba
*hba
;
2384 host
= cmd
->device
->host
;
2385 hba
= shost_priv(host
);
2386 tag
= cmd
->request
->tag
;
2388 err
= ufshcd_issue_tm_cmd(hba
, &hba
->lrb
[tag
], UFS_LOGICAL_RESET
);
2392 for (pos
= 0; pos
< hba
->nutrs
; pos
++) {
2393 if (test_bit(pos
, &hba
->outstanding_reqs
) &&
2394 (hba
->lrb
[tag
].lun
== hba
->lrb
[pos
].lun
)) {
2396 /* clear the respective UTRLCLR register bit */
2397 ufshcd_utrl_clear(hba
, pos
);
2399 clear_bit(pos
, &hba
->outstanding_reqs
);
2401 if (hba
->lrb
[pos
].cmd
) {
2402 scsi_dma_unmap(hba
->lrb
[pos
].cmd
);
2403 hba
->lrb
[pos
].cmd
->result
=
2405 hba
->lrb
[pos
].cmd
->scsi_done(cmd
);
2406 hba
->lrb
[pos
].cmd
= NULL
;
2407 clear_bit_unlock(pos
, &hba
->lrb_in_use
);
2408 wake_up(&hba
->dev_cmd
.tag_wq
);
2417 * ufshcd_host_reset - Main reset function registered with scsi layer
2418 * @cmd: SCSI command pointer
2420 * Returns SUCCESS/FAILED
2422 static int ufshcd_host_reset(struct scsi_cmnd
*cmd
)
2424 struct ufs_hba
*hba
;
2426 hba
= shost_priv(cmd
->device
->host
);
2428 if (hba
->ufshcd_state
== UFSHCD_STATE_RESET
)
2431 return ufshcd_do_reset(hba
);
2435 * ufshcd_abort - abort a specific command
2436 * @cmd: SCSI command pointer
2438 * Returns SUCCESS/FAILED
2440 static int ufshcd_abort(struct scsi_cmnd
*cmd
)
2442 struct Scsi_Host
*host
;
2443 struct ufs_hba
*hba
;
2444 unsigned long flags
;
2448 host
= cmd
->device
->host
;
2449 hba
= shost_priv(host
);
2450 tag
= cmd
->request
->tag
;
2452 spin_lock_irqsave(host
->host_lock
, flags
);
2454 /* check if command is still pending */
2455 if (!(test_bit(tag
, &hba
->outstanding_reqs
))) {
2457 spin_unlock_irqrestore(host
->host_lock
, flags
);
2460 spin_unlock_irqrestore(host
->host_lock
, flags
);
2462 err
= ufshcd_issue_tm_cmd(hba
, &hba
->lrb
[tag
], UFS_ABORT_TASK
);
2466 scsi_dma_unmap(cmd
);
2468 spin_lock_irqsave(host
->host_lock
, flags
);
2470 /* clear the respective UTRLCLR register bit */
2471 ufshcd_utrl_clear(hba
, tag
);
2473 __clear_bit(tag
, &hba
->outstanding_reqs
);
2474 hba
->lrb
[tag
].cmd
= NULL
;
2475 spin_unlock_irqrestore(host
->host_lock
, flags
);
2477 clear_bit_unlock(tag
, &hba
->lrb_in_use
);
2478 wake_up(&hba
->dev_cmd
.tag_wq
);
2484 * ufshcd_async_scan - asynchronous execution for link startup
2485 * @data: data pointer to pass to this function
2486 * @cookie: cookie data
2488 static void ufshcd_async_scan(void *data
, async_cookie_t cookie
)
2490 struct ufs_hba
*hba
= (struct ufs_hba
*)data
;
2493 ret
= ufshcd_link_startup(hba
);
2497 ret
= ufshcd_verify_dev_init(hba
);
2501 ret
= ufshcd_complete_dev_init(hba
);
2505 ufshcd_force_reset_auto_bkops(hba
);
2506 scsi_scan_host(hba
->host
);
2507 pm_runtime_put_sync(hba
->dev
);
2512 static struct scsi_host_template ufshcd_driver_template
= {
2513 .module
= THIS_MODULE
,
2515 .proc_name
= UFSHCD
,
2516 .queuecommand
= ufshcd_queuecommand
,
2517 .slave_alloc
= ufshcd_slave_alloc
,
2518 .slave_destroy
= ufshcd_slave_destroy
,
2519 .eh_abort_handler
= ufshcd_abort
,
2520 .eh_device_reset_handler
= ufshcd_device_reset
,
2521 .eh_host_reset_handler
= ufshcd_host_reset
,
2523 .sg_tablesize
= SG_ALL
,
2524 .cmd_per_lun
= UFSHCD_CMD_PER_LUN
,
2525 .can_queue
= UFSHCD_CAN_QUEUE
,
2529 * ufshcd_suspend - suspend power management function
2530 * @hba: per adapter instance
2531 * @state: power state
2535 int ufshcd_suspend(struct ufs_hba
*hba
, pm_message_t state
)
2539 * 1. Block SCSI requests from SCSI midlayer
2540 * 2. Change the internal driver state to non operational
2541 * 3. Set UTRLRSR and UTMRLRSR bits to zero
2542 * 4. Wait until outstanding commands are completed
2543 * 5. Set HCE to zero to send the UFS host controller to reset state
2548 EXPORT_SYMBOL_GPL(ufshcd_suspend
);
2551 * ufshcd_resume - resume power management function
2552 * @hba: per adapter instance
2556 int ufshcd_resume(struct ufs_hba
*hba
)
2560 * 1. Set HCE to 1, to start the UFS host controller
2561 * initialization process
2562 * 2. Set UTRLRSR and UTMRLRSR bits to 1
2563 * 3. Change the internal driver state to operational
2564 * 4. Unblock SCSI requests from SCSI midlayer
2569 EXPORT_SYMBOL_GPL(ufshcd_resume
);
2571 int ufshcd_runtime_suspend(struct ufs_hba
*hba
)
2577 * The device is idle with no requests in the queue,
2578 * allow background operations.
2580 return ufshcd_enable_auto_bkops(hba
);
2582 EXPORT_SYMBOL(ufshcd_runtime_suspend
);
2584 int ufshcd_runtime_resume(struct ufs_hba
*hba
)
2589 return ufshcd_disable_auto_bkops(hba
);
2591 EXPORT_SYMBOL(ufshcd_runtime_resume
);
2593 int ufshcd_runtime_idle(struct ufs_hba
*hba
)
2597 EXPORT_SYMBOL(ufshcd_runtime_idle
);
2600 * ufshcd_remove - de-allocate SCSI host and host memory space
2601 * data structure memory
2602 * @hba - per adapter instance
2604 void ufshcd_remove(struct ufs_hba
*hba
)
2606 scsi_remove_host(hba
->host
);
2607 /* disable interrupts */
2608 ufshcd_disable_intr(hba
, hba
->intr_mask
);
2609 ufshcd_hba_stop(hba
);
2611 scsi_host_put(hba
->host
);
2613 EXPORT_SYMBOL_GPL(ufshcd_remove
);
2616 * ufshcd_init - Driver initialization routine
2617 * @dev: pointer to device handle
2618 * @hba_handle: driver private handle
2619 * @mmio_base: base register address
2620 * @irq: Interrupt line of device
2621 * Returns 0 on success, non-zero value on failure
2623 int ufshcd_init(struct device
*dev
, struct ufs_hba
**hba_handle
,
2624 void __iomem
*mmio_base
, unsigned int irq
)
2626 struct Scsi_Host
*host
;
2627 struct ufs_hba
*hba
;
2632 "Invalid memory reference for dev is NULL\n");
2639 "Invalid memory reference for mmio_base is NULL\n");
2644 host
= scsi_host_alloc(&ufshcd_driver_template
,
2645 sizeof(struct ufs_hba
));
2647 dev_err(dev
, "scsi_host_alloc failed\n");
2651 hba
= shost_priv(host
);
2654 hba
->mmio_base
= mmio_base
;
2657 /* Read capabilities registers */
2658 ufshcd_hba_capabilities(hba
);
2660 /* Get UFS version supported by the controller */
2661 hba
->ufs_version
= ufshcd_get_ufs_version(hba
);
2663 /* Get Interrupt bit mask per version */
2664 hba
->intr_mask
= ufshcd_get_intr_mask(hba
);
2666 /* Allocate memory for host memory space */
2667 err
= ufshcd_memory_alloc(hba
);
2669 dev_err(hba
->dev
, "Memory allocation failed\n");
2674 ufshcd_host_memory_configure(hba
);
2676 host
->can_queue
= hba
->nutrs
;
2677 host
->cmd_per_lun
= hba
->nutrs
;
2678 host
->max_id
= UFSHCD_MAX_ID
;
2679 host
->max_lun
= UFSHCD_MAX_LUNS
;
2680 host
->max_channel
= UFSHCD_MAX_CHANNEL
;
2681 host
->unique_id
= host
->host_no
;
2682 host
->max_cmd_len
= MAX_CDB_SIZE
;
2684 /* Initailize wait queue for task management */
2685 init_waitqueue_head(&hba
->ufshcd_tm_wait_queue
);
2687 /* Initialize work queues */
2688 INIT_WORK(&hba
->feh_workq
, ufshcd_fatal_err_handler
);
2689 INIT_WORK(&hba
->eeh_work
, ufshcd_exception_event_handler
);
2691 /* Initialize UIC command mutex */
2692 mutex_init(&hba
->uic_cmd_mutex
);
2694 /* Initialize mutex for device management commands */
2695 mutex_init(&hba
->dev_cmd
.lock
);
2697 /* Initialize device management tag acquire wait queue */
2698 init_waitqueue_head(&hba
->dev_cmd
.tag_wq
);
2700 /* IRQ registration */
2701 err
= devm_request_irq(dev
, irq
, ufshcd_intr
, IRQF_SHARED
, UFSHCD
, hba
);
2703 dev_err(hba
->dev
, "request irq failed\n");
2707 /* Enable SCSI tag mapping */
2708 err
= scsi_init_shared_tag_map(host
, host
->can_queue
);
2710 dev_err(hba
->dev
, "init shared queue failed\n");
2714 err
= scsi_add_host(host
, hba
->dev
);
2716 dev_err(hba
->dev
, "scsi_add_host failed\n");
2720 /* Host controller enable */
2721 err
= ufshcd_hba_enable(hba
);
2723 dev_err(hba
->dev
, "Host controller enable failed\n");
2724 goto out_remove_scsi_host
;
2729 /* Hold auto suspend until async scan completes */
2730 pm_runtime_get_sync(dev
);
2732 async_schedule(ufshcd_async_scan
, hba
);
2736 out_remove_scsi_host
:
2737 scsi_remove_host(hba
->host
);
2739 scsi_host_put(host
);
2743 EXPORT_SYMBOL_GPL(ufshcd_init
);
2745 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
2746 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
2747 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
2748 MODULE_LICENSE("GPL");
2749 MODULE_VERSION(UFSHCD_DRIVER_VERSION
);