2 * Universal Flash Storage Host controller driver
4 * This code is based on drivers/scsi/ufs/ufshcd.c
5 * Copyright (C) 2011-2012 Samsung India Software Operations
7 * Santosh Yaraganavi <santosh.sy@samsung.com>
8 * Vinayak Holikatti <h.vinayak@samsung.com>
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version 2
13 * of the License, or (at your option) any later version.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
21 * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
22 * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
23 * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
24 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
25 * solely responsible for determining the appropriateness of using and
26 * distributing the Program and assumes all risks associated with its
27 * exercise of rights under this Agreement, including but not limited to
28 * the risks and costs of program errors, damage to or loss of data,
29 * programs or equipment, and unavailability or interruption of operations.
31 * DISCLAIMER OF LIABILITY
32 * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
33 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
35 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
36 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
37 * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
38 * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
40 * You should have received a copy of the GNU General Public License
41 * along with this program; if not, write to the Free Software
42 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
46 #include <linux/module.h>
47 #include <linux/kernel.h>
48 #include <linux/init.h>
49 #include <linux/pci.h>
50 #include <linux/interrupt.h>
52 #include <linux/delay.h>
53 #include <linux/slab.h>
54 #include <linux/spinlock.h>
55 #include <linux/workqueue.h>
56 #include <linux/errno.h>
57 #include <linux/types.h>
58 #include <linux/wait.h>
59 #include <linux/bitops.h>
62 #include <asm/byteorder.h>
63 #include <scsi/scsi.h>
64 #include <scsi/scsi_cmnd.h>
65 #include <scsi/scsi_host.h>
66 #include <scsi/scsi_tcq.h>
67 #include <scsi/scsi_dbg.h>
68 #include <scsi/scsi_eh.h>
73 #define UFSHCD "ufshcd"
74 #define UFSHCD_DRIVER_VERSION "0.1"
77 UFSHCD_MAX_CHANNEL
= 0,
80 UFSHCD_CMD_PER_LUN
= 32,
81 UFSHCD_CAN_QUEUE
= 32,
86 UFSHCD_STATE_OPERATIONAL
,
91 /* Interrupt configuration options */
98 /* Interrupt aggregation options */
105 * struct uic_command - UIC command structure
106 * @command: UIC command
107 * @argument1: UIC command argument 1
108 * @argument2: UIC command argument 2
109 * @argument3: UIC command argument 3
110 * @cmd_active: Indicate if UIC command is outstanding
111 * @result: UIC command result
123 * struct ufs_hba - per adapter private structure
124 * @mmio_base: UFSHCI base register address
125 * @ucdl_base_addr: UFS Command Descriptor base address
126 * @utrdl_base_addr: UTP Transfer Request Descriptor base address
127 * @utmrdl_base_addr: UTP Task Management Descriptor base address
128 * @ucdl_dma_addr: UFS Command Descriptor DMA address
129 * @utrdl_dma_addr: UTRDL DMA address
130 * @utmrdl_dma_addr: UTMRDL DMA address
131 * @host: Scsi_Host instance of the driver
132 * @pdev: PCI device handle
133 * @lrb: local reference block
134 * @outstanding_tasks: Bits representing outstanding task requests
135 * @outstanding_reqs: Bits representing outstanding transfer requests
136 * @capabilities: UFS Controller Capabilities
137 * @nutrs: Transfer Request Queue depth supported by controller
138 * @nutmrs: Task Management Queue depth supported by controller
139 * @active_uic_cmd: handle of active UIC command
140 * @ufshcd_tm_wait_queue: wait queue for task management
141 * @tm_condition: condition variable for task management
142 * @ufshcd_state: UFSHCD states
143 * @int_enable_mask: Interrupt Mask Bits
144 * @uic_workq: Work queue for UIC completion handling
145 * @feh_workq: Work queue for fatal controller error handling
146 * @errors: HBA errors
149 void __iomem
*mmio_base
;
151 /* Virtual memory reference */
152 struct utp_transfer_cmd_desc
*ucdl_base_addr
;
153 struct utp_transfer_req_desc
*utrdl_base_addr
;
154 struct utp_task_req_desc
*utmrdl_base_addr
;
156 /* DMA memory reference */
157 dma_addr_t ucdl_dma_addr
;
158 dma_addr_t utrdl_dma_addr
;
159 dma_addr_t utmrdl_dma_addr
;
161 struct Scsi_Host
*host
;
162 struct pci_dev
*pdev
;
164 struct ufshcd_lrb
*lrb
;
166 unsigned long outstanding_tasks
;
167 unsigned long outstanding_reqs
;
174 struct uic_command active_uic_cmd
;
175 wait_queue_head_t ufshcd_tm_wait_queue
;
176 unsigned long tm_condition
;
182 struct work_struct uic_workq
;
183 struct work_struct feh_workq
;
190 * struct ufshcd_lrb - local reference block
191 * @utr_descriptor_ptr: UTRD address of the command
192 * @ucd_cmd_ptr: UCD address of the command
193 * @ucd_rsp_ptr: Response UPIU address for this command
194 * @ucd_prdt_ptr: PRDT address of the command
195 * @cmd: pointer to SCSI command
196 * @sense_buffer: pointer to sense buffer address of the SCSI command
197 * @sense_bufflen: Length of the sense buffer
198 * @scsi_status: SCSI status of the command
199 * @command_type: SCSI, UFS, Query.
200 * @task_tag: Task tag of the command
201 * @lun: LUN of the command
204 struct utp_transfer_req_desc
*utr_descriptor_ptr
;
205 struct utp_upiu_cmd
*ucd_cmd_ptr
;
206 struct utp_upiu_rsp
*ucd_rsp_ptr
;
207 struct ufshcd_sg_entry
*ucd_prdt_ptr
;
209 struct scsi_cmnd
*cmd
;
211 unsigned int sense_bufflen
;
220 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
221 * @hba - Pointer to adapter instance
223 * Returns UFSHCI version supported by the controller
225 static inline u32
ufshcd_get_ufs_version(struct ufs_hba
*hba
)
227 return readl(hba
->mmio_base
+ REG_UFS_VERSION
);
231 * ufshcd_is_device_present - Check if any device connected to
232 * the host controller
233 * @reg_hcs - host controller status register value
235 * Returns 0 if device present, non-zero if no device detected
237 static inline int ufshcd_is_device_present(u32 reg_hcs
)
239 return (DEVICE_PRESENT
& reg_hcs
) ? 0 : -1;
243 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
244 * @lrb: pointer to local command reference block
246 * This function is used to get the OCS field from UTRD
247 * Returns the OCS field in the UTRD
249 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb
*lrbp
)
251 return lrbp
->utr_descriptor_ptr
->header
.dword_2
& MASK_OCS
;
255 * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
256 * @task_req_descp: pointer to utp_task_req_desc structure
258 * This function is used to get the OCS field from UTMRD
259 * Returns the OCS field in the UTMRD
262 ufshcd_get_tmr_ocs(struct utp_task_req_desc
*task_req_descp
)
264 return task_req_descp
->header
.dword_2
& MASK_OCS
;
268 * ufshcd_get_tm_free_slot - get a free slot for task management request
269 * @hba: per adapter instance
271 * Returns maximum number of task management request slots in case of
272 * task management queue full or returns the free slot number
274 static inline int ufshcd_get_tm_free_slot(struct ufs_hba
*hba
)
276 return find_first_zero_bit(&hba
->outstanding_tasks
, hba
->nutmrs
);
280 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
281 * @hba: per adapter instance
282 * @pos: position of the bit to be cleared
284 static inline void ufshcd_utrl_clear(struct ufs_hba
*hba
, u32 pos
)
287 (hba
->mmio_base
+ REG_UTP_TRANSFER_REQ_LIST_CLEAR
));
291 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
292 * @reg: Register value of host controller status
294 * Returns integer, 0 on Success and positive value if failed
296 static inline int ufshcd_get_lists_status(u32 reg
)
299 * The mask 0xFF is for the following HCS register bits
309 return (((reg
) & (0xFF)) >> 1) ^ (0x07);
313 * ufshcd_get_uic_cmd_result - Get the UIC command result
314 * @hba: Pointer to adapter instance
316 * This function gets the result of UIC command completion
317 * Returns 0 on success, non zero value on error
319 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba
*hba
)
321 return readl(hba
->mmio_base
+ REG_UIC_COMMAND_ARG_2
) &
322 MASK_UIC_COMMAND_RESULT
;
326 * ufshcd_free_hba_memory - Free allocated memory for LRB, request
328 * @hba: Pointer to adapter instance
330 static inline void ufshcd_free_hba_memory(struct ufs_hba
*hba
)
332 size_t utmrdl_size
, utrdl_size
, ucdl_size
;
336 if (hba
->utmrdl_base_addr
) {
337 utmrdl_size
= sizeof(struct utp_task_req_desc
) * hba
->nutmrs
;
338 dma_free_coherent(&hba
->pdev
->dev
, utmrdl_size
,
339 hba
->utmrdl_base_addr
, hba
->utmrdl_dma_addr
);
342 if (hba
->utrdl_base_addr
) {
344 (sizeof(struct utp_transfer_req_desc
) * hba
->nutrs
);
345 dma_free_coherent(&hba
->pdev
->dev
, utrdl_size
,
346 hba
->utrdl_base_addr
, hba
->utrdl_dma_addr
);
349 if (hba
->ucdl_base_addr
) {
351 (sizeof(struct utp_transfer_cmd_desc
) * hba
->nutrs
);
352 dma_free_coherent(&hba
->pdev
->dev
, ucdl_size
,
353 hba
->ucdl_base_addr
, hba
->ucdl_dma_addr
);
358 * ufshcd_is_valid_req_rsp - checks if controller TR response is valid
359 * @ucd_rsp_ptr: pointer to response UPIU
361 * This function checks the response UPIU for valid transaction type in
363 * Returns 0 on success, non-zero on failure
366 ufshcd_is_valid_req_rsp(struct utp_upiu_rsp
*ucd_rsp_ptr
)
368 return ((be32_to_cpu(ucd_rsp_ptr
->header
.dword_0
) >> 24) ==
369 UPIU_TRANSACTION_RESPONSE
) ? 0 : DID_ERROR
<< 16;
373 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
374 * @ucd_rsp_ptr: pointer to response UPIU
376 * This function gets the response status and scsi_status from response UPIU
377 * Returns the response result code.
380 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp
*ucd_rsp_ptr
)
382 return be32_to_cpu(ucd_rsp_ptr
->header
.dword_1
) & MASK_RSP_UPIU_RESULT
;
386 * ufshcd_config_int_aggr - Configure interrupt aggregation values.
387 * Currently there is no use case where we want to configure
388 * interrupt aggregation dynamically. So to configure interrupt
389 * aggregation, #define INT_AGGR_COUNTER_THRESHOLD_VALUE and
390 * INT_AGGR_TIMEOUT_VALUE are used.
391 * @hba: per adapter instance
392 * @option: Interrupt aggregation option
395 ufshcd_config_int_aggr(struct ufs_hba
*hba
, int option
)
399 writel((INT_AGGR_ENABLE
|
400 INT_AGGR_COUNTER_AND_TIMER_RESET
),
402 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL
));
404 case INT_AGGR_CONFIG
:
405 writel((INT_AGGR_ENABLE
|
406 INT_AGGR_PARAM_WRITE
|
407 INT_AGGR_COUNTER_THRESHOLD_VALUE
|
408 INT_AGGR_TIMEOUT_VALUE
),
410 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL
));
416 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
417 * When run-stop registers are set to 1, it indicates the
418 * host controller that it can process the requests
419 * @hba: per adapter instance
421 static void ufshcd_enable_run_stop_reg(struct ufs_hba
*hba
)
423 writel(UTP_TASK_REQ_LIST_RUN_STOP_BIT
,
425 REG_UTP_TASK_REQ_LIST_RUN_STOP
));
426 writel(UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT
,
428 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP
));
432 * ufshcd_hba_stop - Send controller to reset state
433 * @hba: per adapter instance
435 static inline void ufshcd_hba_stop(struct ufs_hba
*hba
)
437 writel(CONTROLLER_DISABLE
, (hba
->mmio_base
+ REG_CONTROLLER_ENABLE
));
441 * ufshcd_hba_start - Start controller initialization sequence
442 * @hba: per adapter instance
444 static inline void ufshcd_hba_start(struct ufs_hba
*hba
)
446 writel(CONTROLLER_ENABLE
, (hba
->mmio_base
+ REG_CONTROLLER_ENABLE
));
450 * ufshcd_is_hba_active - Get controller state
451 * @hba: per adapter instance
453 * Returns zero if controller is active, 1 otherwise
455 static inline int ufshcd_is_hba_active(struct ufs_hba
*hba
)
457 return (readl(hba
->mmio_base
+ REG_CONTROLLER_ENABLE
) & 0x1) ? 0 : 1;
461 * ufshcd_send_command - Send SCSI or device management commands
462 * @hba: per adapter instance
463 * @task_tag: Task tag of the command
466 void ufshcd_send_command(struct ufs_hba
*hba
, unsigned int task_tag
)
468 __set_bit(task_tag
, &hba
->outstanding_reqs
);
469 writel((1 << task_tag
),
470 (hba
->mmio_base
+ REG_UTP_TRANSFER_REQ_DOOR_BELL
));
474 * ufshcd_copy_sense_data - Copy sense data in case of check condition
475 * @lrb - pointer to local reference block
477 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb
*lrbp
)
480 if (lrbp
->sense_buffer
) {
481 len
= be16_to_cpu(lrbp
->ucd_rsp_ptr
->sense_data_len
);
482 memcpy(lrbp
->sense_buffer
,
483 lrbp
->ucd_rsp_ptr
->sense_data
,
484 min_t(int, len
, SCSI_SENSE_BUFFERSIZE
));
489 * ufshcd_hba_capabilities - Read controller capabilities
490 * @hba: per adapter instance
492 static inline void ufshcd_hba_capabilities(struct ufs_hba
*hba
)
495 readl(hba
->mmio_base
+ REG_CONTROLLER_CAPABILITIES
);
497 /* nutrs and nutmrs are 0 based values */
498 hba
->nutrs
= (hba
->capabilities
& MASK_TRANSFER_REQUESTS_SLOTS
) + 1;
500 ((hba
->capabilities
& MASK_TASK_MANAGEMENT_REQUEST_SLOTS
) >> 16) + 1;
504 * ufshcd_send_uic_command - Send UIC commands to unipro layers
505 * @hba: per adapter instance
506 * @uic_command: UIC command
509 ufshcd_send_uic_command(struct ufs_hba
*hba
, struct uic_command
*uic_cmnd
)
512 writel(uic_cmnd
->argument1
,
513 (hba
->mmio_base
+ REG_UIC_COMMAND_ARG_1
));
514 writel(uic_cmnd
->argument2
,
515 (hba
->mmio_base
+ REG_UIC_COMMAND_ARG_2
));
516 writel(uic_cmnd
->argument3
,
517 (hba
->mmio_base
+ REG_UIC_COMMAND_ARG_3
));
520 writel((uic_cmnd
->command
& COMMAND_OPCODE_MASK
),
521 (hba
->mmio_base
+ REG_UIC_COMMAND
));
525 * ufshcd_map_sg - Map scatter-gather list to prdt
526 * @lrbp - pointer to local reference block
528 * Returns 0 in case of success, non-zero value in case of failure
530 static int ufshcd_map_sg(struct ufshcd_lrb
*lrbp
)
532 struct ufshcd_sg_entry
*prd_table
;
533 struct scatterlist
*sg
;
534 struct scsi_cmnd
*cmd
;
539 sg_segments
= scsi_dma_map(cmd
);
544 lrbp
->utr_descriptor_ptr
->prd_table_length
=
545 cpu_to_le16((u16
) (sg_segments
));
547 prd_table
= (struct ufshcd_sg_entry
*)lrbp
->ucd_prdt_ptr
;
549 scsi_for_each_sg(cmd
, sg
, sg_segments
, i
) {
551 cpu_to_le32(((u32
) sg_dma_len(sg
))-1);
552 prd_table
[i
].base_addr
=
553 cpu_to_le32(lower_32_bits(sg
->dma_address
));
554 prd_table
[i
].upper_addr
=
555 cpu_to_le32(upper_32_bits(sg
->dma_address
));
558 lrbp
->utr_descriptor_ptr
->prd_table_length
= 0;
565 * ufshcd_int_config - enable/disable interrupts
566 * @hba: per adapter instance
567 * @option: interrupt option
569 static void ufshcd_int_config(struct ufs_hba
*hba
, u32 option
)
572 case UFSHCD_INT_ENABLE
:
573 writel(hba
->int_enable_mask
,
574 (hba
->mmio_base
+ REG_INTERRUPT_ENABLE
));
576 case UFSHCD_INT_DISABLE
:
577 if (hba
->ufs_version
== UFSHCI_VERSION_10
)
578 writel(INTERRUPT_DISABLE_MASK_10
,
579 (hba
->mmio_base
+ REG_INTERRUPT_ENABLE
));
581 writel(INTERRUPT_DISABLE_MASK_11
,
582 (hba
->mmio_base
+ REG_INTERRUPT_ENABLE
));
588 * ufshcd_compose_upiu - form UFS Protocol Information Unit(UPIU)
589 * @lrb - pointer to local reference block
591 static void ufshcd_compose_upiu(struct ufshcd_lrb
*lrbp
)
593 struct utp_transfer_req_desc
*req_desc
;
594 struct utp_upiu_cmd
*ucd_cmd_ptr
;
598 ucd_cmd_ptr
= lrbp
->ucd_cmd_ptr
;
599 req_desc
= lrbp
->utr_descriptor_ptr
;
601 switch (lrbp
->command_type
) {
602 case UTP_CMD_TYPE_SCSI
:
603 if (lrbp
->cmd
->sc_data_direction
== DMA_FROM_DEVICE
) {
604 data_direction
= UTP_DEVICE_TO_HOST
;
605 upiu_flags
= UPIU_CMD_FLAGS_READ
;
606 } else if (lrbp
->cmd
->sc_data_direction
== DMA_TO_DEVICE
) {
607 data_direction
= UTP_HOST_TO_DEVICE
;
608 upiu_flags
= UPIU_CMD_FLAGS_WRITE
;
610 data_direction
= UTP_NO_DATA_TRANSFER
;
611 upiu_flags
= UPIU_CMD_FLAGS_NONE
;
614 /* Transfer request descriptor header fields */
615 req_desc
->header
.dword_0
=
616 cpu_to_le32(data_direction
| UTP_SCSI_COMMAND
);
619 * assigning invalid value for command status. Controller
620 * updates OCS on command completion, with the command
623 req_desc
->header
.dword_2
=
624 cpu_to_le32(OCS_INVALID_COMMAND_STATUS
);
626 /* command descriptor fields */
627 ucd_cmd_ptr
->header
.dword_0
=
628 cpu_to_be32(UPIU_HEADER_DWORD(UPIU_TRANSACTION_COMMAND
,
632 ucd_cmd_ptr
->header
.dword_1
=
634 UPIU_HEADER_DWORD(UPIU_COMMAND_SET_TYPE_SCSI
,
639 /* Total EHS length and Data segment length will be zero */
640 ucd_cmd_ptr
->header
.dword_2
= 0;
642 ucd_cmd_ptr
->exp_data_transfer_len
=
643 cpu_to_be32(lrbp
->cmd
->transfersize
);
645 memcpy(ucd_cmd_ptr
->cdb
,
647 (min_t(unsigned short,
651 case UTP_CMD_TYPE_DEV_MANAGE
:
652 /* For query function implementation */
654 case UTP_CMD_TYPE_UFS
:
655 /* For UFS native command implementation */
657 } /* end of switch */
661 * ufshcd_queuecommand - main entry point for SCSI requests
662 * @cmd: command from SCSI Midlayer
663 * @done: call back function
665 * Returns 0 for success, non-zero in case of failure
667 static int ufshcd_queuecommand(struct Scsi_Host
*host
, struct scsi_cmnd
*cmd
)
669 struct ufshcd_lrb
*lrbp
;
675 hba
= shost_priv(host
);
677 tag
= cmd
->request
->tag
;
679 if (hba
->ufshcd_state
!= UFSHCD_STATE_OPERATIONAL
) {
680 err
= SCSI_MLQUEUE_HOST_BUSY
;
684 lrbp
= &hba
->lrb
[tag
];
687 lrbp
->sense_bufflen
= SCSI_SENSE_BUFFERSIZE
;
688 lrbp
->sense_buffer
= cmd
->sense_buffer
;
689 lrbp
->task_tag
= tag
;
690 lrbp
->lun
= cmd
->device
->lun
;
692 lrbp
->command_type
= UTP_CMD_TYPE_SCSI
;
694 /* form UPIU before issuing the command */
695 ufshcd_compose_upiu(lrbp
);
696 err
= ufshcd_map_sg(lrbp
);
700 /* issue command to the controller */
701 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
702 ufshcd_send_command(hba
, tag
);
703 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
709 * ufshcd_memory_alloc - allocate memory for host memory space data structures
710 * @hba: per adapter instance
712 * 1. Allocate DMA memory for Command Descriptor array
713 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
714 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
715 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
717 * 4. Allocate memory for local reference block(lrb).
719 * Returns 0 for success, non-zero in case of failure
721 static int ufshcd_memory_alloc(struct ufs_hba
*hba
)
723 size_t utmrdl_size
, utrdl_size
, ucdl_size
;
725 /* Allocate memory for UTP command descriptors */
726 ucdl_size
= (sizeof(struct utp_transfer_cmd_desc
) * hba
->nutrs
);
727 hba
->ucdl_base_addr
= dma_alloc_coherent(&hba
->pdev
->dev
,
733 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
734 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
735 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
736 * be aligned to 128 bytes as well
738 if (!hba
->ucdl_base_addr
||
739 WARN_ON(hba
->ucdl_dma_addr
& (PAGE_SIZE
- 1))) {
740 dev_err(&hba
->pdev
->dev
,
741 "Command Descriptor Memory allocation failed\n");
746 * Allocate memory for UTP Transfer descriptors
747 * UFSHCI requires 1024 byte alignment of UTRD
749 utrdl_size
= (sizeof(struct utp_transfer_req_desc
) * hba
->nutrs
);
750 hba
->utrdl_base_addr
= dma_alloc_coherent(&hba
->pdev
->dev
,
752 &hba
->utrdl_dma_addr
,
754 if (!hba
->utrdl_base_addr
||
755 WARN_ON(hba
->utrdl_dma_addr
& (PAGE_SIZE
- 1))) {
756 dev_err(&hba
->pdev
->dev
,
757 "Transfer Descriptor Memory allocation failed\n");
762 * Allocate memory for UTP Task Management descriptors
763 * UFSHCI requires 1024 byte alignment of UTMRD
765 utmrdl_size
= sizeof(struct utp_task_req_desc
) * hba
->nutmrs
;
766 hba
->utmrdl_base_addr
= dma_alloc_coherent(&hba
->pdev
->dev
,
768 &hba
->utmrdl_dma_addr
,
770 if (!hba
->utmrdl_base_addr
||
771 WARN_ON(hba
->utmrdl_dma_addr
& (PAGE_SIZE
- 1))) {
772 dev_err(&hba
->pdev
->dev
,
773 "Task Management Descriptor Memory allocation failed\n");
777 /* Allocate memory for local reference block */
778 hba
->lrb
= kcalloc(hba
->nutrs
, sizeof(struct ufshcd_lrb
), GFP_KERNEL
);
780 dev_err(&hba
->pdev
->dev
, "LRB Memory allocation failed\n");
785 ufshcd_free_hba_memory(hba
);
790 * ufshcd_host_memory_configure - configure local reference block with
792 * @hba: per adapter instance
794 * Configure Host memory space
795 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
797 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
799 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
800 * into local reference block.
802 static void ufshcd_host_memory_configure(struct ufs_hba
*hba
)
804 struct utp_transfer_cmd_desc
*cmd_descp
;
805 struct utp_transfer_req_desc
*utrdlp
;
806 dma_addr_t cmd_desc_dma_addr
;
807 dma_addr_t cmd_desc_element_addr
;
813 utrdlp
= hba
->utrdl_base_addr
;
814 cmd_descp
= hba
->ucdl_base_addr
;
817 offsetof(struct utp_transfer_cmd_desc
, response_upiu
);
819 offsetof(struct utp_transfer_cmd_desc
, prd_table
);
821 cmd_desc_size
= sizeof(struct utp_transfer_cmd_desc
);
822 cmd_desc_dma_addr
= hba
->ucdl_dma_addr
;
824 for (i
= 0; i
< hba
->nutrs
; i
++) {
825 /* Configure UTRD with command descriptor base address */
826 cmd_desc_element_addr
=
827 (cmd_desc_dma_addr
+ (cmd_desc_size
* i
));
828 utrdlp
[i
].command_desc_base_addr_lo
=
829 cpu_to_le32(lower_32_bits(cmd_desc_element_addr
));
830 utrdlp
[i
].command_desc_base_addr_hi
=
831 cpu_to_le32(upper_32_bits(cmd_desc_element_addr
));
833 /* Response upiu and prdt offset should be in double words */
834 utrdlp
[i
].response_upiu_offset
=
835 cpu_to_le16((response_offset
>> 2));
836 utrdlp
[i
].prd_table_offset
=
837 cpu_to_le16((prdt_offset
>> 2));
838 utrdlp
[i
].response_upiu_length
=
839 cpu_to_le16(ALIGNED_UPIU_SIZE
);
841 hba
->lrb
[i
].utr_descriptor_ptr
= (utrdlp
+ i
);
842 hba
->lrb
[i
].ucd_cmd_ptr
=
843 (struct utp_upiu_cmd
*)(cmd_descp
+ i
);
844 hba
->lrb
[i
].ucd_rsp_ptr
=
845 (struct utp_upiu_rsp
*)cmd_descp
[i
].response_upiu
;
846 hba
->lrb
[i
].ucd_prdt_ptr
=
847 (struct ufshcd_sg_entry
*)cmd_descp
[i
].prd_table
;
852 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
853 * @hba: per adapter instance
855 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
856 * in order to initialize the Unipro link startup procedure.
857 * Once the Unipro links are up, the device connected to the controller
860 * Returns 0 on success, non-zero value on failure
862 static int ufshcd_dme_link_startup(struct ufs_hba
*hba
)
864 struct uic_command
*uic_cmd
;
867 /* check if controller is ready to accept UIC commands */
868 if (((readl(hba
->mmio_base
+ REG_CONTROLLER_STATUS
)) &
869 UIC_COMMAND_READY
) == 0x0) {
870 dev_err(&hba
->pdev
->dev
,
871 "Controller not ready"
872 " to accept UIC commands\n");
876 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
878 /* form UIC command */
879 uic_cmd
= &hba
->active_uic_cmd
;
880 uic_cmd
->command
= UIC_CMD_DME_LINK_STARTUP
;
881 uic_cmd
->argument1
= 0;
882 uic_cmd
->argument2
= 0;
883 uic_cmd
->argument3
= 0;
885 /* enable UIC related interrupts */
886 hba
->int_enable_mask
|= UIC_COMMAND_COMPL
;
887 ufshcd_int_config(hba
, UFSHCD_INT_ENABLE
);
889 /* sending UIC commands to controller */
890 ufshcd_send_uic_command(hba
, uic_cmd
);
891 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
896 * ufshcd_make_hba_operational - Make UFS controller operational
897 * @hba: per adapter instance
899 * To bring UFS host controller to operational state,
900 * 1. Check if device is present
901 * 2. Configure run-stop-registers
902 * 3. Enable required interrupts
903 * 4. Configure interrupt aggregation
905 * Returns 0 on success, non-zero value on failure
907 static int ufshcd_make_hba_operational(struct ufs_hba
*hba
)
912 /* check if device present */
913 reg
= readl((hba
->mmio_base
+ REG_CONTROLLER_STATUS
));
914 if (ufshcd_is_device_present(reg
)) {
915 dev_err(&hba
->pdev
->dev
, "cc: Device not present\n");
921 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
922 * DEI, HEI bits must be 0
924 if (!(ufshcd_get_lists_status(reg
))) {
925 ufshcd_enable_run_stop_reg(hba
);
927 dev_err(&hba
->pdev
->dev
,
928 "Host controller not ready to process requests");
933 /* Enable required interrupts */
934 hba
->int_enable_mask
|= (UTP_TRANSFER_REQ_COMPL
|
938 CONTROLLER_FATAL_ERROR
|
939 SYSTEM_BUS_FATAL_ERROR
);
940 ufshcd_int_config(hba
, UFSHCD_INT_ENABLE
);
942 /* Configure interrupt aggregation */
943 ufshcd_config_int_aggr(hba
, INT_AGGR_CONFIG
);
945 if (hba
->ufshcd_state
== UFSHCD_STATE_RESET
)
946 scsi_unblock_requests(hba
->host
);
948 hba
->ufshcd_state
= UFSHCD_STATE_OPERATIONAL
;
949 scsi_scan_host(hba
->host
);
955 * ufshcd_hba_enable - initialize the controller
956 * @hba: per adapter instance
958 * The controller resets itself and controller firmware initialization
959 * sequence kicks off. When controller is ready it will set
960 * the Host Controller Enable bit to 1.
962 * Returns 0 on success, non-zero value on failure
964 static int ufshcd_hba_enable(struct ufs_hba
*hba
)
969 * msleep of 1 and 5 used in this function might result in msleep(20),
970 * but it was necessary to send the UFS FPGA to reset mode during
971 * development and testing of this driver. msleep can be changed to
972 * mdelay and retry count can be reduced based on the controller.
974 if (!ufshcd_is_hba_active(hba
)) {
976 /* change controller state to "reset state" */
977 ufshcd_hba_stop(hba
);
980 * This delay is based on the testing done with UFS host
981 * controller FPGA. The delay can be changed based on the
982 * host controller used.
987 /* start controller initialization sequence */
988 ufshcd_hba_start(hba
);
991 * To initialize a UFS host controller HCE bit must be set to 1.
992 * During initialization the HCE bit value changes from 1->0->1.
993 * When the host controller completes initialization sequence
994 * it sets the value of HCE bit to 1. The same HCE bit is read back
995 * to check if the controller has completed initialization sequence.
996 * So without this delay the value HCE = 1, set in the previous
997 * instruction might be read back.
998 * This delay can be changed based on the controller.
1002 /* wait for the host controller to complete initialization */
1004 while (ufshcd_is_hba_active(hba
)) {
1008 dev_err(&hba
->pdev
->dev
,
1009 "Controller enable failed\n");
1018 * ufshcd_initialize_hba - start the initialization process
1019 * @hba: per adapter instance
1021 * 1. Enable the controller via ufshcd_hba_enable.
1022 * 2. Program the Transfer Request List Address with the starting address of
1024 * 3. Program the Task Management Request List Address with starting address
1027 * Returns 0 on success, non-zero value on failure.
1029 static int ufshcd_initialize_hba(struct ufs_hba
*hba
)
1031 if (ufshcd_hba_enable(hba
))
1034 /* Configure UTRL and UTMRL base address registers */
1035 writel(lower_32_bits(hba
->utrdl_dma_addr
),
1036 (hba
->mmio_base
+ REG_UTP_TRANSFER_REQ_LIST_BASE_L
));
1037 writel(upper_32_bits(hba
->utrdl_dma_addr
),
1038 (hba
->mmio_base
+ REG_UTP_TRANSFER_REQ_LIST_BASE_H
));
1039 writel(lower_32_bits(hba
->utmrdl_dma_addr
),
1040 (hba
->mmio_base
+ REG_UTP_TASK_REQ_LIST_BASE_L
));
1041 writel(upper_32_bits(hba
->utmrdl_dma_addr
),
1042 (hba
->mmio_base
+ REG_UTP_TASK_REQ_LIST_BASE_H
));
1044 /* Initialize unipro link startup procedure */
1045 return ufshcd_dme_link_startup(hba
);
1049 * ufshcd_do_reset - reset the host controller
1050 * @hba: per adapter instance
1052 * Returns SUCCESS/FAILED
1054 static int ufshcd_do_reset(struct ufs_hba
*hba
)
1056 struct ufshcd_lrb
*lrbp
;
1057 unsigned long flags
;
1060 /* block commands from midlayer */
1061 scsi_block_requests(hba
->host
);
1063 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
1064 hba
->ufshcd_state
= UFSHCD_STATE_RESET
;
1066 /* send controller to reset state */
1067 ufshcd_hba_stop(hba
);
1068 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
1070 /* abort outstanding commands */
1071 for (tag
= 0; tag
< hba
->nutrs
; tag
++) {
1072 if (test_bit(tag
, &hba
->outstanding_reqs
)) {
1073 lrbp
= &hba
->lrb
[tag
];
1074 scsi_dma_unmap(lrbp
->cmd
);
1075 lrbp
->cmd
->result
= DID_RESET
<< 16;
1076 lrbp
->cmd
->scsi_done(lrbp
->cmd
);
1081 /* clear outstanding request/task bit maps */
1082 hba
->outstanding_reqs
= 0;
1083 hba
->outstanding_tasks
= 0;
1085 /* start the initialization process */
1086 if (ufshcd_initialize_hba(hba
)) {
1087 dev_err(&hba
->pdev
->dev
,
1088 "Reset: Controller initialization failed\n");
1095 * ufshcd_slave_alloc - handle initial SCSI device configurations
1096 * @sdev: pointer to SCSI device
1100 static int ufshcd_slave_alloc(struct scsi_device
*sdev
)
1102 struct ufs_hba
*hba
;
1104 hba
= shost_priv(sdev
->host
);
1105 sdev
->tagged_supported
= 1;
1107 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
1108 sdev
->use_10_for_ms
= 1;
1109 scsi_set_tag_type(sdev
, MSG_SIMPLE_TAG
);
1112 * Inform SCSI Midlayer that the LUN queue depth is same as the
1113 * controller queue depth. If a LUN queue depth is less than the
1114 * controller queue depth and if the LUN reports
1115 * SAM_STAT_TASK_SET_FULL, the LUN queue depth will be adjusted
1116 * with scsi_adjust_queue_depth.
1118 scsi_activate_tcq(sdev
, hba
->nutrs
);
1123 * ufshcd_slave_destroy - remove SCSI device configurations
1124 * @sdev: pointer to SCSI device
1126 static void ufshcd_slave_destroy(struct scsi_device
*sdev
)
1128 struct ufs_hba
*hba
;
1130 hba
= shost_priv(sdev
->host
);
1131 scsi_deactivate_tcq(sdev
, hba
->nutrs
);
1135 * ufshcd_task_req_compl - handle task management request completion
1136 * @hba: per adapter instance
1137 * @index: index of the completed request
1139 * Returns SUCCESS/FAILED
1141 static int ufshcd_task_req_compl(struct ufs_hba
*hba
, u32 index
)
1143 struct utp_task_req_desc
*task_req_descp
;
1144 struct utp_upiu_task_rsp
*task_rsp_upiup
;
1145 unsigned long flags
;
1149 spin_lock_irqsave(hba
->host
->host_lock
, flags
);
1151 /* Clear completed tasks from outstanding_tasks */
1152 __clear_bit(index
, &hba
->outstanding_tasks
);
1154 task_req_descp
= hba
->utmrdl_base_addr
;
1155 ocs_value
= ufshcd_get_tmr_ocs(&task_req_descp
[index
]);
1157 if (ocs_value
== OCS_SUCCESS
) {
1158 task_rsp_upiup
= (struct utp_upiu_task_rsp
*)
1159 task_req_descp
[index
].task_rsp_upiu
;
1160 task_result
= be32_to_cpu(task_rsp_upiup
->header
.dword_1
);
1161 task_result
= ((task_result
& MASK_TASK_RESPONSE
) >> 8);
1163 if (task_result
!= UPIU_TASK_MANAGEMENT_FUNC_COMPL
&&
1164 task_result
!= UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED
)
1165 task_result
= FAILED
;
1167 task_result
= FAILED
;
1168 dev_err(&hba
->pdev
->dev
,
1169 "trc: Invalid ocs = %x\n", ocs_value
);
1171 spin_unlock_irqrestore(hba
->host
->host_lock
, flags
);
1176 * ufshcd_adjust_lun_qdepth - Update LUN queue depth if device responds with
1177 * SAM_STAT_TASK_SET_FULL SCSI command status.
1178 * @cmd: pointer to SCSI command
1180 static void ufshcd_adjust_lun_qdepth(struct scsi_cmnd
*cmd
)
1182 struct ufs_hba
*hba
;
1186 hba
= shost_priv(cmd
->device
->host
);
1189 * LUN queue depth can be obtained by counting outstanding commands
1192 for (i
= 0; i
< hba
->nutrs
; i
++) {
1193 if (test_bit(i
, &hba
->outstanding_reqs
)) {
1196 * Check if the outstanding command belongs
1197 * to the LUN which reported SAM_STAT_TASK_SET_FULL.
1199 if (cmd
->device
->lun
== hba
->lrb
[i
].lun
)
1205 * LUN queue depth will be total outstanding commands, except the
1206 * command for which the LUN reported SAM_STAT_TASK_SET_FULL.
1208 scsi_adjust_queue_depth(cmd
->device
, MSG_SIMPLE_TAG
, lun_qdepth
- 1);
1212 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
1213 * @lrb: pointer to local reference block of completed command
1214 * @scsi_status: SCSI command status
1216 * Returns value base on SCSI command status
1219 ufshcd_scsi_cmd_status(struct ufshcd_lrb
*lrbp
, int scsi_status
)
1223 switch (scsi_status
) {
1225 result
|= DID_OK
<< 16 |
1226 COMMAND_COMPLETE
<< 8 |
1229 case SAM_STAT_CHECK_CONDITION
:
1230 result
|= DID_OK
<< 16 |
1231 COMMAND_COMPLETE
<< 8 |
1232 SAM_STAT_CHECK_CONDITION
;
1233 ufshcd_copy_sense_data(lrbp
);
1236 result
|= SAM_STAT_BUSY
;
1238 case SAM_STAT_TASK_SET_FULL
:
1241 * If a LUN reports SAM_STAT_TASK_SET_FULL, then the LUN queue
1242 * depth needs to be adjusted to the exact number of
1243 * outstanding commands the LUN can handle at any given time.
1245 ufshcd_adjust_lun_qdepth(lrbp
->cmd
);
1246 result
|= SAM_STAT_TASK_SET_FULL
;
1248 case SAM_STAT_TASK_ABORTED
:
1249 result
|= SAM_STAT_TASK_ABORTED
;
1252 result
|= DID_ERROR
<< 16;
1254 } /* end of switch */
1260 * ufshcd_transfer_rsp_status - Get overall status of the response
1261 * @hba: per adapter instance
1262 * @lrb: pointer to local reference block of completed command
1264 * Returns result of the command to notify SCSI midlayer
1267 ufshcd_transfer_rsp_status(struct ufs_hba
*hba
, struct ufshcd_lrb
*lrbp
)
1273 /* overall command status of utrd */
1274 ocs
= ufshcd_get_tr_ocs(lrbp
);
1279 /* check if the returned transfer response is valid */
1280 result
= ufshcd_is_valid_req_rsp(lrbp
->ucd_rsp_ptr
);
1282 dev_err(&hba
->pdev
->dev
,
1283 "Invalid response = %x\n", result
);
1288 * get the response UPIU result to extract
1289 * the SCSI command status
1291 result
= ufshcd_get_rsp_upiu_result(lrbp
->ucd_rsp_ptr
);
1294 * get the result based on SCSI status response
1295 * to notify the SCSI midlayer of the command status
1297 scsi_status
= result
& MASK_SCSI_STATUS
;
1298 result
= ufshcd_scsi_cmd_status(lrbp
, scsi_status
);
1301 result
|= DID_ABORT
<< 16;
1303 case OCS_INVALID_CMD_TABLE_ATTR
:
1304 case OCS_INVALID_PRDT_ATTR
:
1305 case OCS_MISMATCH_DATA_BUF_SIZE
:
1306 case OCS_MISMATCH_RESP_UPIU_SIZE
:
1307 case OCS_PEER_COMM_FAILURE
:
1308 case OCS_FATAL_ERROR
:
1310 result
|= DID_ERROR
<< 16;
1311 dev_err(&hba
->pdev
->dev
,
1312 "OCS error from controller = %x\n", ocs
);
1314 } /* end of switch */
1320 * ufshcd_transfer_req_compl - handle SCSI and query command completion
1321 * @hba: per adapter instance
1323 static void ufshcd_transfer_req_compl(struct ufs_hba
*hba
)
1325 struct ufshcd_lrb
*lrb
;
1326 unsigned long completed_reqs
;
1333 readl(hba
->mmio_base
+ REG_UTP_TRANSFER_REQ_DOOR_BELL
);
1334 completed_reqs
= tr_doorbell
^ hba
->outstanding_reqs
;
1336 for (index
= 0; index
< hba
->nutrs
; index
++) {
1337 if (test_bit(index
, &completed_reqs
)) {
1339 result
= ufshcd_transfer_rsp_status(hba
, &lrb
[index
]);
1341 if (lrb
[index
].cmd
) {
1342 scsi_dma_unmap(lrb
[index
].cmd
);
1343 lrb
[index
].cmd
->result
= result
;
1344 lrb
[index
].cmd
->scsi_done(lrb
[index
].cmd
);
1346 /* Mark completed command as NULL in LRB */
1347 lrb
[index
].cmd
= NULL
;
1352 /* clear corresponding bits of completed commands */
1353 hba
->outstanding_reqs
^= completed_reqs
;
1355 /* Reset interrupt aggregation counters */
1356 ufshcd_config_int_aggr(hba
, INT_AGGR_RESET
);
1360 * ufshcd_uic_cc_handler - handle UIC command completion
1361 * @work: pointer to a work queue structure
1363 * Returns 0 on success, non-zero value on failure
1365 static void ufshcd_uic_cc_handler (struct work_struct
*work
)
1367 struct ufs_hba
*hba
;
1369 hba
= container_of(work
, struct ufs_hba
, uic_workq
);
1371 if ((hba
->active_uic_cmd
.command
== UIC_CMD_DME_LINK_STARTUP
) &&
1372 !(ufshcd_get_uic_cmd_result(hba
))) {
1374 if (ufshcd_make_hba_operational(hba
))
1375 dev_err(&hba
->pdev
->dev
,
1376 "cc: hba not operational state\n");
1382 * ufshcd_fatal_err_handler - handle fatal errors
1383 * @hba: per adapter instance
1385 static void ufshcd_fatal_err_handler(struct work_struct
*work
)
1387 struct ufs_hba
*hba
;
1388 hba
= container_of(work
, struct ufs_hba
, feh_workq
);
1390 /* check if reset is already in progress */
1391 if (hba
->ufshcd_state
!= UFSHCD_STATE_RESET
)
1392 ufshcd_do_reset(hba
);
1396 * ufshcd_err_handler - Check for fatal errors
1397 * @work: pointer to a work queue structure
1399 static void ufshcd_err_handler(struct ufs_hba
*hba
)
1403 if (hba
->errors
& INT_FATAL_ERRORS
)
1406 if (hba
->errors
& UIC_ERROR
) {
1408 reg
= readl(hba
->mmio_base
+
1409 REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER
);
1410 if (reg
& UIC_DATA_LINK_LAYER_ERROR_PA_INIT
)
1415 hba
->ufshcd_state
= UFSHCD_STATE_ERROR
;
1416 schedule_work(&hba
->feh_workq
);
1420 * ufshcd_tmc_handler - handle task management function completion
1421 * @hba: per adapter instance
1423 static void ufshcd_tmc_handler(struct ufs_hba
*hba
)
1427 tm_doorbell
= readl(hba
->mmio_base
+ REG_UTP_TASK_REQ_DOOR_BELL
);
1428 hba
->tm_condition
= tm_doorbell
^ hba
->outstanding_tasks
;
1429 wake_up_interruptible(&hba
->ufshcd_tm_wait_queue
);
1433 * ufshcd_sl_intr - Interrupt service routine
1434 * @hba: per adapter instance
1435 * @intr_status: contains interrupts generated by the controller
1437 static void ufshcd_sl_intr(struct ufs_hba
*hba
, u32 intr_status
)
1439 hba
->errors
= UFSHCD_ERROR_MASK
& intr_status
;
1441 ufshcd_err_handler(hba
);
1443 if (intr_status
& UIC_COMMAND_COMPL
)
1444 schedule_work(&hba
->uic_workq
);
1446 if (intr_status
& UTP_TASK_REQ_COMPL
)
1447 ufshcd_tmc_handler(hba
);
1449 if (intr_status
& UTP_TRANSFER_REQ_COMPL
)
1450 ufshcd_transfer_req_compl(hba
);
1454 * ufshcd_intr - Main interrupt service routine
1456 * @__hba: pointer to adapter instance
1458 * Returns IRQ_HANDLED - If interrupt is valid
1459 * IRQ_NONE - If invalid interrupt
1461 static irqreturn_t
ufshcd_intr(int irq
, void *__hba
)
1464 irqreturn_t retval
= IRQ_NONE
;
1465 struct ufs_hba
*hba
= __hba
;
1467 spin_lock(hba
->host
->host_lock
);
1468 intr_status
= readl(hba
->mmio_base
+ REG_INTERRUPT_STATUS
);
1471 ufshcd_sl_intr(hba
, intr_status
);
1473 /* If UFSHCI 1.0 then clear interrupt status register */
1474 if (hba
->ufs_version
== UFSHCI_VERSION_10
)
1476 (hba
->mmio_base
+ REG_INTERRUPT_STATUS
));
1477 retval
= IRQ_HANDLED
;
1479 spin_unlock(hba
->host
->host_lock
);
1484 * ufshcd_issue_tm_cmd - issues task management commands to controller
1485 * @hba: per adapter instance
1486 * @lrbp: pointer to local reference block
1488 * Returns SUCCESS/FAILED
1491 ufshcd_issue_tm_cmd(struct ufs_hba
*hba
,
1492 struct ufshcd_lrb
*lrbp
,
1495 struct utp_task_req_desc
*task_req_descp
;
1496 struct utp_upiu_task_req
*task_req_upiup
;
1497 struct Scsi_Host
*host
;
1498 unsigned long flags
;
1504 spin_lock_irqsave(host
->host_lock
, flags
);
1506 /* If task management queue is full */
1507 free_slot
= ufshcd_get_tm_free_slot(hba
);
1508 if (free_slot
>= hba
->nutmrs
) {
1509 spin_unlock_irqrestore(host
->host_lock
, flags
);
1510 dev_err(&hba
->pdev
->dev
, "Task management queue full\n");
1515 task_req_descp
= hba
->utmrdl_base_addr
;
1516 task_req_descp
+= free_slot
;
1518 /* Configure task request descriptor */
1519 task_req_descp
->header
.dword_0
= cpu_to_le32(UTP_REQ_DESC_INT_CMD
);
1520 task_req_descp
->header
.dword_2
=
1521 cpu_to_le32(OCS_INVALID_COMMAND_STATUS
);
1523 /* Configure task request UPIU */
1525 (struct utp_upiu_task_req
*) task_req_descp
->task_req_upiu
;
1526 task_req_upiup
->header
.dword_0
=
1527 cpu_to_be32(UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ
, 0,
1528 lrbp
->lun
, lrbp
->task_tag
));
1529 task_req_upiup
->header
.dword_1
=
1530 cpu_to_be32(UPIU_HEADER_DWORD(0, tm_function
, 0, 0));
1532 task_req_upiup
->input_param1
= lrbp
->lun
;
1533 task_req_upiup
->input_param1
=
1534 cpu_to_be32(task_req_upiup
->input_param1
);
1535 task_req_upiup
->input_param2
= lrbp
->task_tag
;
1536 task_req_upiup
->input_param2
=
1537 cpu_to_be32(task_req_upiup
->input_param2
);
1539 /* send command to the controller */
1540 __set_bit(free_slot
, &hba
->outstanding_tasks
);
1541 writel((1 << free_slot
),
1542 (hba
->mmio_base
+ REG_UTP_TASK_REQ_DOOR_BELL
));
1544 spin_unlock_irqrestore(host
->host_lock
, flags
);
1546 /* wait until the task management command is completed */
1548 wait_event_interruptible_timeout(hba
->ufshcd_tm_wait_queue
,
1549 (test_bit(free_slot
,
1550 &hba
->tm_condition
) != 0),
1553 dev_err(&hba
->pdev
->dev
,
1554 "Task management command timed-out\n");
1558 clear_bit(free_slot
, &hba
->tm_condition
);
1559 return ufshcd_task_req_compl(hba
, free_slot
);
1565 * ufshcd_device_reset - reset device and abort all the pending commands
1566 * @cmd: SCSI command pointer
1568 * Returns SUCCESS/FAILED
1570 static int ufshcd_device_reset(struct scsi_cmnd
*cmd
)
1572 struct Scsi_Host
*host
;
1573 struct ufs_hba
*hba
;
1578 host
= cmd
->device
->host
;
1579 hba
= shost_priv(host
);
1580 tag
= cmd
->request
->tag
;
1582 err
= ufshcd_issue_tm_cmd(hba
, &hba
->lrb
[tag
], UFS_LOGICAL_RESET
);
1586 for (pos
= 0; pos
< hba
->nutrs
; pos
++) {
1587 if (test_bit(pos
, &hba
->outstanding_reqs
) &&
1588 (hba
->lrb
[tag
].lun
== hba
->lrb
[pos
].lun
)) {
1590 /* clear the respective UTRLCLR register bit */
1591 ufshcd_utrl_clear(hba
, pos
);
1593 clear_bit(pos
, &hba
->outstanding_reqs
);
1595 if (hba
->lrb
[pos
].cmd
) {
1596 scsi_dma_unmap(hba
->lrb
[pos
].cmd
);
1597 hba
->lrb
[pos
].cmd
->result
=
1599 hba
->lrb
[pos
].cmd
->scsi_done(cmd
);
1600 hba
->lrb
[pos
].cmd
= NULL
;
1609 * ufshcd_host_reset - Main reset function registered with scsi layer
1610 * @cmd: SCSI command pointer
1612 * Returns SUCCESS/FAILED
1614 static int ufshcd_host_reset(struct scsi_cmnd
*cmd
)
1616 struct ufs_hba
*hba
;
1618 hba
= shost_priv(cmd
->device
->host
);
1620 if (hba
->ufshcd_state
== UFSHCD_STATE_RESET
)
1623 return (ufshcd_do_reset(hba
) == SUCCESS
) ? SUCCESS
: FAILED
;
1627 * ufshcd_abort - abort a specific command
1628 * @cmd: SCSI command pointer
1630 * Returns SUCCESS/FAILED
1632 static int ufshcd_abort(struct scsi_cmnd
*cmd
)
1634 struct Scsi_Host
*host
;
1635 struct ufs_hba
*hba
;
1636 unsigned long flags
;
1640 host
= cmd
->device
->host
;
1641 hba
= shost_priv(host
);
1642 tag
= cmd
->request
->tag
;
1644 spin_lock_irqsave(host
->host_lock
, flags
);
1646 /* check if command is still pending */
1647 if (!(test_bit(tag
, &hba
->outstanding_reqs
))) {
1649 spin_unlock_irqrestore(host
->host_lock
, flags
);
1652 spin_unlock_irqrestore(host
->host_lock
, flags
);
1654 err
= ufshcd_issue_tm_cmd(hba
, &hba
->lrb
[tag
], UFS_ABORT_TASK
);
1658 scsi_dma_unmap(cmd
);
1660 spin_lock_irqsave(host
->host_lock
, flags
);
1662 /* clear the respective UTRLCLR register bit */
1663 ufshcd_utrl_clear(hba
, tag
);
1665 __clear_bit(tag
, &hba
->outstanding_reqs
);
1666 hba
->lrb
[tag
].cmd
= NULL
;
1667 spin_unlock_irqrestore(host
->host_lock
, flags
);
1672 static struct scsi_host_template ufshcd_driver_template
= {
1673 .module
= THIS_MODULE
,
1675 .proc_name
= UFSHCD
,
1676 .queuecommand
= ufshcd_queuecommand
,
1677 .slave_alloc
= ufshcd_slave_alloc
,
1678 .slave_destroy
= ufshcd_slave_destroy
,
1679 .eh_abort_handler
= ufshcd_abort
,
1680 .eh_device_reset_handler
= ufshcd_device_reset
,
1681 .eh_host_reset_handler
= ufshcd_host_reset
,
1683 .sg_tablesize
= SG_ALL
,
1684 .cmd_per_lun
= UFSHCD_CMD_PER_LUN
,
1685 .can_queue
= UFSHCD_CAN_QUEUE
,
1689 * ufshcd_shutdown - main function to put the controller in reset state
1690 * @pdev: pointer to PCI device handle
1692 static void ufshcd_shutdown(struct pci_dev
*pdev
)
1694 ufshcd_hba_stop((struct ufs_hba
*)pci_get_drvdata(pdev
));
1699 * ufshcd_suspend - suspend power management function
1700 * @pdev: pointer to PCI device handle
1701 * @state: power state
1705 static int ufshcd_suspend(struct pci_dev
*pdev
, pm_message_t state
)
1709 * 1. Block SCSI requests from SCSI midlayer
1710 * 2. Change the internal driver state to non operational
1711 * 3. Set UTRLRSR and UTMRLRSR bits to zero
1712 * 4. Wait until outstanding commands are completed
1713 * 5. Set HCE to zero to send the UFS host controller to reset state
1720 * ufshcd_resume - resume power management function
1721 * @pdev: pointer to PCI device handle
1725 static int ufshcd_resume(struct pci_dev
*pdev
)
1729 * 1. Set HCE to 1, to start the UFS host controller
1730 * initialization process
1731 * 2. Set UTRLRSR and UTMRLRSR bits to 1
1732 * 3. Change the internal driver state to operational
1733 * 4. Unblock SCSI requests from SCSI midlayer
1738 #endif /* CONFIG_PM */
1741 * ufshcd_hba_free - free allocated memory for
1742 * host memory space data structures
1743 * @hba: per adapter instance
1745 static void ufshcd_hba_free(struct ufs_hba
*hba
)
1747 iounmap(hba
->mmio_base
);
1748 ufshcd_free_hba_memory(hba
);
1749 pci_release_regions(hba
->pdev
);
1753 * ufshcd_remove - de-allocate PCI/SCSI host and host memory space
1754 * data structure memory
1755 * @pdev - pointer to PCI handle
1757 static void ufshcd_remove(struct pci_dev
*pdev
)
1759 struct ufs_hba
*hba
= pci_get_drvdata(pdev
);
1761 /* disable interrupts */
1762 ufshcd_int_config(hba
, UFSHCD_INT_DISABLE
);
1763 free_irq(pdev
->irq
, hba
);
1765 ufshcd_hba_stop(hba
);
1766 ufshcd_hba_free(hba
);
1768 scsi_remove_host(hba
->host
);
1769 scsi_host_put(hba
->host
);
1770 pci_set_drvdata(pdev
, NULL
);
1771 pci_clear_master(pdev
);
1772 pci_disable_device(pdev
);
1776 * ufshcd_set_dma_mask - Set dma mask based on the controller
1777 * addressing capability
1778 * @pdev: PCI device structure
1780 * Returns 0 for success, non-zero for failure
1782 static int ufshcd_set_dma_mask(struct ufs_hba
*hba
)
1788 * If controller supports 64 bit addressing mode, then set the DMA
1789 * mask to 64-bit, else set the DMA mask to 32-bit
1791 if (hba
->capabilities
& MASK_64_ADDRESSING_SUPPORT
)
1792 dma_mask
= DMA_BIT_MASK(64);
1794 dma_mask
= DMA_BIT_MASK(32);
1796 err
= pci_set_dma_mask(hba
->pdev
, dma_mask
);
1800 err
= pci_set_consistent_dma_mask(hba
->pdev
, dma_mask
);
1806 * ufshcd_probe - probe routine of the driver
1807 * @pdev: pointer to PCI device handle
1808 * @id: PCI device id
1810 * Returns 0 on success, non-zero value on failure
1812 static int __devinit
1813 ufshcd_probe(struct pci_dev
*pdev
, const struct pci_device_id
*id
)
1815 struct Scsi_Host
*host
;
1816 struct ufs_hba
*hba
;
1819 err
= pci_enable_device(pdev
);
1821 dev_err(&pdev
->dev
, "pci_enable_device failed\n");
1825 pci_set_master(pdev
);
1827 host
= scsi_host_alloc(&ufshcd_driver_template
,
1828 sizeof(struct ufs_hba
));
1830 dev_err(&pdev
->dev
, "scsi_host_alloc failed\n");
1834 hba
= shost_priv(host
);
1836 err
= pci_request_regions(pdev
, UFSHCD
);
1838 dev_err(&pdev
->dev
, "request regions failed\n");
1842 hba
->mmio_base
= pci_ioremap_bar(pdev
, 0);
1843 if (!hba
->mmio_base
) {
1844 dev_err(&pdev
->dev
, "memory map failed\n");
1846 goto out_release_regions
;
1852 /* Read capabilities registers */
1853 ufshcd_hba_capabilities(hba
);
1855 /* Get UFS version supported by the controller */
1856 hba
->ufs_version
= ufshcd_get_ufs_version(hba
);
1858 err
= ufshcd_set_dma_mask(hba
);
1860 dev_err(&pdev
->dev
, "set dma mask failed\n");
1864 /* Allocate memory for host memory space */
1865 err
= ufshcd_memory_alloc(hba
);
1867 dev_err(&pdev
->dev
, "Memory allocation failed\n");
1872 ufshcd_host_memory_configure(hba
);
1874 host
->can_queue
= hba
->nutrs
;
1875 host
->cmd_per_lun
= hba
->nutrs
;
1876 host
->max_id
= UFSHCD_MAX_ID
;
1877 host
->max_lun
= UFSHCD_MAX_LUNS
;
1878 host
->max_channel
= UFSHCD_MAX_CHANNEL
;
1879 host
->unique_id
= host
->host_no
;
1880 host
->max_cmd_len
= MAX_CDB_SIZE
;
1882 /* Initailize wait queue for task management */
1883 init_waitqueue_head(&hba
->ufshcd_tm_wait_queue
);
1885 /* Initialize work queues */
1886 INIT_WORK(&hba
->uic_workq
, ufshcd_uic_cc_handler
);
1887 INIT_WORK(&hba
->feh_workq
, ufshcd_fatal_err_handler
);
1889 /* IRQ registration */
1890 err
= request_irq(pdev
->irq
, ufshcd_intr
, IRQF_SHARED
, UFSHCD
, hba
);
1892 dev_err(&pdev
->dev
, "request irq failed\n");
1896 /* Enable SCSI tag mapping */
1897 err
= scsi_init_shared_tag_map(host
, host
->can_queue
);
1899 dev_err(&pdev
->dev
, "init shared queue failed\n");
1903 pci_set_drvdata(pdev
, hba
);
1905 err
= scsi_add_host(host
, &pdev
->dev
);
1907 dev_err(&pdev
->dev
, "scsi_add_host failed\n");
1911 /* Initialization routine */
1912 err
= ufshcd_initialize_hba(hba
);
1914 dev_err(&pdev
->dev
, "Initialization failed\n");
1921 free_irq(pdev
->irq
, hba
);
1923 ufshcd_free_hba_memory(hba
);
1925 iounmap(hba
->mmio_base
);
1926 out_release_regions
:
1927 pci_release_regions(pdev
);
1929 scsi_host_put(host
);
1930 pci_clear_master(pdev
);
1931 pci_disable_device(pdev
);
1936 static DEFINE_PCI_DEVICE_TABLE(ufshcd_pci_tbl
) = {
1937 { PCI_VENDOR_ID_SAMSUNG
, 0xC00C, PCI_ANY_ID
, PCI_ANY_ID
, 0, 0, 0 },
1938 { } /* terminate list */
1941 MODULE_DEVICE_TABLE(pci
, ufshcd_pci_tbl
);
1943 static struct pci_driver ufshcd_pci_driver
= {
1945 .id_table
= ufshcd_pci_tbl
,
1946 .probe
= ufshcd_probe
,
1947 .remove
= __devexit_p(ufshcd_remove
),
1948 .shutdown
= ufshcd_shutdown
,
1950 .suspend
= ufshcd_suspend
,
1951 .resume
= ufshcd_resume
,
1956 * ufshcd_init - Driver registration routine
1958 static int __init
ufshcd_init(void)
1960 return pci_register_driver(&ufshcd_pci_driver
);
1962 module_init(ufshcd_init
);
1965 * ufshcd_exit - Driver exit clean-up routine
1967 static void __exit
ufshcd_exit(void)
1969 pci_unregister_driver(&ufshcd_pci_driver
);
1971 module_exit(ufshcd_exit
);
1974 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>, "
1975 "Vinayak Holikatti <h.vinayak@samsung.com>");
1976 MODULE_DESCRIPTION("Generic UFS host controller driver");
1977 MODULE_LICENSE("GPL");
1978 MODULE_VERSION(UFSHCD_DRIVER_VERSION
);