2 * Driver for sTec s1120 PCIe SSDs. sTec was acquired in 2013 by HGST and HGST
3 * was acquired by Western Digital in 2012.
5 * Copyright 2012 sTec, Inc.
6 * Copyright (c) 2017 Western Digital Corporation or its affiliates.
8 * This file is part of the Linux kernel, and is made available under
9 * the terms of the GNU General Public License version 2.
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/pci.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/blkdev.h>
19 #include <linux/blk-mq.h>
20 #include <linux/sched.h>
21 #include <linux/interrupt.h>
22 #include <linux/compiler.h>
23 #include <linux/workqueue.h>
24 #include <linux/delay.h>
25 #include <linux/time.h>
26 #include <linux/hdreg.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/completion.h>
29 #include <linux/scatterlist.h>
30 #include <linux/version.h>
31 #include <linux/err.h>
32 #include <linux/aer.h>
33 #include <linux/wait.h>
34 #include <linux/stringify.h>
35 #include <linux/slab_def.h>
36 #include <scsi/scsi.h>
39 #include <linux/uaccess.h>
40 #include <asm/unaligned.h>
42 #include "skd_s1120.h"
44 static int skd_dbg_level
;
45 static int skd_isr_comp_limit
= 4;
47 #define SKD_ASSERT(expr) \
49 if (unlikely(!(expr))) { \
50 pr_err("Assertion failed! %s,%s,%s,line=%d\n", \
51 # expr, __FILE__, __func__, __LINE__); \
55 #define DRV_NAME "skd"
56 #define PFX DRV_NAME ": "
58 MODULE_LICENSE("GPL");
60 MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver");
62 #define PCI_VENDOR_ID_STEC 0x1B39
63 #define PCI_DEVICE_ID_S1120 0x0001
65 #define SKD_FUA_NV (1 << 1)
66 #define SKD_MINORS_PER_DEVICE 16
68 #define SKD_MAX_QUEUE_DEPTH 200u
70 #define SKD_PAUSE_TIMEOUT (5 * 1000)
72 #define SKD_N_FITMSG_BYTES (512u)
73 #define SKD_MAX_REQ_PER_MSG 14
75 #define SKD_N_SPECIAL_FITMSG_BYTES (128u)
77 /* SG elements are 32 bytes, so we can make this 4096 and still be under the
78 * 128KB limit. That allows 4096*4K = 16M xfer size
80 #define SKD_N_SG_PER_REQ_DEFAULT 256u
82 #define SKD_N_COMPLETION_ENTRY 256u
83 #define SKD_N_READ_CAP_BYTES (8u)
85 #define SKD_N_INTERNAL_BYTES (512u)
87 #define SKD_SKCOMP_SIZE \
88 ((sizeof(struct fit_completion_entry_v1) + \
89 sizeof(struct fit_comp_error_info)) * SKD_N_COMPLETION_ENTRY)
91 /* 5 bits of uniqifier, 0xF800 */
92 #define SKD_ID_TABLE_MASK (3u << 8u)
93 #define SKD_ID_RW_REQUEST (0u << 8u)
94 #define SKD_ID_INTERNAL (1u << 8u)
95 #define SKD_ID_FIT_MSG (3u << 8u)
96 #define SKD_ID_SLOT_MASK 0x00FFu
97 #define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
99 #define SKD_N_MAX_SECTORS 2048u
101 #define SKD_MAX_RETRIES 2u
103 #define SKD_TIMER_SECONDS(seconds) (seconds)
104 #define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
106 #define INQ_STD_NBYTES 36
108 enum skd_drvr_state
{
112 SKD_DRVR_STATE_STARTING
,
113 SKD_DRVR_STATE_ONLINE
,
114 SKD_DRVR_STATE_PAUSING
,
115 SKD_DRVR_STATE_PAUSED
,
116 SKD_DRVR_STATE_RESTARTING
,
117 SKD_DRVR_STATE_RESUMING
,
118 SKD_DRVR_STATE_STOPPING
,
119 SKD_DRVR_STATE_FAULT
,
120 SKD_DRVR_STATE_DISAPPEARED
,
121 SKD_DRVR_STATE_PROTOCOL_MISMATCH
,
122 SKD_DRVR_STATE_BUSY_ERASE
,
123 SKD_DRVR_STATE_BUSY_SANITIZE
,
124 SKD_DRVR_STATE_BUSY_IMMINENT
,
125 SKD_DRVR_STATE_WAIT_BOOT
,
126 SKD_DRVR_STATE_SYNCING
,
129 #define SKD_WAIT_BOOT_TIMO SKD_TIMER_SECONDS(90u)
130 #define SKD_STARTING_TIMO SKD_TIMER_SECONDS(8u)
131 #define SKD_RESTARTING_TIMO SKD_TIMER_MINUTES(4u)
132 #define SKD_BUSY_TIMO SKD_TIMER_MINUTES(20u)
133 #define SKD_STARTED_BUSY_TIMO SKD_TIMER_SECONDS(60u)
134 #define SKD_START_WAIT_SECONDS 90u
140 SKD_REQ_STATE_COMPLETED
,
141 SKD_REQ_STATE_TIMEOUT
,
144 enum skd_check_status_action
{
145 SKD_CHECK_STATUS_REPORT_GOOD
,
146 SKD_CHECK_STATUS_REPORT_SMART_ALERT
,
147 SKD_CHECK_STATUS_REQUEUE_REQUEST
,
148 SKD_CHECK_STATUS_REPORT_ERROR
,
149 SKD_CHECK_STATUS_BUSY_IMMINENT
,
153 struct fit_msg_hdr fmh
;
154 struct skd_scsi_request scsi
[SKD_MAX_REQ_PER_MSG
];
157 struct skd_fitmsg_context
{
162 struct skd_msg_buf
*msg_buf
;
163 dma_addr_t mb_dma_address
;
166 struct skd_request_context
{
167 enum skd_req_state state
;
174 enum dma_data_direction data_dir
;
175 struct scatterlist
*sg
;
179 struct fit_sg_descriptor
*sksg_list
;
180 dma_addr_t sksg_dma_address
;
182 struct fit_completion_entry_v1 completion
;
184 struct fit_comp_error_info err_info
;
189 struct skd_special_context
{
190 struct skd_request_context req
;
193 dma_addr_t db_dma_address
;
195 struct skd_msg_buf
*msg_buf
;
196 dma_addr_t mb_dma_address
;
199 typedef enum skd_irq_type
{
205 #define SKD_MAX_BARS 2
208 void __iomem
*mem_map
[SKD_MAX_BARS
];
209 resource_size_t mem_phys
[SKD_MAX_BARS
];
210 u32 mem_size
[SKD_MAX_BARS
];
212 struct skd_msix_entry
*msix_entries
;
214 struct pci_dev
*pdev
;
215 int pcie_error_reporting_is_enabled
;
218 struct gendisk
*disk
;
219 struct blk_mq_tag_set tag_set
;
220 struct request_queue
*queue
;
221 struct skd_fitmsg_context
*skmsg
;
222 struct device
*class_dev
;
230 enum skd_drvr_state state
;
233 u32 cur_max_queue_depth
;
234 u32 queue_low_water_mark
;
235 u32 dev_max_queue_depth
;
237 u32 num_fitmsg_context
;
240 struct skd_fitmsg_context
*skmsg_table
;
242 struct skd_special_context internal_skspcl
;
243 u32 read_cap_blocksize
;
244 u32 read_cap_last_lba
;
245 int read_cap_is_valid
;
246 int inquiry_is_valid
;
247 u8 inq_serial_num
[13]; /*12 chars plus null term */
251 struct kmem_cache
*msgbuf_cache
;
252 struct kmem_cache
*sglist_cache
;
253 struct kmem_cache
*databuf_cache
;
254 struct fit_completion_entry_v1
*skcomp_table
;
255 struct fit_comp_error_info
*skerr_table
;
256 dma_addr_t cq_dma_address
;
258 wait_queue_head_t waitq
;
260 struct timer_list timer
;
270 u32 connect_time_stamp
;
272 #define SKD_MAX_CONNECT_RETRIES 16
277 struct work_struct start_queue
;
278 struct work_struct completion_worker
;
281 #define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
282 #define SKD_READL(DEV, OFF) skd_reg_read32(DEV, OFF)
283 #define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
285 static inline u32
skd_reg_read32(struct skd_device
*skdev
, u32 offset
)
287 u32 val
= readl(skdev
->mem_map
[1] + offset
);
289 if (unlikely(skdev
->dbg_level
>= 2))
290 dev_dbg(&skdev
->pdev
->dev
, "offset %x = %x\n", offset
, val
);
294 static inline void skd_reg_write32(struct skd_device
*skdev
, u32 val
,
297 writel(val
, skdev
->mem_map
[1] + offset
);
298 if (unlikely(skdev
->dbg_level
>= 2))
299 dev_dbg(&skdev
->pdev
->dev
, "offset %x = %x\n", offset
, val
);
302 static inline void skd_reg_write64(struct skd_device
*skdev
, u64 val
,
305 writeq(val
, skdev
->mem_map
[1] + offset
);
306 if (unlikely(skdev
->dbg_level
>= 2))
307 dev_dbg(&skdev
->pdev
->dev
, "offset %x = %016llx\n", offset
,
312 #define SKD_IRQ_DEFAULT SKD_IRQ_MSIX
313 static int skd_isr_type
= SKD_IRQ_DEFAULT
;
315 module_param(skd_isr_type
, int, 0444);
316 MODULE_PARM_DESC(skd_isr_type
, "Interrupt type capability."
317 " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
319 #define SKD_MAX_REQ_PER_MSG_DEFAULT 1
320 static int skd_max_req_per_msg
= SKD_MAX_REQ_PER_MSG_DEFAULT
;
322 module_param(skd_max_req_per_msg
, int, 0444);
323 MODULE_PARM_DESC(skd_max_req_per_msg
,
324 "Maximum SCSI requests packed in a single message."
325 " (1-" __stringify(SKD_MAX_REQ_PER_MSG
) ", default==1)");
327 #define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
328 #define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
329 static int skd_max_queue_depth
= SKD_MAX_QUEUE_DEPTH_DEFAULT
;
331 module_param(skd_max_queue_depth
, int, 0444);
332 MODULE_PARM_DESC(skd_max_queue_depth
,
333 "Maximum SCSI requests issued to s1120."
334 " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR
")");
336 static int skd_sgs_per_request
= SKD_N_SG_PER_REQ_DEFAULT
;
337 module_param(skd_sgs_per_request
, int, 0444);
338 MODULE_PARM_DESC(skd_sgs_per_request
,
339 "Maximum SG elements per block request."
340 " (1-4096, default==256)");
342 static int skd_max_pass_thru
= 1;
343 module_param(skd_max_pass_thru
, int, 0444);
344 MODULE_PARM_DESC(skd_max_pass_thru
,
345 "Maximum SCSI pass-thru at a time. IGNORED");
347 module_param(skd_dbg_level
, int, 0444);
348 MODULE_PARM_DESC(skd_dbg_level
, "s1120 debug level (0,1,2)");
350 module_param(skd_isr_comp_limit
, int, 0444);
351 MODULE_PARM_DESC(skd_isr_comp_limit
, "s1120 isr comp limit (0=none) default=4");
353 /* Major device number dynamically assigned. */
354 static u32 skd_major
;
356 static void skd_destruct(struct skd_device
*skdev
);
357 static const struct block_device_operations skd_blockdev_ops
;
358 static void skd_send_fitmsg(struct skd_device
*skdev
,
359 struct skd_fitmsg_context
*skmsg
);
360 static void skd_send_special_fitmsg(struct skd_device
*skdev
,
361 struct skd_special_context
*skspcl
);
362 static bool skd_preop_sg_list(struct skd_device
*skdev
,
363 struct skd_request_context
*skreq
);
364 static void skd_postop_sg_list(struct skd_device
*skdev
,
365 struct skd_request_context
*skreq
);
367 static void skd_restart_device(struct skd_device
*skdev
);
368 static int skd_quiesce_dev(struct skd_device
*skdev
);
369 static int skd_unquiesce_dev(struct skd_device
*skdev
);
370 static void skd_disable_interrupts(struct skd_device
*skdev
);
371 static void skd_isr_fwstate(struct skd_device
*skdev
);
372 static void skd_recover_requests(struct skd_device
*skdev
);
373 static void skd_soft_reset(struct skd_device
*skdev
);
375 const char *skd_drive_state_to_str(int state
);
376 const char *skd_skdev_state_to_str(enum skd_drvr_state state
);
377 static void skd_log_skdev(struct skd_device
*skdev
, const char *event
);
378 static void skd_log_skreq(struct skd_device
*skdev
,
379 struct skd_request_context
*skreq
, const char *event
);
382 *****************************************************************************
383 * READ/WRITE REQUESTS
384 *****************************************************************************
386 static void skd_inc_in_flight(struct request
*rq
, void *data
, bool reserved
)
393 static int skd_in_flight(struct skd_device
*skdev
)
397 blk_mq_tagset_busy_iter(&skdev
->tag_set
, skd_inc_in_flight
, &count
);
403 skd_prep_rw_cdb(struct skd_scsi_request
*scsi_req
,
404 int data_dir
, unsigned lba
,
407 if (data_dir
== READ
)
408 scsi_req
->cdb
[0] = READ_10
;
410 scsi_req
->cdb
[0] = WRITE_10
;
412 scsi_req
->cdb
[1] = 0;
413 scsi_req
->cdb
[2] = (lba
& 0xff000000) >> 24;
414 scsi_req
->cdb
[3] = (lba
& 0xff0000) >> 16;
415 scsi_req
->cdb
[4] = (lba
& 0xff00) >> 8;
416 scsi_req
->cdb
[5] = (lba
& 0xff);
417 scsi_req
->cdb
[6] = 0;
418 scsi_req
->cdb
[7] = (count
& 0xff00) >> 8;
419 scsi_req
->cdb
[8] = count
& 0xff;
420 scsi_req
->cdb
[9] = 0;
424 skd_prep_zerosize_flush_cdb(struct skd_scsi_request
*scsi_req
,
425 struct skd_request_context
*skreq
)
427 skreq
->flush_cmd
= 1;
429 scsi_req
->cdb
[0] = SYNCHRONIZE_CACHE
;
430 scsi_req
->cdb
[1] = 0;
431 scsi_req
->cdb
[2] = 0;
432 scsi_req
->cdb
[3] = 0;
433 scsi_req
->cdb
[4] = 0;
434 scsi_req
->cdb
[5] = 0;
435 scsi_req
->cdb
[6] = 0;
436 scsi_req
->cdb
[7] = 0;
437 scsi_req
->cdb
[8] = 0;
438 scsi_req
->cdb
[9] = 0;
442 * Return true if and only if all pending requests should be failed.
444 static bool skd_fail_all(struct request_queue
*q
)
446 struct skd_device
*skdev
= q
->queuedata
;
448 SKD_ASSERT(skdev
->state
!= SKD_DRVR_STATE_ONLINE
);
450 skd_log_skdev(skdev
, "req_not_online");
451 switch (skdev
->state
) {
452 case SKD_DRVR_STATE_PAUSING
:
453 case SKD_DRVR_STATE_PAUSED
:
454 case SKD_DRVR_STATE_STARTING
:
455 case SKD_DRVR_STATE_RESTARTING
:
456 case SKD_DRVR_STATE_WAIT_BOOT
:
457 /* In case of starting, we haven't started the queue,
458 * so we can't get here... but requests are
459 * possibly hanging out waiting for us because we
460 * reported the dev/skd0 already. They'll wait
461 * forever if connect doesn't complete.
462 * What to do??? delay dev/skd0 ??
464 case SKD_DRVR_STATE_BUSY
:
465 case SKD_DRVR_STATE_BUSY_IMMINENT
:
466 case SKD_DRVR_STATE_BUSY_ERASE
:
469 case SKD_DRVR_STATE_BUSY_SANITIZE
:
470 case SKD_DRVR_STATE_STOPPING
:
471 case SKD_DRVR_STATE_SYNCING
:
472 case SKD_DRVR_STATE_FAULT
:
473 case SKD_DRVR_STATE_DISAPPEARED
:
479 static blk_status_t
skd_mq_queue_rq(struct blk_mq_hw_ctx
*hctx
,
480 const struct blk_mq_queue_data
*mqd
)
482 struct request
*const req
= mqd
->rq
;
483 struct request_queue
*const q
= req
->q
;
484 struct skd_device
*skdev
= q
->queuedata
;
485 struct skd_fitmsg_context
*skmsg
;
486 struct fit_msg_hdr
*fmh
;
487 const u32 tag
= blk_mq_unique_tag(req
);
488 struct skd_request_context
*const skreq
= blk_mq_rq_to_pdu(req
);
489 struct skd_scsi_request
*scsi_req
;
490 unsigned long flags
= 0;
491 const u32 lba
= blk_rq_pos(req
);
492 const u32 count
= blk_rq_sectors(req
);
493 const int data_dir
= rq_data_dir(req
);
495 if (unlikely(skdev
->state
!= SKD_DRVR_STATE_ONLINE
))
496 return skd_fail_all(q
) ? BLK_STS_IOERR
: BLK_STS_RESOURCE
;
498 blk_mq_start_request(req
);
500 WARN_ONCE(tag
>= skd_max_queue_depth
, "%#x > %#x (nr_requests = %lu)\n",
501 tag
, skd_max_queue_depth
, q
->nr_requests
);
503 SKD_ASSERT(skreq
->state
== SKD_REQ_STATE_IDLE
);
505 dev_dbg(&skdev
->pdev
->dev
,
506 "new req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n", req
, lba
,
507 lba
, count
, count
, data_dir
);
509 skreq
->id
= tag
+ SKD_ID_RW_REQUEST
;
510 skreq
->flush_cmd
= 0;
512 skreq
->sg_byte_count
= 0;
514 skreq
->fitmsg_id
= 0;
516 skreq
->data_dir
= data_dir
== READ
? DMA_FROM_DEVICE
: DMA_TO_DEVICE
;
518 if (req
->bio
&& !skd_preop_sg_list(skdev
, skreq
)) {
519 dev_dbg(&skdev
->pdev
->dev
, "error Out\n");
520 skreq
->status
= BLK_STS_RESOURCE
;
521 blk_mq_complete_request(req
);
525 dma_sync_single_for_device(&skdev
->pdev
->dev
, skreq
->sksg_dma_address
,
527 sizeof(struct fit_sg_descriptor
),
530 /* Either a FIT msg is in progress or we have to start one. */
531 if (skd_max_req_per_msg
== 1) {
534 spin_lock_irqsave(&skdev
->lock
, flags
);
535 skmsg
= skdev
->skmsg
;
538 skmsg
= &skdev
->skmsg_table
[tag
];
539 skdev
->skmsg
= skmsg
;
541 /* Initialize the FIT msg header */
542 fmh
= &skmsg
->msg_buf
->fmh
;
543 memset(fmh
, 0, sizeof(*fmh
));
544 fmh
->protocol_id
= FIT_PROTOCOL_ID_SOFIT
;
545 skmsg
->length
= sizeof(*fmh
);
547 fmh
= &skmsg
->msg_buf
->fmh
;
550 skreq
->fitmsg_id
= skmsg
->id
;
552 scsi_req
= &skmsg
->msg_buf
->scsi
[fmh
->num_protocol_cmds_coalesced
];
553 memset(scsi_req
, 0, sizeof(*scsi_req
));
555 scsi_req
->hdr
.tag
= skreq
->id
;
556 scsi_req
->hdr
.sg_list_dma_address
=
557 cpu_to_be64(skreq
->sksg_dma_address
);
559 if (req_op(req
) == REQ_OP_FLUSH
) {
560 skd_prep_zerosize_flush_cdb(scsi_req
, skreq
);
561 SKD_ASSERT(skreq
->flush_cmd
== 1);
563 skd_prep_rw_cdb(scsi_req
, data_dir
, lba
, count
);
566 if (req
->cmd_flags
& REQ_FUA
)
567 scsi_req
->cdb
[1] |= SKD_FUA_NV
;
569 scsi_req
->hdr
.sg_list_len_bytes
= cpu_to_be32(skreq
->sg_byte_count
);
571 /* Complete resource allocations. */
572 skreq
->state
= SKD_REQ_STATE_BUSY
;
574 skmsg
->length
+= sizeof(struct skd_scsi_request
);
575 fmh
->num_protocol_cmds_coalesced
++;
577 dev_dbg(&skdev
->pdev
->dev
, "req=0x%x busy=%d\n", skreq
->id
,
578 skd_in_flight(skdev
));
581 * If the FIT msg buffer is full send it.
583 if (skd_max_req_per_msg
== 1) {
584 skd_send_fitmsg(skdev
, skmsg
);
587 fmh
->num_protocol_cmds_coalesced
>= skd_max_req_per_msg
) {
588 skd_send_fitmsg(skdev
, skmsg
);
591 spin_unlock_irqrestore(&skdev
->lock
, flags
);
597 static enum blk_eh_timer_return
skd_timed_out(struct request
*req
,
600 struct skd_device
*skdev
= req
->q
->queuedata
;
602 dev_err(&skdev
->pdev
->dev
, "request with tag %#x timed out\n",
603 blk_mq_unique_tag(req
));
605 return BLK_EH_RESET_TIMER
;
608 static void skd_complete_rq(struct request
*req
)
610 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(req
);
612 blk_mq_end_request(req
, skreq
->status
);
615 static bool skd_preop_sg_list(struct skd_device
*skdev
,
616 struct skd_request_context
*skreq
)
618 struct request
*req
= blk_mq_rq_from_pdu(skreq
);
619 struct scatterlist
*sgl
= &skreq
->sg
[0], *sg
;
623 skreq
->sg_byte_count
= 0;
625 WARN_ON_ONCE(skreq
->data_dir
!= DMA_TO_DEVICE
&&
626 skreq
->data_dir
!= DMA_FROM_DEVICE
);
628 n_sg
= blk_rq_map_sg(skdev
->queue
, req
, sgl
);
633 * Map scatterlist to PCI bus addresses.
634 * Note PCI might change the number of entries.
636 n_sg
= pci_map_sg(skdev
->pdev
, sgl
, n_sg
, skreq
->data_dir
);
640 SKD_ASSERT(n_sg
<= skdev
->sgs_per_request
);
644 for_each_sg(sgl
, sg
, n_sg
, i
) {
645 struct fit_sg_descriptor
*sgd
= &skreq
->sksg_list
[i
];
646 u32 cnt
= sg_dma_len(sg
);
647 uint64_t dma_addr
= sg_dma_address(sg
);
649 sgd
->control
= FIT_SGD_CONTROL_NOT_LAST
;
650 sgd
->byte_count
= cnt
;
651 skreq
->sg_byte_count
+= cnt
;
652 sgd
->host_side_addr
= dma_addr
;
653 sgd
->dev_side_addr
= 0;
656 skreq
->sksg_list
[n_sg
- 1].next_desc_ptr
= 0LL;
657 skreq
->sksg_list
[n_sg
- 1].control
= FIT_SGD_CONTROL_LAST
;
659 if (unlikely(skdev
->dbg_level
> 1)) {
660 dev_dbg(&skdev
->pdev
->dev
,
661 "skreq=%x sksg_list=%p sksg_dma=%llx\n",
662 skreq
->id
, skreq
->sksg_list
, skreq
->sksg_dma_address
);
663 for (i
= 0; i
< n_sg
; i
++) {
664 struct fit_sg_descriptor
*sgd
= &skreq
->sksg_list
[i
];
666 dev_dbg(&skdev
->pdev
->dev
,
667 " sg[%d] count=%u ctrl=0x%x addr=0x%llx next=0x%llx\n",
668 i
, sgd
->byte_count
, sgd
->control
,
669 sgd
->host_side_addr
, sgd
->next_desc_ptr
);
676 static void skd_postop_sg_list(struct skd_device
*skdev
,
677 struct skd_request_context
*skreq
)
680 * restore the next ptr for next IO request so we
681 * don't have to set it every time.
683 skreq
->sksg_list
[skreq
->n_sg
- 1].next_desc_ptr
=
684 skreq
->sksg_dma_address
+
685 ((skreq
->n_sg
) * sizeof(struct fit_sg_descriptor
));
686 pci_unmap_sg(skdev
->pdev
, &skreq
->sg
[0], skreq
->n_sg
, skreq
->data_dir
);
690 *****************************************************************************
692 *****************************************************************************
695 static void skd_timer_tick_not_online(struct skd_device
*skdev
);
697 static void skd_start_queue(struct work_struct
*work
)
699 struct skd_device
*skdev
= container_of(work
, typeof(*skdev
),
703 * Although it is safe to call blk_start_queue() from interrupt
704 * context, blk_mq_start_hw_queues() must not be called from
707 blk_mq_start_hw_queues(skdev
->queue
);
710 static void skd_timer_tick(ulong arg
)
712 struct skd_device
*skdev
= (struct skd_device
*)arg
;
713 unsigned long reqflags
;
716 if (skdev
->state
== SKD_DRVR_STATE_FAULT
)
717 /* The driver has declared fault, and we want it to
718 * stay that way until driver is reloaded.
722 spin_lock_irqsave(&skdev
->lock
, reqflags
);
724 state
= SKD_READL(skdev
, FIT_STATUS
);
725 state
&= FIT_SR_DRIVE_STATE_MASK
;
726 if (state
!= skdev
->drive_state
)
727 skd_isr_fwstate(skdev
);
729 if (skdev
->state
!= SKD_DRVR_STATE_ONLINE
)
730 skd_timer_tick_not_online(skdev
);
732 mod_timer(&skdev
->timer
, (jiffies
+ HZ
));
734 spin_unlock_irqrestore(&skdev
->lock
, reqflags
);
737 static void skd_timer_tick_not_online(struct skd_device
*skdev
)
739 switch (skdev
->state
) {
740 case SKD_DRVR_STATE_IDLE
:
741 case SKD_DRVR_STATE_LOAD
:
743 case SKD_DRVR_STATE_BUSY_SANITIZE
:
744 dev_dbg(&skdev
->pdev
->dev
,
745 "drive busy sanitize[%x], driver[%x]\n",
746 skdev
->drive_state
, skdev
->state
);
747 /* If we've been in sanitize for 3 seconds, we figure we're not
748 * going to get anymore completions, so recover requests now
750 if (skdev
->timer_countdown
> 0) {
751 skdev
->timer_countdown
--;
754 skd_recover_requests(skdev
);
757 case SKD_DRVR_STATE_BUSY
:
758 case SKD_DRVR_STATE_BUSY_IMMINENT
:
759 case SKD_DRVR_STATE_BUSY_ERASE
:
760 dev_dbg(&skdev
->pdev
->dev
, "busy[%x], countdown=%d\n",
761 skdev
->state
, skdev
->timer_countdown
);
762 if (skdev
->timer_countdown
> 0) {
763 skdev
->timer_countdown
--;
766 dev_dbg(&skdev
->pdev
->dev
,
767 "busy[%x], timedout=%d, restarting device.",
768 skdev
->state
, skdev
->timer_countdown
);
769 skd_restart_device(skdev
);
772 case SKD_DRVR_STATE_WAIT_BOOT
:
773 case SKD_DRVR_STATE_STARTING
:
774 if (skdev
->timer_countdown
> 0) {
775 skdev
->timer_countdown
--;
778 /* For now, we fault the drive. Could attempt resets to
779 * revcover at some point. */
780 skdev
->state
= SKD_DRVR_STATE_FAULT
;
782 dev_err(&skdev
->pdev
->dev
, "DriveFault Connect Timeout (%x)\n",
785 /*start the queue so we can respond with error to requests */
786 /* wakeup anyone waiting for startup complete */
787 schedule_work(&skdev
->start_queue
);
788 skdev
->gendisk_on
= -1;
789 wake_up_interruptible(&skdev
->waitq
);
792 case SKD_DRVR_STATE_ONLINE
:
793 /* shouldn't get here. */
796 case SKD_DRVR_STATE_PAUSING
:
797 case SKD_DRVR_STATE_PAUSED
:
800 case SKD_DRVR_STATE_RESTARTING
:
801 if (skdev
->timer_countdown
> 0) {
802 skdev
->timer_countdown
--;
805 /* For now, we fault the drive. Could attempt resets to
806 * revcover at some point. */
807 skdev
->state
= SKD_DRVR_STATE_FAULT
;
808 dev_err(&skdev
->pdev
->dev
,
809 "DriveFault Reconnect Timeout (%x)\n",
813 * Recovering does two things:
814 * 1. completes IO with error
815 * 2. reclaims dma resources
816 * When is it safe to recover requests?
817 * - if the drive state is faulted
818 * - if the state is still soft reset after out timeout
819 * - if the drive registers are dead (state = FF)
820 * If it is "unsafe", we still need to recover, so we will
821 * disable pci bus mastering and disable our interrupts.
824 if ((skdev
->drive_state
== FIT_SR_DRIVE_SOFT_RESET
) ||
825 (skdev
->drive_state
== FIT_SR_DRIVE_FAULT
) ||
826 (skdev
->drive_state
== FIT_SR_DRIVE_STATE_MASK
))
827 /* It never came out of soft reset. Try to
828 * recover the requests and then let them
829 * fail. This is to mitigate hung processes. */
830 skd_recover_requests(skdev
);
832 dev_err(&skdev
->pdev
->dev
, "Disable BusMaster (%x)\n",
834 pci_disable_device(skdev
->pdev
);
835 skd_disable_interrupts(skdev
);
836 skd_recover_requests(skdev
);
839 /*start the queue so we can respond with error to requests */
840 /* wakeup anyone waiting for startup complete */
841 schedule_work(&skdev
->start_queue
);
842 skdev
->gendisk_on
= -1;
843 wake_up_interruptible(&skdev
->waitq
);
846 case SKD_DRVR_STATE_RESUMING
:
847 case SKD_DRVR_STATE_STOPPING
:
848 case SKD_DRVR_STATE_SYNCING
:
849 case SKD_DRVR_STATE_FAULT
:
850 case SKD_DRVR_STATE_DISAPPEARED
:
856 static int skd_start_timer(struct skd_device
*skdev
)
860 setup_timer(&skdev
->timer
, skd_timer_tick
, (ulong
)skdev
);
862 rc
= mod_timer(&skdev
->timer
, (jiffies
+ HZ
));
864 dev_err(&skdev
->pdev
->dev
, "failed to start timer %d\n", rc
);
868 static void skd_kill_timer(struct skd_device
*skdev
)
870 del_timer_sync(&skdev
->timer
);
874 *****************************************************************************
875 * INTERNAL REQUESTS -- generated by driver itself
876 *****************************************************************************
879 static int skd_format_internal_skspcl(struct skd_device
*skdev
)
881 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
882 struct fit_sg_descriptor
*sgd
= &skspcl
->req
.sksg_list
[0];
883 struct fit_msg_hdr
*fmh
;
884 uint64_t dma_address
;
885 struct skd_scsi_request
*scsi
;
887 fmh
= &skspcl
->msg_buf
->fmh
;
888 fmh
->protocol_id
= FIT_PROTOCOL_ID_SOFIT
;
889 fmh
->num_protocol_cmds_coalesced
= 1;
891 scsi
= &skspcl
->msg_buf
->scsi
[0];
892 memset(scsi
, 0, sizeof(*scsi
));
893 dma_address
= skspcl
->req
.sksg_dma_address
;
894 scsi
->hdr
.sg_list_dma_address
= cpu_to_be64(dma_address
);
895 skspcl
->req
.n_sg
= 1;
896 sgd
->control
= FIT_SGD_CONTROL_LAST
;
898 sgd
->host_side_addr
= skspcl
->db_dma_address
;
899 sgd
->dev_side_addr
= 0;
900 sgd
->next_desc_ptr
= 0LL;
905 #define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
907 static void skd_send_internal_skspcl(struct skd_device
*skdev
,
908 struct skd_special_context
*skspcl
,
911 struct fit_sg_descriptor
*sgd
= &skspcl
->req
.sksg_list
[0];
912 struct skd_scsi_request
*scsi
;
913 unsigned char *buf
= skspcl
->data_buf
;
916 if (skspcl
->req
.state
!= SKD_REQ_STATE_IDLE
)
918 * A refresh is already in progress.
919 * Just wait for it to finish.
923 skspcl
->req
.state
= SKD_REQ_STATE_BUSY
;
925 scsi
= &skspcl
->msg_buf
->scsi
[0];
926 scsi
->hdr
.tag
= skspcl
->req
.id
;
928 memset(scsi
->cdb
, 0, sizeof(scsi
->cdb
));
931 case TEST_UNIT_READY
:
932 scsi
->cdb
[0] = TEST_UNIT_READY
;
934 scsi
->hdr
.sg_list_len_bytes
= 0;
938 scsi
->cdb
[0] = READ_CAPACITY
;
939 sgd
->byte_count
= SKD_N_READ_CAP_BYTES
;
940 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
944 scsi
->cdb
[0] = INQUIRY
;
945 scsi
->cdb
[1] = 0x01; /* evpd */
946 scsi
->cdb
[2] = 0x80; /* serial number page */
948 sgd
->byte_count
= 16;
949 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
952 case SYNCHRONIZE_CACHE
:
953 scsi
->cdb
[0] = SYNCHRONIZE_CACHE
;
955 scsi
->hdr
.sg_list_len_bytes
= 0;
959 scsi
->cdb
[0] = WRITE_BUFFER
;
961 scsi
->cdb
[7] = (WR_BUF_SIZE
& 0xFF00) >> 8;
962 scsi
->cdb
[8] = WR_BUF_SIZE
& 0xFF;
963 sgd
->byte_count
= WR_BUF_SIZE
;
964 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
965 /* fill incrementing byte pattern */
966 for (i
= 0; i
< sgd
->byte_count
; i
++)
971 scsi
->cdb
[0] = READ_BUFFER
;
973 scsi
->cdb
[7] = (WR_BUF_SIZE
& 0xFF00) >> 8;
974 scsi
->cdb
[8] = WR_BUF_SIZE
& 0xFF;
975 sgd
->byte_count
= WR_BUF_SIZE
;
976 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
977 memset(skspcl
->data_buf
, 0, sgd
->byte_count
);
981 SKD_ASSERT("Don't know what to send");
985 skd_send_special_fitmsg(skdev
, skspcl
);
988 static void skd_refresh_device_data(struct skd_device
*skdev
)
990 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
992 skd_send_internal_skspcl(skdev
, skspcl
, TEST_UNIT_READY
);
995 static int skd_chk_read_buf(struct skd_device
*skdev
,
996 struct skd_special_context
*skspcl
)
998 unsigned char *buf
= skspcl
->data_buf
;
1001 /* check for incrementing byte pattern */
1002 for (i
= 0; i
< WR_BUF_SIZE
; i
++)
1003 if (buf
[i
] != (i
& 0xFF))
1009 static void skd_log_check_status(struct skd_device
*skdev
, u8 status
, u8 key
,
1010 u8 code
, u8 qual
, u8 fruc
)
1012 /* If the check condition is of special interest, log a message */
1013 if ((status
== SAM_STAT_CHECK_CONDITION
) && (key
== 0x02)
1014 && (code
== 0x04) && (qual
== 0x06)) {
1015 dev_err(&skdev
->pdev
->dev
,
1016 "*** LOST_WRITE_DATA ERROR *** key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
1017 key
, code
, qual
, fruc
);
1021 static void skd_complete_internal(struct skd_device
*skdev
,
1022 struct fit_completion_entry_v1
*skcomp
,
1023 struct fit_comp_error_info
*skerr
,
1024 struct skd_special_context
*skspcl
)
1026 u8
*buf
= skspcl
->data_buf
;
1029 struct skd_scsi_request
*scsi
= &skspcl
->msg_buf
->scsi
[0];
1031 lockdep_assert_held(&skdev
->lock
);
1033 SKD_ASSERT(skspcl
== &skdev
->internal_skspcl
);
1035 dev_dbg(&skdev
->pdev
->dev
, "complete internal %x\n", scsi
->cdb
[0]);
1037 dma_sync_single_for_cpu(&skdev
->pdev
->dev
,
1038 skspcl
->db_dma_address
,
1039 skspcl
->req
.sksg_list
[0].byte_count
,
1042 skspcl
->req
.completion
= *skcomp
;
1043 skspcl
->req
.state
= SKD_REQ_STATE_IDLE
;
1045 status
= skspcl
->req
.completion
.status
;
1047 skd_log_check_status(skdev
, status
, skerr
->key
, skerr
->code
,
1048 skerr
->qual
, skerr
->fruc
);
1050 switch (scsi
->cdb
[0]) {
1051 case TEST_UNIT_READY
:
1052 if (status
== SAM_STAT_GOOD
)
1053 skd_send_internal_skspcl(skdev
, skspcl
, WRITE_BUFFER
);
1054 else if ((status
== SAM_STAT_CHECK_CONDITION
) &&
1055 (skerr
->key
== MEDIUM_ERROR
))
1056 skd_send_internal_skspcl(skdev
, skspcl
, WRITE_BUFFER
);
1058 if (skdev
->state
== SKD_DRVR_STATE_STOPPING
) {
1059 dev_dbg(&skdev
->pdev
->dev
,
1060 "TUR failed, don't send anymore state 0x%x\n",
1064 dev_dbg(&skdev
->pdev
->dev
,
1065 "**** TUR failed, retry skerr\n");
1066 skd_send_internal_skspcl(skdev
, skspcl
,
1072 if (status
== SAM_STAT_GOOD
)
1073 skd_send_internal_skspcl(skdev
, skspcl
, READ_BUFFER
);
1075 if (skdev
->state
== SKD_DRVR_STATE_STOPPING
) {
1076 dev_dbg(&skdev
->pdev
->dev
,
1077 "write buffer failed, don't send anymore state 0x%x\n",
1081 dev_dbg(&skdev
->pdev
->dev
,
1082 "**** write buffer failed, retry skerr\n");
1083 skd_send_internal_skspcl(skdev
, skspcl
,
1089 if (status
== SAM_STAT_GOOD
) {
1090 if (skd_chk_read_buf(skdev
, skspcl
) == 0)
1091 skd_send_internal_skspcl(skdev
, skspcl
,
1094 dev_err(&skdev
->pdev
->dev
,
1095 "*** W/R Buffer mismatch %d ***\n",
1096 skdev
->connect_retries
);
1097 if (skdev
->connect_retries
<
1098 SKD_MAX_CONNECT_RETRIES
) {
1099 skdev
->connect_retries
++;
1100 skd_soft_reset(skdev
);
1102 dev_err(&skdev
->pdev
->dev
,
1103 "W/R Buffer Connect Error\n");
1109 if (skdev
->state
== SKD_DRVR_STATE_STOPPING
) {
1110 dev_dbg(&skdev
->pdev
->dev
,
1111 "read buffer failed, don't send anymore state 0x%x\n",
1115 dev_dbg(&skdev
->pdev
->dev
,
1116 "**** read buffer failed, retry skerr\n");
1117 skd_send_internal_skspcl(skdev
, skspcl
,
1123 skdev
->read_cap_is_valid
= 0;
1124 if (status
== SAM_STAT_GOOD
) {
1125 skdev
->read_cap_last_lba
=
1126 (buf
[0] << 24) | (buf
[1] << 16) |
1127 (buf
[2] << 8) | buf
[3];
1128 skdev
->read_cap_blocksize
=
1129 (buf
[4] << 24) | (buf
[5] << 16) |
1130 (buf
[6] << 8) | buf
[7];
1132 dev_dbg(&skdev
->pdev
->dev
, "last lba %d, bs %d\n",
1133 skdev
->read_cap_last_lba
,
1134 skdev
->read_cap_blocksize
);
1136 set_capacity(skdev
->disk
, skdev
->read_cap_last_lba
+ 1);
1138 skdev
->read_cap_is_valid
= 1;
1140 skd_send_internal_skspcl(skdev
, skspcl
, INQUIRY
);
1141 } else if ((status
== SAM_STAT_CHECK_CONDITION
) &&
1142 (skerr
->key
== MEDIUM_ERROR
)) {
1143 skdev
->read_cap_last_lba
= ~0;
1144 set_capacity(skdev
->disk
, skdev
->read_cap_last_lba
+ 1);
1145 dev_dbg(&skdev
->pdev
->dev
, "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n");
1146 skd_send_internal_skspcl(skdev
, skspcl
, INQUIRY
);
1148 dev_dbg(&skdev
->pdev
->dev
, "**** READCAP failed, retry TUR\n");
1149 skd_send_internal_skspcl(skdev
, skspcl
,
1155 skdev
->inquiry_is_valid
= 0;
1156 if (status
== SAM_STAT_GOOD
) {
1157 skdev
->inquiry_is_valid
= 1;
1159 for (i
= 0; i
< 12; i
++)
1160 skdev
->inq_serial_num
[i
] = buf
[i
+ 4];
1161 skdev
->inq_serial_num
[12] = 0;
1164 if (skd_unquiesce_dev(skdev
) < 0)
1165 dev_dbg(&skdev
->pdev
->dev
, "**** failed, to ONLINE device\n");
1166 /* connection is complete */
1167 skdev
->connect_retries
= 0;
1170 case SYNCHRONIZE_CACHE
:
1171 if (status
== SAM_STAT_GOOD
)
1172 skdev
->sync_done
= 1;
1174 skdev
->sync_done
= -1;
1175 wake_up_interruptible(&skdev
->waitq
);
1179 SKD_ASSERT("we didn't send this");
1184 *****************************************************************************
1186 *****************************************************************************
1189 static void skd_send_fitmsg(struct skd_device
*skdev
,
1190 struct skd_fitmsg_context
*skmsg
)
1194 dev_dbg(&skdev
->pdev
->dev
, "dma address 0x%llx, busy=%d\n",
1195 skmsg
->mb_dma_address
, skd_in_flight(skdev
));
1196 dev_dbg(&skdev
->pdev
->dev
, "msg_buf %p\n", skmsg
->msg_buf
);
1198 qcmd
= skmsg
->mb_dma_address
;
1199 qcmd
|= FIT_QCMD_QID_NORMAL
;
1201 if (unlikely(skdev
->dbg_level
> 1)) {
1202 u8
*bp
= (u8
*)skmsg
->msg_buf
;
1204 for (i
= 0; i
< skmsg
->length
; i
+= 8) {
1205 dev_dbg(&skdev
->pdev
->dev
, "msg[%2d] %8ph\n", i
,
1212 if (skmsg
->length
> 256)
1213 qcmd
|= FIT_QCMD_MSGSIZE_512
;
1214 else if (skmsg
->length
> 128)
1215 qcmd
|= FIT_QCMD_MSGSIZE_256
;
1216 else if (skmsg
->length
> 64)
1217 qcmd
|= FIT_QCMD_MSGSIZE_128
;
1220 * This makes no sense because the FIT msg header is
1221 * 64 bytes. If the msg is only 64 bytes long it has
1224 qcmd
|= FIT_QCMD_MSGSIZE_64
;
1226 dma_sync_single_for_device(&skdev
->pdev
->dev
, skmsg
->mb_dma_address
,
1227 skmsg
->length
, DMA_TO_DEVICE
);
1229 /* Make sure skd_msg_buf is written before the doorbell is triggered. */
1232 SKD_WRITEQ(skdev
, qcmd
, FIT_Q_COMMAND
);
1235 static void skd_send_special_fitmsg(struct skd_device
*skdev
,
1236 struct skd_special_context
*skspcl
)
1240 WARN_ON_ONCE(skspcl
->req
.n_sg
!= 1);
1242 if (unlikely(skdev
->dbg_level
> 1)) {
1243 u8
*bp
= (u8
*)skspcl
->msg_buf
;
1246 for (i
= 0; i
< SKD_N_SPECIAL_FITMSG_BYTES
; i
+= 8) {
1247 dev_dbg(&skdev
->pdev
->dev
, " spcl[%2d] %8ph\n", i
,
1253 dev_dbg(&skdev
->pdev
->dev
,
1254 "skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
1255 skspcl
, skspcl
->req
.id
, skspcl
->req
.sksg_list
,
1256 skspcl
->req
.sksg_dma_address
);
1257 for (i
= 0; i
< skspcl
->req
.n_sg
; i
++) {
1258 struct fit_sg_descriptor
*sgd
=
1259 &skspcl
->req
.sksg_list
[i
];
1261 dev_dbg(&skdev
->pdev
->dev
,
1262 " sg[%d] count=%u ctrl=0x%x addr=0x%llx next=0x%llx\n",
1263 i
, sgd
->byte_count
, sgd
->control
,
1264 sgd
->host_side_addr
, sgd
->next_desc_ptr
);
1269 * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
1270 * and one 64-byte SSDI command.
1272 qcmd
= skspcl
->mb_dma_address
;
1273 qcmd
|= FIT_QCMD_QID_NORMAL
+ FIT_QCMD_MSGSIZE_128
;
1275 dma_sync_single_for_device(&skdev
->pdev
->dev
, skspcl
->mb_dma_address
,
1276 SKD_N_SPECIAL_FITMSG_BYTES
, DMA_TO_DEVICE
);
1277 dma_sync_single_for_device(&skdev
->pdev
->dev
,
1278 skspcl
->req
.sksg_dma_address
,
1279 1 * sizeof(struct fit_sg_descriptor
),
1281 dma_sync_single_for_device(&skdev
->pdev
->dev
,
1282 skspcl
->db_dma_address
,
1283 skspcl
->req
.sksg_list
[0].byte_count
,
1286 /* Make sure skd_msg_buf is written before the doorbell is triggered. */
1289 SKD_WRITEQ(skdev
, qcmd
, FIT_Q_COMMAND
);
1293 *****************************************************************************
1295 *****************************************************************************
1298 static void skd_complete_other(struct skd_device
*skdev
,
1299 struct fit_completion_entry_v1
*skcomp
,
1300 struct fit_comp_error_info
*skerr
);
1309 enum skd_check_status_action action
;
1312 static struct sns_info skd_chkstat_table
[] = {
1314 { 0x70, 0x02, RECOVERED_ERROR
, 0, 0, 0x1c,
1315 SKD_CHECK_STATUS_REPORT_GOOD
},
1318 { 0x70, 0x02, NO_SENSE
, 0x0B, 0x00, 0x1E, /* warnings */
1319 SKD_CHECK_STATUS_REPORT_SMART_ALERT
},
1320 { 0x70, 0x02, NO_SENSE
, 0x5D, 0x00, 0x1E, /* thresholds */
1321 SKD_CHECK_STATUS_REPORT_SMART_ALERT
},
1322 { 0x70, 0x02, RECOVERED_ERROR
, 0x0B, 0x01, 0x1F, /* temperature over trigger */
1323 SKD_CHECK_STATUS_REPORT_SMART_ALERT
},
1325 /* Retry (with limits) */
1326 { 0x70, 0x02, 0x0B, 0, 0, 0x1C, /* This one is for DMA ERROR */
1327 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1328 { 0x70, 0x02, 0x06, 0x0B, 0x00, 0x1E, /* warnings */
1329 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1330 { 0x70, 0x02, 0x06, 0x5D, 0x00, 0x1E, /* thresholds */
1331 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1332 { 0x70, 0x02, 0x06, 0x80, 0x30, 0x1F, /* backup power */
1333 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1335 /* Busy (or about to be) */
1336 { 0x70, 0x02, 0x06, 0x3f, 0x01, 0x1F, /* fw changed */
1337 SKD_CHECK_STATUS_BUSY_IMMINENT
},
1341 * Look up status and sense data to decide how to handle the error
1343 * mask says which fields must match e.g., mask=0x18 means check
1344 * type and stat, ignore key, asc, ascq.
1347 static enum skd_check_status_action
1348 skd_check_status(struct skd_device
*skdev
,
1349 u8 cmp_status
, struct fit_comp_error_info
*skerr
)
1353 dev_err(&skdev
->pdev
->dev
, "key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
1354 skerr
->key
, skerr
->code
, skerr
->qual
, skerr
->fruc
);
1356 dev_dbg(&skdev
->pdev
->dev
,
1357 "stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
1358 skerr
->type
, cmp_status
, skerr
->key
, skerr
->code
, skerr
->qual
,
1361 /* Does the info match an entry in the good category? */
1362 for (i
= 0; i
< ARRAY_SIZE(skd_chkstat_table
); i
++) {
1363 struct sns_info
*sns
= &skd_chkstat_table
[i
];
1365 if (sns
->mask
& 0x10)
1366 if (skerr
->type
!= sns
->type
)
1369 if (sns
->mask
& 0x08)
1370 if (cmp_status
!= sns
->stat
)
1373 if (sns
->mask
& 0x04)
1374 if (skerr
->key
!= sns
->key
)
1377 if (sns
->mask
& 0x02)
1378 if (skerr
->code
!= sns
->asc
)
1381 if (sns
->mask
& 0x01)
1382 if (skerr
->qual
!= sns
->ascq
)
1385 if (sns
->action
== SKD_CHECK_STATUS_REPORT_SMART_ALERT
) {
1386 dev_err(&skdev
->pdev
->dev
,
1387 "SMART Alert: sense key/asc/ascq %02x/%02x/%02x\n",
1388 skerr
->key
, skerr
->code
, skerr
->qual
);
1393 /* No other match, so nonzero status means error,
1394 * zero status means good
1397 dev_dbg(&skdev
->pdev
->dev
, "status check: error\n");
1398 return SKD_CHECK_STATUS_REPORT_ERROR
;
1401 dev_dbg(&skdev
->pdev
->dev
, "status check good default\n");
1402 return SKD_CHECK_STATUS_REPORT_GOOD
;
1405 static void skd_resolve_req_exception(struct skd_device
*skdev
,
1406 struct skd_request_context
*skreq
,
1407 struct request
*req
)
1409 u8 cmp_status
= skreq
->completion
.status
;
1411 switch (skd_check_status(skdev
, cmp_status
, &skreq
->err_info
)) {
1412 case SKD_CHECK_STATUS_REPORT_GOOD
:
1413 case SKD_CHECK_STATUS_REPORT_SMART_ALERT
:
1414 skreq
->status
= BLK_STS_OK
;
1415 blk_mq_complete_request(req
);
1418 case SKD_CHECK_STATUS_BUSY_IMMINENT
:
1419 skd_log_skreq(skdev
, skreq
, "retry(busy)");
1420 blk_requeue_request(skdev
->queue
, req
);
1421 dev_info(&skdev
->pdev
->dev
, "drive BUSY imminent\n");
1422 skdev
->state
= SKD_DRVR_STATE_BUSY_IMMINENT
;
1423 skdev
->timer_countdown
= SKD_TIMER_MINUTES(20);
1424 skd_quiesce_dev(skdev
);
1427 case SKD_CHECK_STATUS_REQUEUE_REQUEST
:
1428 if ((unsigned long) ++req
->special
< SKD_MAX_RETRIES
) {
1429 skd_log_skreq(skdev
, skreq
, "retry");
1430 blk_requeue_request(skdev
->queue
, req
);
1435 case SKD_CHECK_STATUS_REPORT_ERROR
:
1437 skreq
->status
= BLK_STS_IOERR
;
1438 blk_mq_complete_request(req
);
1443 static void skd_release_skreq(struct skd_device
*skdev
,
1444 struct skd_request_context
*skreq
)
1447 * Reclaim the skd_request_context
1449 skreq
->state
= SKD_REQ_STATE_IDLE
;
1452 static int skd_isr_completion_posted(struct skd_device
*skdev
,
1453 int limit
, int *enqueued
)
1455 struct fit_completion_entry_v1
*skcmp
;
1456 struct fit_comp_error_info
*skerr
;
1461 struct skd_request_context
*skreq
;
1469 lockdep_assert_held(&skdev
->lock
);
1472 SKD_ASSERT(skdev
->skcomp_ix
< SKD_N_COMPLETION_ENTRY
);
1474 skcmp
= &skdev
->skcomp_table
[skdev
->skcomp_ix
];
1475 cmp_cycle
= skcmp
->cycle
;
1476 cmp_cntxt
= skcmp
->tag
;
1477 cmp_status
= skcmp
->status
;
1478 cmp_bytes
= be32_to_cpu(skcmp
->num_returned_bytes
);
1480 skerr
= &skdev
->skerr_table
[skdev
->skcomp_ix
];
1482 dev_dbg(&skdev
->pdev
->dev
,
1483 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d busy=%d rbytes=0x%x proto=%d\n",
1484 skdev
->skcomp_cycle
, skdev
->skcomp_ix
, cmp_cycle
,
1485 cmp_cntxt
, cmp_status
, skd_in_flight(skdev
),
1486 cmp_bytes
, skdev
->proto_ver
);
1488 if (cmp_cycle
!= skdev
->skcomp_cycle
) {
1489 dev_dbg(&skdev
->pdev
->dev
, "end of completions\n");
1493 * Update the completion queue head index and possibly
1494 * the completion cycle count. 8-bit wrap-around.
1497 if (skdev
->skcomp_ix
>= SKD_N_COMPLETION_ENTRY
) {
1498 skdev
->skcomp_ix
= 0;
1499 skdev
->skcomp_cycle
++;
1503 * The command context is a unique 32-bit ID. The low order
1504 * bits help locate the request. The request is usually a
1505 * r/w request (see skd_start() above) or a special request.
1508 tag
= req_id
& SKD_ID_SLOT_AND_TABLE_MASK
;
1510 /* Is this other than a r/w request? */
1511 if (tag
>= skdev
->num_req_context
) {
1513 * This is not a completion for a r/w request.
1515 WARN_ON_ONCE(blk_mq_tag_to_rq(skdev
->tag_set
.tags
[hwq
],
1517 skd_complete_other(skdev
, skcmp
, skerr
);
1521 rq
= blk_mq_tag_to_rq(skdev
->tag_set
.tags
[hwq
], tag
);
1522 if (WARN(!rq
, "No request for tag %#x -> %#x\n", cmp_cntxt
,
1525 skreq
= blk_mq_rq_to_pdu(rq
);
1528 * Make sure the request ID for the slot matches.
1530 if (skreq
->id
!= req_id
) {
1531 dev_err(&skdev
->pdev
->dev
,
1532 "Completion mismatch comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
1533 req_id
, skreq
->id
, cmp_cntxt
);
1538 SKD_ASSERT(skreq
->state
== SKD_REQ_STATE_BUSY
);
1540 skreq
->completion
= *skcmp
;
1541 if (unlikely(cmp_status
== SAM_STAT_CHECK_CONDITION
)) {
1542 skreq
->err_info
= *skerr
;
1543 skd_log_check_status(skdev
, cmp_status
, skerr
->key
,
1544 skerr
->code
, skerr
->qual
,
1547 /* Release DMA resources for the request. */
1548 if (skreq
->n_sg
> 0)
1549 skd_postop_sg_list(skdev
, skreq
);
1551 skd_release_skreq(skdev
, skreq
);
1554 * Capture the outcome and post it back to the native request.
1556 if (likely(cmp_status
== SAM_STAT_GOOD
)) {
1557 skreq
->status
= BLK_STS_OK
;
1558 blk_mq_complete_request(rq
);
1560 skd_resolve_req_exception(skdev
, skreq
, rq
);
1563 /* skd_isr_comp_limit equal zero means no limit */
1565 if (++processed
>= limit
) {
1572 if (skdev
->state
== SKD_DRVR_STATE_PAUSING
&&
1573 skd_in_flight(skdev
) == 0) {
1574 skdev
->state
= SKD_DRVR_STATE_PAUSED
;
1575 wake_up_interruptible(&skdev
->waitq
);
1581 static void skd_complete_other(struct skd_device
*skdev
,
1582 struct fit_completion_entry_v1
*skcomp
,
1583 struct fit_comp_error_info
*skerr
)
1588 struct skd_special_context
*skspcl
;
1590 lockdep_assert_held(&skdev
->lock
);
1592 req_id
= skcomp
->tag
;
1593 req_table
= req_id
& SKD_ID_TABLE_MASK
;
1594 req_slot
= req_id
& SKD_ID_SLOT_MASK
;
1596 dev_dbg(&skdev
->pdev
->dev
, "table=0x%x id=0x%x slot=%d\n", req_table
,
1600 * Based on the request id, determine how to dispatch this completion.
1601 * This swich/case is finding the good cases and forwarding the
1602 * completion entry. Errors are reported below the switch.
1604 switch (req_table
) {
1605 case SKD_ID_RW_REQUEST
:
1607 * The caller, skd_isr_completion_posted() above,
1608 * handles r/w requests. The only way we get here
1609 * is if the req_slot is out of bounds.
1613 case SKD_ID_INTERNAL
:
1614 if (req_slot
== 0) {
1615 skspcl
= &skdev
->internal_skspcl
;
1616 if (skspcl
->req
.id
== req_id
&&
1617 skspcl
->req
.state
== SKD_REQ_STATE_BUSY
) {
1618 skd_complete_internal(skdev
,
1619 skcomp
, skerr
, skspcl
);
1625 case SKD_ID_FIT_MSG
:
1627 * These id's should never appear in a completion record.
1633 * These id's should never appear anywhere;
1639 * If we get here it is a bad or stale id.
1643 static void skd_reset_skcomp(struct skd_device
*skdev
)
1645 memset(skdev
->skcomp_table
, 0, SKD_SKCOMP_SIZE
);
1647 skdev
->skcomp_ix
= 0;
1648 skdev
->skcomp_cycle
= 1;
1652 *****************************************************************************
1654 *****************************************************************************
1656 static void skd_completion_worker(struct work_struct
*work
)
1658 struct skd_device
*skdev
=
1659 container_of(work
, struct skd_device
, completion_worker
);
1660 unsigned long flags
;
1661 int flush_enqueued
= 0;
1663 spin_lock_irqsave(&skdev
->lock
, flags
);
1666 * pass in limit=0, which means no limit..
1667 * process everything in compq
1669 skd_isr_completion_posted(skdev
, 0, &flush_enqueued
);
1670 schedule_work(&skdev
->start_queue
);
1672 spin_unlock_irqrestore(&skdev
->lock
, flags
);
1675 static void skd_isr_msg_from_dev(struct skd_device
*skdev
);
1678 skd_isr(int irq
, void *ptr
)
1680 struct skd_device
*skdev
= ptr
;
1685 int flush_enqueued
= 0;
1687 spin_lock(&skdev
->lock
);
1690 intstat
= SKD_READL(skdev
, FIT_INT_STATUS_HOST
);
1692 ack
= FIT_INT_DEF_MASK
;
1695 dev_dbg(&skdev
->pdev
->dev
, "intstat=0x%x ack=0x%x\n", intstat
,
1698 /* As long as there is an int pending on device, keep
1699 * running loop. When none, get out, but if we've never
1700 * done any processing, call completion handler?
1703 /* No interrupts on device, but run the completion
1707 if (likely (skdev
->state
1708 == SKD_DRVR_STATE_ONLINE
))
1715 SKD_WRITEL(skdev
, ack
, FIT_INT_STATUS_HOST
);
1717 if (likely((skdev
->state
!= SKD_DRVR_STATE_LOAD
) &&
1718 (skdev
->state
!= SKD_DRVR_STATE_STOPPING
))) {
1719 if (intstat
& FIT_ISH_COMPLETION_POSTED
) {
1721 * If we have already deferred completion
1722 * processing, don't bother running it again
1726 skd_isr_completion_posted(skdev
,
1727 skd_isr_comp_limit
, &flush_enqueued
);
1730 if (intstat
& FIT_ISH_FW_STATE_CHANGE
) {
1731 skd_isr_fwstate(skdev
);
1732 if (skdev
->state
== SKD_DRVR_STATE_FAULT
||
1734 SKD_DRVR_STATE_DISAPPEARED
) {
1735 spin_unlock(&skdev
->lock
);
1740 if (intstat
& FIT_ISH_MSG_FROM_DEV
)
1741 skd_isr_msg_from_dev(skdev
);
1745 if (unlikely(flush_enqueued
))
1746 schedule_work(&skdev
->start_queue
);
1749 schedule_work(&skdev
->completion_worker
);
1750 else if (!flush_enqueued
)
1751 schedule_work(&skdev
->start_queue
);
1753 spin_unlock(&skdev
->lock
);
1758 static void skd_drive_fault(struct skd_device
*skdev
)
1760 skdev
->state
= SKD_DRVR_STATE_FAULT
;
1761 dev_err(&skdev
->pdev
->dev
, "Drive FAULT\n");
1764 static void skd_drive_disappeared(struct skd_device
*skdev
)
1766 skdev
->state
= SKD_DRVR_STATE_DISAPPEARED
;
1767 dev_err(&skdev
->pdev
->dev
, "Drive DISAPPEARED\n");
1770 static void skd_isr_fwstate(struct skd_device
*skdev
)
1775 int prev_driver_state
= skdev
->state
;
1777 sense
= SKD_READL(skdev
, FIT_STATUS
);
1778 state
= sense
& FIT_SR_DRIVE_STATE_MASK
;
1780 dev_err(&skdev
->pdev
->dev
, "s1120 state %s(%d)=>%s(%d)\n",
1781 skd_drive_state_to_str(skdev
->drive_state
), skdev
->drive_state
,
1782 skd_drive_state_to_str(state
), state
);
1784 skdev
->drive_state
= state
;
1786 switch (skdev
->drive_state
) {
1787 case FIT_SR_DRIVE_INIT
:
1788 if (skdev
->state
== SKD_DRVR_STATE_PROTOCOL_MISMATCH
) {
1789 skd_disable_interrupts(skdev
);
1792 if (skdev
->state
== SKD_DRVR_STATE_RESTARTING
)
1793 skd_recover_requests(skdev
);
1794 if (skdev
->state
== SKD_DRVR_STATE_WAIT_BOOT
) {
1795 skdev
->timer_countdown
= SKD_STARTING_TIMO
;
1796 skdev
->state
= SKD_DRVR_STATE_STARTING
;
1797 skd_soft_reset(skdev
);
1800 mtd
= FIT_MXD_CONS(FIT_MTD_FITFW_INIT
, 0, 0);
1801 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1802 skdev
->last_mtd
= mtd
;
1805 case FIT_SR_DRIVE_ONLINE
:
1806 skdev
->cur_max_queue_depth
= skd_max_queue_depth
;
1807 if (skdev
->cur_max_queue_depth
> skdev
->dev_max_queue_depth
)
1808 skdev
->cur_max_queue_depth
= skdev
->dev_max_queue_depth
;
1810 skdev
->queue_low_water_mark
=
1811 skdev
->cur_max_queue_depth
* 2 / 3 + 1;
1812 if (skdev
->queue_low_water_mark
< 1)
1813 skdev
->queue_low_water_mark
= 1;
1814 dev_info(&skdev
->pdev
->dev
,
1815 "Queue depth limit=%d dev=%d lowat=%d\n",
1816 skdev
->cur_max_queue_depth
,
1817 skdev
->dev_max_queue_depth
,
1818 skdev
->queue_low_water_mark
);
1820 skd_refresh_device_data(skdev
);
1823 case FIT_SR_DRIVE_BUSY
:
1824 skdev
->state
= SKD_DRVR_STATE_BUSY
;
1825 skdev
->timer_countdown
= SKD_BUSY_TIMO
;
1826 skd_quiesce_dev(skdev
);
1828 case FIT_SR_DRIVE_BUSY_SANITIZE
:
1829 /* set timer for 3 seconds, we'll abort any unfinished
1830 * commands after that expires
1832 skdev
->state
= SKD_DRVR_STATE_BUSY_SANITIZE
;
1833 skdev
->timer_countdown
= SKD_TIMER_SECONDS(3);
1834 schedule_work(&skdev
->start_queue
);
1836 case FIT_SR_DRIVE_BUSY_ERASE
:
1837 skdev
->state
= SKD_DRVR_STATE_BUSY_ERASE
;
1838 skdev
->timer_countdown
= SKD_BUSY_TIMO
;
1840 case FIT_SR_DRIVE_OFFLINE
:
1841 skdev
->state
= SKD_DRVR_STATE_IDLE
;
1843 case FIT_SR_DRIVE_SOFT_RESET
:
1844 switch (skdev
->state
) {
1845 case SKD_DRVR_STATE_STARTING
:
1846 case SKD_DRVR_STATE_RESTARTING
:
1847 /* Expected by a caller of skd_soft_reset() */
1850 skdev
->state
= SKD_DRVR_STATE_RESTARTING
;
1854 case FIT_SR_DRIVE_FW_BOOTING
:
1855 dev_dbg(&skdev
->pdev
->dev
, "ISR FIT_SR_DRIVE_FW_BOOTING\n");
1856 skdev
->state
= SKD_DRVR_STATE_WAIT_BOOT
;
1857 skdev
->timer_countdown
= SKD_WAIT_BOOT_TIMO
;
1860 case FIT_SR_DRIVE_DEGRADED
:
1861 case FIT_SR_PCIE_LINK_DOWN
:
1862 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD
:
1865 case FIT_SR_DRIVE_FAULT
:
1866 skd_drive_fault(skdev
);
1867 skd_recover_requests(skdev
);
1868 schedule_work(&skdev
->start_queue
);
1871 /* PCIe bus returned all Fs? */
1873 dev_info(&skdev
->pdev
->dev
, "state=0x%x sense=0x%x\n", state
,
1875 skd_drive_disappeared(skdev
);
1876 skd_recover_requests(skdev
);
1877 schedule_work(&skdev
->start_queue
);
1881 * Uknown FW State. Wait for a state we recognize.
1885 dev_err(&skdev
->pdev
->dev
, "Driver state %s(%d)=>%s(%d)\n",
1886 skd_skdev_state_to_str(prev_driver_state
), prev_driver_state
,
1887 skd_skdev_state_to_str(skdev
->state
), skdev
->state
);
1890 static void skd_recover_request(struct request
*req
, void *data
, bool reserved
)
1892 struct skd_device
*const skdev
= data
;
1893 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(req
);
1895 if (skreq
->state
!= SKD_REQ_STATE_BUSY
)
1898 skd_log_skreq(skdev
, skreq
, "recover");
1900 /* Release DMA resources for the request. */
1901 if (skreq
->n_sg
> 0)
1902 skd_postop_sg_list(skdev
, skreq
);
1904 skreq
->state
= SKD_REQ_STATE_IDLE
;
1905 skreq
->status
= BLK_STS_IOERR
;
1906 blk_mq_complete_request(req
);
1909 static void skd_recover_requests(struct skd_device
*skdev
)
1911 blk_mq_tagset_busy_iter(&skdev
->tag_set
, skd_recover_request
, skdev
);
1914 static void skd_isr_msg_from_dev(struct skd_device
*skdev
)
1920 mfd
= SKD_READL(skdev
, FIT_MSG_FROM_DEVICE
);
1922 dev_dbg(&skdev
->pdev
->dev
, "mfd=0x%x last_mtd=0x%x\n", mfd
,
1925 /* ignore any mtd that is an ack for something we didn't send */
1926 if (FIT_MXD_TYPE(mfd
) != FIT_MXD_TYPE(skdev
->last_mtd
))
1929 switch (FIT_MXD_TYPE(mfd
)) {
1930 case FIT_MTD_FITFW_INIT
:
1931 skdev
->proto_ver
= FIT_PROTOCOL_MAJOR_VER(mfd
);
1933 if (skdev
->proto_ver
!= FIT_PROTOCOL_VERSION_1
) {
1934 dev_err(&skdev
->pdev
->dev
, "protocol mismatch\n");
1935 dev_err(&skdev
->pdev
->dev
, " got=%d support=%d\n",
1936 skdev
->proto_ver
, FIT_PROTOCOL_VERSION_1
);
1937 dev_err(&skdev
->pdev
->dev
, " please upgrade driver\n");
1938 skdev
->state
= SKD_DRVR_STATE_PROTOCOL_MISMATCH
;
1939 skd_soft_reset(skdev
);
1942 mtd
= FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH
, 0, 0);
1943 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1944 skdev
->last_mtd
= mtd
;
1947 case FIT_MTD_GET_CMDQ_DEPTH
:
1948 skdev
->dev_max_queue_depth
= FIT_MXD_DATA(mfd
);
1949 mtd
= FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH
, 0,
1950 SKD_N_COMPLETION_ENTRY
);
1951 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1952 skdev
->last_mtd
= mtd
;
1955 case FIT_MTD_SET_COMPQ_DEPTH
:
1956 SKD_WRITEQ(skdev
, skdev
->cq_dma_address
, FIT_MSG_TO_DEVICE_ARG
);
1957 mtd
= FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR
, 0, 0);
1958 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1959 skdev
->last_mtd
= mtd
;
1962 case FIT_MTD_SET_COMPQ_ADDR
:
1963 skd_reset_skcomp(skdev
);
1964 mtd
= FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID
, 0, skdev
->devno
);
1965 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1966 skdev
->last_mtd
= mtd
;
1969 case FIT_MTD_CMD_LOG_HOST_ID
:
1970 skdev
->connect_time_stamp
= get_seconds();
1971 data
= skdev
->connect_time_stamp
& 0xFFFF;
1972 mtd
= FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO
, 0, data
);
1973 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1974 skdev
->last_mtd
= mtd
;
1977 case FIT_MTD_CMD_LOG_TIME_STAMP_LO
:
1978 skdev
->drive_jiffies
= FIT_MXD_DATA(mfd
);
1979 data
= (skdev
->connect_time_stamp
>> 16) & 0xFFFF;
1980 mtd
= FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI
, 0, data
);
1981 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1982 skdev
->last_mtd
= mtd
;
1985 case FIT_MTD_CMD_LOG_TIME_STAMP_HI
:
1986 skdev
->drive_jiffies
|= (FIT_MXD_DATA(mfd
) << 16);
1987 mtd
= FIT_MXD_CONS(FIT_MTD_ARM_QUEUE
, 0, 0);
1988 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1989 skdev
->last_mtd
= mtd
;
1991 dev_err(&skdev
->pdev
->dev
, "Time sync driver=0x%x device=0x%x\n",
1992 skdev
->connect_time_stamp
, skdev
->drive_jiffies
);
1995 case FIT_MTD_ARM_QUEUE
:
1996 skdev
->last_mtd
= 0;
1998 * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
2007 static void skd_disable_interrupts(struct skd_device
*skdev
)
2011 sense
= SKD_READL(skdev
, FIT_CONTROL
);
2012 sense
&= ~FIT_CR_ENABLE_INTERRUPTS
;
2013 SKD_WRITEL(skdev
, sense
, FIT_CONTROL
);
2014 dev_dbg(&skdev
->pdev
->dev
, "sense 0x%x\n", sense
);
2016 /* Note that the 1s is written. A 1-bit means
2017 * disable, a 0 means enable.
2019 SKD_WRITEL(skdev
, ~0, FIT_INT_MASK_HOST
);
2022 static void skd_enable_interrupts(struct skd_device
*skdev
)
2026 /* unmask interrupts first */
2027 val
= FIT_ISH_FW_STATE_CHANGE
+
2028 FIT_ISH_COMPLETION_POSTED
+ FIT_ISH_MSG_FROM_DEV
;
2030 /* Note that the compliment of mask is written. A 1-bit means
2031 * disable, a 0 means enable. */
2032 SKD_WRITEL(skdev
, ~val
, FIT_INT_MASK_HOST
);
2033 dev_dbg(&skdev
->pdev
->dev
, "interrupt mask=0x%x\n", ~val
);
2035 val
= SKD_READL(skdev
, FIT_CONTROL
);
2036 val
|= FIT_CR_ENABLE_INTERRUPTS
;
2037 dev_dbg(&skdev
->pdev
->dev
, "control=0x%x\n", val
);
2038 SKD_WRITEL(skdev
, val
, FIT_CONTROL
);
2042 *****************************************************************************
2043 * START, STOP, RESTART, QUIESCE, UNQUIESCE
2044 *****************************************************************************
2047 static void skd_soft_reset(struct skd_device
*skdev
)
2051 val
= SKD_READL(skdev
, FIT_CONTROL
);
2052 val
|= (FIT_CR_SOFT_RESET
);
2053 dev_dbg(&skdev
->pdev
->dev
, "control=0x%x\n", val
);
2054 SKD_WRITEL(skdev
, val
, FIT_CONTROL
);
2057 static void skd_start_device(struct skd_device
*skdev
)
2059 unsigned long flags
;
2063 spin_lock_irqsave(&skdev
->lock
, flags
);
2065 /* ack all ghost interrupts */
2066 SKD_WRITEL(skdev
, FIT_INT_DEF_MASK
, FIT_INT_STATUS_HOST
);
2068 sense
= SKD_READL(skdev
, FIT_STATUS
);
2070 dev_dbg(&skdev
->pdev
->dev
, "initial status=0x%x\n", sense
);
2072 state
= sense
& FIT_SR_DRIVE_STATE_MASK
;
2073 skdev
->drive_state
= state
;
2074 skdev
->last_mtd
= 0;
2076 skdev
->state
= SKD_DRVR_STATE_STARTING
;
2077 skdev
->timer_countdown
= SKD_STARTING_TIMO
;
2079 skd_enable_interrupts(skdev
);
2081 switch (skdev
->drive_state
) {
2082 case FIT_SR_DRIVE_OFFLINE
:
2083 dev_err(&skdev
->pdev
->dev
, "Drive offline...\n");
2086 case FIT_SR_DRIVE_FW_BOOTING
:
2087 dev_dbg(&skdev
->pdev
->dev
, "FIT_SR_DRIVE_FW_BOOTING\n");
2088 skdev
->state
= SKD_DRVR_STATE_WAIT_BOOT
;
2089 skdev
->timer_countdown
= SKD_WAIT_BOOT_TIMO
;
2092 case FIT_SR_DRIVE_BUSY_SANITIZE
:
2093 dev_info(&skdev
->pdev
->dev
, "Start: BUSY_SANITIZE\n");
2094 skdev
->state
= SKD_DRVR_STATE_BUSY_SANITIZE
;
2095 skdev
->timer_countdown
= SKD_STARTED_BUSY_TIMO
;
2098 case FIT_SR_DRIVE_BUSY_ERASE
:
2099 dev_info(&skdev
->pdev
->dev
, "Start: BUSY_ERASE\n");
2100 skdev
->state
= SKD_DRVR_STATE_BUSY_ERASE
;
2101 skdev
->timer_countdown
= SKD_STARTED_BUSY_TIMO
;
2104 case FIT_SR_DRIVE_INIT
:
2105 case FIT_SR_DRIVE_ONLINE
:
2106 skd_soft_reset(skdev
);
2109 case FIT_SR_DRIVE_BUSY
:
2110 dev_err(&skdev
->pdev
->dev
, "Drive Busy...\n");
2111 skdev
->state
= SKD_DRVR_STATE_BUSY
;
2112 skdev
->timer_countdown
= SKD_STARTED_BUSY_TIMO
;
2115 case FIT_SR_DRIVE_SOFT_RESET
:
2116 dev_err(&skdev
->pdev
->dev
, "drive soft reset in prog\n");
2119 case FIT_SR_DRIVE_FAULT
:
2120 /* Fault state is bad...soft reset won't do it...
2121 * Hard reset, maybe, but does it work on device?
2122 * For now, just fault so the system doesn't hang.
2124 skd_drive_fault(skdev
);
2125 /*start the queue so we can respond with error to requests */
2126 dev_dbg(&skdev
->pdev
->dev
, "starting queue\n");
2127 schedule_work(&skdev
->start_queue
);
2128 skdev
->gendisk_on
= -1;
2129 wake_up_interruptible(&skdev
->waitq
);
2133 /* Most likely the device isn't there or isn't responding
2134 * to the BAR1 addresses. */
2135 skd_drive_disappeared(skdev
);
2136 /*start the queue so we can respond with error to requests */
2137 dev_dbg(&skdev
->pdev
->dev
,
2138 "starting queue to error-out reqs\n");
2139 schedule_work(&skdev
->start_queue
);
2140 skdev
->gendisk_on
= -1;
2141 wake_up_interruptible(&skdev
->waitq
);
2145 dev_err(&skdev
->pdev
->dev
, "Start: unknown state %x\n",
2146 skdev
->drive_state
);
2150 state
= SKD_READL(skdev
, FIT_CONTROL
);
2151 dev_dbg(&skdev
->pdev
->dev
, "FIT Control Status=0x%x\n", state
);
2153 state
= SKD_READL(skdev
, FIT_INT_STATUS_HOST
);
2154 dev_dbg(&skdev
->pdev
->dev
, "Intr Status=0x%x\n", state
);
2156 state
= SKD_READL(skdev
, FIT_INT_MASK_HOST
);
2157 dev_dbg(&skdev
->pdev
->dev
, "Intr Mask=0x%x\n", state
);
2159 state
= SKD_READL(skdev
, FIT_MSG_FROM_DEVICE
);
2160 dev_dbg(&skdev
->pdev
->dev
, "Msg from Dev=0x%x\n", state
);
2162 state
= SKD_READL(skdev
, FIT_HW_VERSION
);
2163 dev_dbg(&skdev
->pdev
->dev
, "HW version=0x%x\n", state
);
2165 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2168 static void skd_stop_device(struct skd_device
*skdev
)
2170 unsigned long flags
;
2171 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
2175 spin_lock_irqsave(&skdev
->lock
, flags
);
2177 if (skdev
->state
!= SKD_DRVR_STATE_ONLINE
) {
2178 dev_err(&skdev
->pdev
->dev
, "%s not online no sync\n", __func__
);
2182 if (skspcl
->req
.state
!= SKD_REQ_STATE_IDLE
) {
2183 dev_err(&skdev
->pdev
->dev
, "%s no special\n", __func__
);
2187 skdev
->state
= SKD_DRVR_STATE_SYNCING
;
2188 skdev
->sync_done
= 0;
2190 skd_send_internal_skspcl(skdev
, skspcl
, SYNCHRONIZE_CACHE
);
2192 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2194 wait_event_interruptible_timeout(skdev
->waitq
,
2195 (skdev
->sync_done
), (10 * HZ
));
2197 spin_lock_irqsave(&skdev
->lock
, flags
);
2199 switch (skdev
->sync_done
) {
2201 dev_err(&skdev
->pdev
->dev
, "%s no sync\n", __func__
);
2204 dev_err(&skdev
->pdev
->dev
, "%s sync done\n", __func__
);
2207 dev_err(&skdev
->pdev
->dev
, "%s sync error\n", __func__
);
2211 skdev
->state
= SKD_DRVR_STATE_STOPPING
;
2212 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2214 skd_kill_timer(skdev
);
2216 spin_lock_irqsave(&skdev
->lock
, flags
);
2217 skd_disable_interrupts(skdev
);
2219 /* ensure all ints on device are cleared */
2220 /* soft reset the device to unload with a clean slate */
2221 SKD_WRITEL(skdev
, FIT_INT_DEF_MASK
, FIT_INT_STATUS_HOST
);
2222 SKD_WRITEL(skdev
, FIT_CR_SOFT_RESET
, FIT_CONTROL
);
2224 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2226 /* poll every 100ms, 1 second timeout */
2227 for (i
= 0; i
< 10; i
++) {
2229 SKD_READL(skdev
, FIT_STATUS
) & FIT_SR_DRIVE_STATE_MASK
;
2230 if (dev_state
== FIT_SR_DRIVE_INIT
)
2232 set_current_state(TASK_INTERRUPTIBLE
);
2233 schedule_timeout(msecs_to_jiffies(100));
2236 if (dev_state
!= FIT_SR_DRIVE_INIT
)
2237 dev_err(&skdev
->pdev
->dev
, "%s state error 0x%02x\n", __func__
,
2241 /* assume spinlock is held */
2242 static void skd_restart_device(struct skd_device
*skdev
)
2246 /* ack all ghost interrupts */
2247 SKD_WRITEL(skdev
, FIT_INT_DEF_MASK
, FIT_INT_STATUS_HOST
);
2249 state
= SKD_READL(skdev
, FIT_STATUS
);
2251 dev_dbg(&skdev
->pdev
->dev
, "drive status=0x%x\n", state
);
2253 state
&= FIT_SR_DRIVE_STATE_MASK
;
2254 skdev
->drive_state
= state
;
2255 skdev
->last_mtd
= 0;
2257 skdev
->state
= SKD_DRVR_STATE_RESTARTING
;
2258 skdev
->timer_countdown
= SKD_RESTARTING_TIMO
;
2260 skd_soft_reset(skdev
);
2263 /* assume spinlock is held */
2264 static int skd_quiesce_dev(struct skd_device
*skdev
)
2268 switch (skdev
->state
) {
2269 case SKD_DRVR_STATE_BUSY
:
2270 case SKD_DRVR_STATE_BUSY_IMMINENT
:
2271 dev_dbg(&skdev
->pdev
->dev
, "stopping queue\n");
2272 blk_mq_stop_hw_queues(skdev
->queue
);
2274 case SKD_DRVR_STATE_ONLINE
:
2275 case SKD_DRVR_STATE_STOPPING
:
2276 case SKD_DRVR_STATE_SYNCING
:
2277 case SKD_DRVR_STATE_PAUSING
:
2278 case SKD_DRVR_STATE_PAUSED
:
2279 case SKD_DRVR_STATE_STARTING
:
2280 case SKD_DRVR_STATE_RESTARTING
:
2281 case SKD_DRVR_STATE_RESUMING
:
2284 dev_dbg(&skdev
->pdev
->dev
, "state [%d] not implemented\n",
2290 /* assume spinlock is held */
2291 static int skd_unquiesce_dev(struct skd_device
*skdev
)
2293 int prev_driver_state
= skdev
->state
;
2295 skd_log_skdev(skdev
, "unquiesce");
2296 if (skdev
->state
== SKD_DRVR_STATE_ONLINE
) {
2297 dev_dbg(&skdev
->pdev
->dev
, "**** device already ONLINE\n");
2300 if (skdev
->drive_state
!= FIT_SR_DRIVE_ONLINE
) {
2302 * If there has been an state change to other than
2303 * ONLINE, we will rely on controller state change
2304 * to come back online and restart the queue.
2305 * The BUSY state means that driver is ready to
2306 * continue normal processing but waiting for controller
2307 * to become available.
2309 skdev
->state
= SKD_DRVR_STATE_BUSY
;
2310 dev_dbg(&skdev
->pdev
->dev
, "drive BUSY state\n");
2315 * Drive has just come online, driver is either in startup,
2316 * paused performing a task, or bust waiting for hardware.
2318 switch (skdev
->state
) {
2319 case SKD_DRVR_STATE_PAUSED
:
2320 case SKD_DRVR_STATE_BUSY
:
2321 case SKD_DRVR_STATE_BUSY_IMMINENT
:
2322 case SKD_DRVR_STATE_BUSY_ERASE
:
2323 case SKD_DRVR_STATE_STARTING
:
2324 case SKD_DRVR_STATE_RESTARTING
:
2325 case SKD_DRVR_STATE_FAULT
:
2326 case SKD_DRVR_STATE_IDLE
:
2327 case SKD_DRVR_STATE_LOAD
:
2328 skdev
->state
= SKD_DRVR_STATE_ONLINE
;
2329 dev_err(&skdev
->pdev
->dev
, "Driver state %s(%d)=>%s(%d)\n",
2330 skd_skdev_state_to_str(prev_driver_state
),
2331 prev_driver_state
, skd_skdev_state_to_str(skdev
->state
),
2333 dev_dbg(&skdev
->pdev
->dev
,
2334 "**** device ONLINE...starting block queue\n");
2335 dev_dbg(&skdev
->pdev
->dev
, "starting queue\n");
2336 dev_info(&skdev
->pdev
->dev
, "STEC s1120 ONLINE\n");
2337 schedule_work(&skdev
->start_queue
);
2338 skdev
->gendisk_on
= 1;
2339 wake_up_interruptible(&skdev
->waitq
);
2342 case SKD_DRVR_STATE_DISAPPEARED
:
2344 dev_dbg(&skdev
->pdev
->dev
,
2345 "**** driver state %d, not implemented\n",
2353 *****************************************************************************
2354 * PCIe MSI/MSI-X INTERRUPT HANDLERS
2355 *****************************************************************************
2358 static irqreturn_t
skd_reserved_isr(int irq
, void *skd_host_data
)
2360 struct skd_device
*skdev
= skd_host_data
;
2361 unsigned long flags
;
2363 spin_lock_irqsave(&skdev
->lock
, flags
);
2364 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2365 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2366 dev_err(&skdev
->pdev
->dev
, "MSIX reserved irq %d = 0x%x\n", irq
,
2367 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2368 SKD_WRITEL(skdev
, FIT_INT_RESERVED_MASK
, FIT_INT_STATUS_HOST
);
2369 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2373 static irqreturn_t
skd_statec_isr(int irq
, void *skd_host_data
)
2375 struct skd_device
*skdev
= skd_host_data
;
2376 unsigned long flags
;
2378 spin_lock_irqsave(&skdev
->lock
, flags
);
2379 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2380 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2381 SKD_WRITEL(skdev
, FIT_ISH_FW_STATE_CHANGE
, FIT_INT_STATUS_HOST
);
2382 skd_isr_fwstate(skdev
);
2383 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2387 static irqreturn_t
skd_comp_q(int irq
, void *skd_host_data
)
2389 struct skd_device
*skdev
= skd_host_data
;
2390 unsigned long flags
;
2391 int flush_enqueued
= 0;
2394 spin_lock_irqsave(&skdev
->lock
, flags
);
2395 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2396 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2397 SKD_WRITEL(skdev
, FIT_ISH_COMPLETION_POSTED
, FIT_INT_STATUS_HOST
);
2398 deferred
= skd_isr_completion_posted(skdev
, skd_isr_comp_limit
,
2401 schedule_work(&skdev
->start_queue
);
2404 schedule_work(&skdev
->completion_worker
);
2405 else if (!flush_enqueued
)
2406 schedule_work(&skdev
->start_queue
);
2408 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2413 static irqreturn_t
skd_msg_isr(int irq
, void *skd_host_data
)
2415 struct skd_device
*skdev
= skd_host_data
;
2416 unsigned long flags
;
2418 spin_lock_irqsave(&skdev
->lock
, flags
);
2419 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2420 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2421 SKD_WRITEL(skdev
, FIT_ISH_MSG_FROM_DEV
, FIT_INT_STATUS_HOST
);
2422 skd_isr_msg_from_dev(skdev
);
2423 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2427 static irqreturn_t
skd_qfull_isr(int irq
, void *skd_host_data
)
2429 struct skd_device
*skdev
= skd_host_data
;
2430 unsigned long flags
;
2432 spin_lock_irqsave(&skdev
->lock
, flags
);
2433 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2434 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2435 SKD_WRITEL(skdev
, FIT_INT_QUEUE_FULL
, FIT_INT_STATUS_HOST
);
2436 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2441 *****************************************************************************
2442 * PCIe MSI/MSI-X SETUP
2443 *****************************************************************************
2446 struct skd_msix_entry
{
2450 struct skd_init_msix_entry
{
2452 irq_handler_t handler
;
2455 #define SKD_MAX_MSIX_COUNT 13
2456 #define SKD_MIN_MSIX_COUNT 7
2457 #define SKD_BASE_MSIX_IRQ 4
2459 static struct skd_init_msix_entry msix_entries
[SKD_MAX_MSIX_COUNT
] = {
2460 { "(DMA 0)", skd_reserved_isr
},
2461 { "(DMA 1)", skd_reserved_isr
},
2462 { "(DMA 2)", skd_reserved_isr
},
2463 { "(DMA 3)", skd_reserved_isr
},
2464 { "(State Change)", skd_statec_isr
},
2465 { "(COMPL_Q)", skd_comp_q
},
2466 { "(MSG)", skd_msg_isr
},
2467 { "(Reserved)", skd_reserved_isr
},
2468 { "(Reserved)", skd_reserved_isr
},
2469 { "(Queue Full 0)", skd_qfull_isr
},
2470 { "(Queue Full 1)", skd_qfull_isr
},
2471 { "(Queue Full 2)", skd_qfull_isr
},
2472 { "(Queue Full 3)", skd_qfull_isr
},
2475 static int skd_acquire_msix(struct skd_device
*skdev
)
2478 struct pci_dev
*pdev
= skdev
->pdev
;
2480 rc
= pci_alloc_irq_vectors(pdev
, SKD_MAX_MSIX_COUNT
, SKD_MAX_MSIX_COUNT
,
2483 dev_err(&skdev
->pdev
->dev
, "failed to enable MSI-X %d\n", rc
);
2487 skdev
->msix_entries
= kcalloc(SKD_MAX_MSIX_COUNT
,
2488 sizeof(struct skd_msix_entry
), GFP_KERNEL
);
2489 if (!skdev
->msix_entries
) {
2491 dev_err(&skdev
->pdev
->dev
, "msix table allocation error\n");
2495 /* Enable MSI-X vectors for the base queue */
2496 for (i
= 0; i
< SKD_MAX_MSIX_COUNT
; i
++) {
2497 struct skd_msix_entry
*qentry
= &skdev
->msix_entries
[i
];
2499 snprintf(qentry
->isr_name
, sizeof(qentry
->isr_name
),
2500 "%s%d-msix %s", DRV_NAME
, skdev
->devno
,
2501 msix_entries
[i
].name
);
2503 rc
= devm_request_irq(&skdev
->pdev
->dev
,
2504 pci_irq_vector(skdev
->pdev
, i
),
2505 msix_entries
[i
].handler
, 0,
2506 qentry
->isr_name
, skdev
);
2508 dev_err(&skdev
->pdev
->dev
,
2509 "Unable to register(%d) MSI-X handler %d: %s\n",
2510 rc
, i
, qentry
->isr_name
);
2515 dev_dbg(&skdev
->pdev
->dev
, "%d msix irq(s) enabled\n",
2516 SKD_MAX_MSIX_COUNT
);
2521 devm_free_irq(&pdev
->dev
, pci_irq_vector(pdev
, i
), skdev
);
2523 kfree(skdev
->msix_entries
);
2524 skdev
->msix_entries
= NULL
;
2528 static int skd_acquire_irq(struct skd_device
*skdev
)
2530 struct pci_dev
*pdev
= skdev
->pdev
;
2531 unsigned int irq_flag
= PCI_IRQ_LEGACY
;
2534 if (skd_isr_type
== SKD_IRQ_MSIX
) {
2535 rc
= skd_acquire_msix(skdev
);
2539 dev_err(&skdev
->pdev
->dev
,
2540 "failed to enable MSI-X, re-trying with MSI %d\n", rc
);
2543 snprintf(skdev
->isr_name
, sizeof(skdev
->isr_name
), "%s%d", DRV_NAME
,
2546 if (skd_isr_type
!= SKD_IRQ_LEGACY
)
2547 irq_flag
|= PCI_IRQ_MSI
;
2548 rc
= pci_alloc_irq_vectors(pdev
, 1, 1, irq_flag
);
2550 dev_err(&skdev
->pdev
->dev
,
2551 "failed to allocate the MSI interrupt %d\n", rc
);
2555 rc
= devm_request_irq(&pdev
->dev
, pdev
->irq
, skd_isr
,
2556 pdev
->msi_enabled
? 0 : IRQF_SHARED
,
2557 skdev
->isr_name
, skdev
);
2559 pci_free_irq_vectors(pdev
);
2560 dev_err(&skdev
->pdev
->dev
, "failed to allocate interrupt %d\n",
2568 static void skd_release_irq(struct skd_device
*skdev
)
2570 struct pci_dev
*pdev
= skdev
->pdev
;
2572 if (skdev
->msix_entries
) {
2575 for (i
= 0; i
< SKD_MAX_MSIX_COUNT
; i
++) {
2576 devm_free_irq(&pdev
->dev
, pci_irq_vector(pdev
, i
),
2580 kfree(skdev
->msix_entries
);
2581 skdev
->msix_entries
= NULL
;
2583 devm_free_irq(&pdev
->dev
, pdev
->irq
, skdev
);
2586 pci_free_irq_vectors(pdev
);
2590 *****************************************************************************
2592 *****************************************************************************
2595 static void *skd_alloc_dma(struct skd_device
*skdev
, struct kmem_cache
*s
,
2596 dma_addr_t
*dma_handle
, gfp_t gfp
,
2597 enum dma_data_direction dir
)
2599 struct device
*dev
= &skdev
->pdev
->dev
;
2602 buf
= kmem_cache_alloc(s
, gfp
);
2605 *dma_handle
= dma_map_single(dev
, buf
, s
->size
, dir
);
2606 if (dma_mapping_error(dev
, *dma_handle
)) {
2613 static void skd_free_dma(struct skd_device
*skdev
, struct kmem_cache
*s
,
2614 void *vaddr
, dma_addr_t dma_handle
,
2615 enum dma_data_direction dir
)
2620 dma_unmap_single(&skdev
->pdev
->dev
, dma_handle
, s
->size
, dir
);
2621 kmem_cache_free(s
, vaddr
);
2624 static int skd_cons_skcomp(struct skd_device
*skdev
)
2627 struct fit_completion_entry_v1
*skcomp
;
2629 dev_dbg(&skdev
->pdev
->dev
,
2630 "comp pci_alloc, total bytes %zd entries %d\n",
2631 SKD_SKCOMP_SIZE
, SKD_N_COMPLETION_ENTRY
);
2633 skcomp
= pci_zalloc_consistent(skdev
->pdev
, SKD_SKCOMP_SIZE
,
2634 &skdev
->cq_dma_address
);
2636 if (skcomp
== NULL
) {
2641 skdev
->skcomp_table
= skcomp
;
2642 skdev
->skerr_table
= (struct fit_comp_error_info
*)((char *)skcomp
+
2644 SKD_N_COMPLETION_ENTRY
);
2650 static int skd_cons_skmsg(struct skd_device
*skdev
)
2655 dev_dbg(&skdev
->pdev
->dev
,
2656 "skmsg_table kcalloc, struct %lu, count %u total %lu\n",
2657 sizeof(struct skd_fitmsg_context
), skdev
->num_fitmsg_context
,
2658 sizeof(struct skd_fitmsg_context
) * skdev
->num_fitmsg_context
);
2660 skdev
->skmsg_table
= kcalloc(skdev
->num_fitmsg_context
,
2661 sizeof(struct skd_fitmsg_context
),
2663 if (skdev
->skmsg_table
== NULL
) {
2668 for (i
= 0; i
< skdev
->num_fitmsg_context
; i
++) {
2669 struct skd_fitmsg_context
*skmsg
;
2671 skmsg
= &skdev
->skmsg_table
[i
];
2673 skmsg
->id
= i
+ SKD_ID_FIT_MSG
;
2675 skmsg
->msg_buf
= pci_alloc_consistent(skdev
->pdev
,
2677 &skmsg
->mb_dma_address
);
2679 if (skmsg
->msg_buf
== NULL
) {
2684 WARN(((uintptr_t)skmsg
->msg_buf
| skmsg
->mb_dma_address
) &
2685 (FIT_QCMD_ALIGN
- 1),
2686 "not aligned: msg_buf %p mb_dma_address %#llx\n",
2687 skmsg
->msg_buf
, skmsg
->mb_dma_address
);
2688 memset(skmsg
->msg_buf
, 0, SKD_N_FITMSG_BYTES
);
2695 static struct fit_sg_descriptor
*skd_cons_sg_list(struct skd_device
*skdev
,
2697 dma_addr_t
*ret_dma_addr
)
2699 struct fit_sg_descriptor
*sg_list
;
2701 sg_list
= skd_alloc_dma(skdev
, skdev
->sglist_cache
, ret_dma_addr
,
2702 GFP_DMA
| __GFP_ZERO
, DMA_TO_DEVICE
);
2704 if (sg_list
!= NULL
) {
2705 uint64_t dma_address
= *ret_dma_addr
;
2708 for (i
= 0; i
< n_sg
- 1; i
++) {
2710 ndp_off
= (i
+ 1) * sizeof(struct fit_sg_descriptor
);
2712 sg_list
[i
].next_desc_ptr
= dma_address
+ ndp_off
;
2714 sg_list
[i
].next_desc_ptr
= 0LL;
2720 static void skd_free_sg_list(struct skd_device
*skdev
,
2721 struct fit_sg_descriptor
*sg_list
,
2722 dma_addr_t dma_addr
)
2724 if (WARN_ON_ONCE(!sg_list
))
2727 skd_free_dma(skdev
, skdev
->sglist_cache
, sg_list
, dma_addr
,
2731 static int skd_init_request(struct blk_mq_tag_set
*set
, struct request
*rq
,
2732 unsigned int hctx_idx
, unsigned int numa_node
)
2734 struct skd_device
*skdev
= set
->driver_data
;
2735 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(rq
);
2737 skreq
->state
= SKD_REQ_STATE_IDLE
;
2738 skreq
->sg
= (void *)(skreq
+ 1);
2739 sg_init_table(skreq
->sg
, skd_sgs_per_request
);
2740 skreq
->sksg_list
= skd_cons_sg_list(skdev
, skd_sgs_per_request
,
2741 &skreq
->sksg_dma_address
);
2743 return skreq
->sksg_list
? 0 : -ENOMEM
;
2746 static void skd_exit_request(struct blk_mq_tag_set
*set
, struct request
*rq
,
2747 unsigned int hctx_idx
)
2749 struct skd_device
*skdev
= set
->driver_data
;
2750 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(rq
);
2752 skd_free_sg_list(skdev
, skreq
->sksg_list
, skreq
->sksg_dma_address
);
2755 static int skd_cons_sksb(struct skd_device
*skdev
)
2758 struct skd_special_context
*skspcl
;
2760 skspcl
= &skdev
->internal_skspcl
;
2762 skspcl
->req
.id
= 0 + SKD_ID_INTERNAL
;
2763 skspcl
->req
.state
= SKD_REQ_STATE_IDLE
;
2765 skspcl
->data_buf
= skd_alloc_dma(skdev
, skdev
->databuf_cache
,
2766 &skspcl
->db_dma_address
,
2767 GFP_DMA
| __GFP_ZERO
,
2769 if (skspcl
->data_buf
== NULL
) {
2774 skspcl
->msg_buf
= skd_alloc_dma(skdev
, skdev
->msgbuf_cache
,
2775 &skspcl
->mb_dma_address
,
2776 GFP_DMA
| __GFP_ZERO
, DMA_TO_DEVICE
);
2777 if (skspcl
->msg_buf
== NULL
) {
2782 skspcl
->req
.sksg_list
= skd_cons_sg_list(skdev
, 1,
2783 &skspcl
->req
.sksg_dma_address
);
2784 if (skspcl
->req
.sksg_list
== NULL
) {
2789 if (!skd_format_internal_skspcl(skdev
)) {
2798 static const struct blk_mq_ops skd_mq_ops
= {
2799 .queue_rq
= skd_mq_queue_rq
,
2800 .complete
= skd_complete_rq
,
2801 .timeout
= skd_timed_out
,
2802 .init_request
= skd_init_request
,
2803 .exit_request
= skd_exit_request
,
2806 static int skd_cons_disk(struct skd_device
*skdev
)
2809 struct gendisk
*disk
;
2810 struct request_queue
*q
;
2811 unsigned long flags
;
2813 disk
= alloc_disk(SKD_MINORS_PER_DEVICE
);
2820 sprintf(disk
->disk_name
, DRV_NAME
"%u", skdev
->devno
);
2822 disk
->major
= skdev
->major
;
2823 disk
->first_minor
= skdev
->devno
* SKD_MINORS_PER_DEVICE
;
2824 disk
->fops
= &skd_blockdev_ops
;
2825 disk
->private_data
= skdev
;
2827 memset(&skdev
->tag_set
, 0, sizeof(skdev
->tag_set
));
2828 skdev
->tag_set
.ops
= &skd_mq_ops
;
2829 skdev
->tag_set
.nr_hw_queues
= 1;
2830 skdev
->tag_set
.queue_depth
= skd_max_queue_depth
;
2831 skdev
->tag_set
.cmd_size
= sizeof(struct skd_request_context
) +
2832 skdev
->sgs_per_request
* sizeof(struct scatterlist
);
2833 skdev
->tag_set
.numa_node
= NUMA_NO_NODE
;
2834 skdev
->tag_set
.flags
= BLK_MQ_F_SHOULD_MERGE
|
2836 BLK_ALLOC_POLICY_TO_MQ_FLAG(BLK_TAG_ALLOC_FIFO
);
2837 skdev
->tag_set
.driver_data
= skdev
;
2838 rc
= blk_mq_alloc_tag_set(&skdev
->tag_set
);
2841 q
= blk_mq_init_queue(&skdev
->tag_set
);
2843 blk_mq_free_tag_set(&skdev
->tag_set
);
2847 q
->queuedata
= skdev
;
2852 blk_queue_write_cache(q
, true, true);
2853 blk_queue_max_segments(q
, skdev
->sgs_per_request
);
2854 blk_queue_max_hw_sectors(q
, SKD_N_MAX_SECTORS
);
2856 /* set optimal I/O size to 8KB */
2857 blk_queue_io_opt(q
, 8192);
2859 queue_flag_set_unlocked(QUEUE_FLAG_NONROT
, q
);
2860 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM
, q
);
2862 blk_queue_rq_timeout(q
, 8 * HZ
);
2864 spin_lock_irqsave(&skdev
->lock
, flags
);
2865 dev_dbg(&skdev
->pdev
->dev
, "stopping queue\n");
2866 blk_mq_stop_hw_queues(skdev
->queue
);
2867 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2873 #define SKD_N_DEV_TABLE 16u
2874 static u32 skd_next_devno
;
2876 static struct skd_device
*skd_construct(struct pci_dev
*pdev
)
2878 struct skd_device
*skdev
;
2879 int blk_major
= skd_major
;
2883 skdev
= kzalloc(sizeof(*skdev
), GFP_KERNEL
);
2886 dev_err(&pdev
->dev
, "memory alloc failure\n");
2890 skdev
->state
= SKD_DRVR_STATE_LOAD
;
2892 skdev
->devno
= skd_next_devno
++;
2893 skdev
->major
= blk_major
;
2894 skdev
->dev_max_queue_depth
= 0;
2896 skdev
->num_req_context
= skd_max_queue_depth
;
2897 skdev
->num_fitmsg_context
= skd_max_queue_depth
;
2898 skdev
->cur_max_queue_depth
= 1;
2899 skdev
->queue_low_water_mark
= 1;
2900 skdev
->proto_ver
= 99;
2901 skdev
->sgs_per_request
= skd_sgs_per_request
;
2902 skdev
->dbg_level
= skd_dbg_level
;
2904 spin_lock_init(&skdev
->lock
);
2906 INIT_WORK(&skdev
->start_queue
, skd_start_queue
);
2907 INIT_WORK(&skdev
->completion_worker
, skd_completion_worker
);
2909 size
= max(SKD_N_FITMSG_BYTES
, SKD_N_SPECIAL_FITMSG_BYTES
);
2910 skdev
->msgbuf_cache
= kmem_cache_create("skd-msgbuf", size
, 0,
2911 SLAB_HWCACHE_ALIGN
, NULL
);
2912 if (!skdev
->msgbuf_cache
)
2914 WARN_ONCE(kmem_cache_size(skdev
->msgbuf_cache
) < size
,
2915 "skd-msgbuf: %d < %zd\n",
2916 kmem_cache_size(skdev
->msgbuf_cache
), size
);
2917 size
= skd_sgs_per_request
* sizeof(struct fit_sg_descriptor
);
2918 skdev
->sglist_cache
= kmem_cache_create("skd-sglist", size
, 0,
2919 SLAB_HWCACHE_ALIGN
, NULL
);
2920 if (!skdev
->sglist_cache
)
2922 WARN_ONCE(kmem_cache_size(skdev
->sglist_cache
) < size
,
2923 "skd-sglist: %d < %zd\n",
2924 kmem_cache_size(skdev
->sglist_cache
), size
);
2925 size
= SKD_N_INTERNAL_BYTES
;
2926 skdev
->databuf_cache
= kmem_cache_create("skd-databuf", size
, 0,
2927 SLAB_HWCACHE_ALIGN
, NULL
);
2928 if (!skdev
->databuf_cache
)
2930 WARN_ONCE(kmem_cache_size(skdev
->databuf_cache
) < size
,
2931 "skd-databuf: %d < %zd\n",
2932 kmem_cache_size(skdev
->databuf_cache
), size
);
2934 dev_dbg(&skdev
->pdev
->dev
, "skcomp\n");
2935 rc
= skd_cons_skcomp(skdev
);
2939 dev_dbg(&skdev
->pdev
->dev
, "skmsg\n");
2940 rc
= skd_cons_skmsg(skdev
);
2944 dev_dbg(&skdev
->pdev
->dev
, "sksb\n");
2945 rc
= skd_cons_sksb(skdev
);
2949 dev_dbg(&skdev
->pdev
->dev
, "disk\n");
2950 rc
= skd_cons_disk(skdev
);
2954 dev_dbg(&skdev
->pdev
->dev
, "VICTORY\n");
2958 dev_dbg(&skdev
->pdev
->dev
, "construct failed\n");
2959 skd_destruct(skdev
);
2964 *****************************************************************************
2966 *****************************************************************************
2969 static void skd_free_skcomp(struct skd_device
*skdev
)
2971 if (skdev
->skcomp_table
)
2972 pci_free_consistent(skdev
->pdev
, SKD_SKCOMP_SIZE
,
2973 skdev
->skcomp_table
, skdev
->cq_dma_address
);
2975 skdev
->skcomp_table
= NULL
;
2976 skdev
->cq_dma_address
= 0;
2979 static void skd_free_skmsg(struct skd_device
*skdev
)
2983 if (skdev
->skmsg_table
== NULL
)
2986 for (i
= 0; i
< skdev
->num_fitmsg_context
; i
++) {
2987 struct skd_fitmsg_context
*skmsg
;
2989 skmsg
= &skdev
->skmsg_table
[i
];
2991 if (skmsg
->msg_buf
!= NULL
) {
2992 pci_free_consistent(skdev
->pdev
, SKD_N_FITMSG_BYTES
,
2994 skmsg
->mb_dma_address
);
2996 skmsg
->msg_buf
= NULL
;
2997 skmsg
->mb_dma_address
= 0;
3000 kfree(skdev
->skmsg_table
);
3001 skdev
->skmsg_table
= NULL
;
3004 static void skd_free_sksb(struct skd_device
*skdev
)
3006 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
3008 skd_free_dma(skdev
, skdev
->databuf_cache
, skspcl
->data_buf
,
3009 skspcl
->db_dma_address
, DMA_BIDIRECTIONAL
);
3011 skspcl
->data_buf
= NULL
;
3012 skspcl
->db_dma_address
= 0;
3014 skd_free_dma(skdev
, skdev
->msgbuf_cache
, skspcl
->msg_buf
,
3015 skspcl
->mb_dma_address
, DMA_TO_DEVICE
);
3017 skspcl
->msg_buf
= NULL
;
3018 skspcl
->mb_dma_address
= 0;
3020 skd_free_sg_list(skdev
, skspcl
->req
.sksg_list
,
3021 skspcl
->req
.sksg_dma_address
);
3023 skspcl
->req
.sksg_list
= NULL
;
3024 skspcl
->req
.sksg_dma_address
= 0;
3027 static void skd_free_disk(struct skd_device
*skdev
)
3029 struct gendisk
*disk
= skdev
->disk
;
3031 if (disk
&& (disk
->flags
& GENHD_FL_UP
))
3035 blk_cleanup_queue(skdev
->queue
);
3036 skdev
->queue
= NULL
;
3041 if (skdev
->tag_set
.tags
)
3042 blk_mq_free_tag_set(&skdev
->tag_set
);
3048 static void skd_destruct(struct skd_device
*skdev
)
3053 cancel_work_sync(&skdev
->start_queue
);
3055 dev_dbg(&skdev
->pdev
->dev
, "disk\n");
3056 skd_free_disk(skdev
);
3058 dev_dbg(&skdev
->pdev
->dev
, "sksb\n");
3059 skd_free_sksb(skdev
);
3061 dev_dbg(&skdev
->pdev
->dev
, "skmsg\n");
3062 skd_free_skmsg(skdev
);
3064 dev_dbg(&skdev
->pdev
->dev
, "skcomp\n");
3065 skd_free_skcomp(skdev
);
3067 kmem_cache_destroy(skdev
->databuf_cache
);
3068 kmem_cache_destroy(skdev
->sglist_cache
);
3069 kmem_cache_destroy(skdev
->msgbuf_cache
);
3071 dev_dbg(&skdev
->pdev
->dev
, "skdev\n");
3076 *****************************************************************************
3077 * BLOCK DEVICE (BDEV) GLUE
3078 *****************************************************************************
3081 static int skd_bdev_getgeo(struct block_device
*bdev
, struct hd_geometry
*geo
)
3083 struct skd_device
*skdev
;
3086 skdev
= bdev
->bd_disk
->private_data
;
3088 dev_dbg(&skdev
->pdev
->dev
, "%s: CMD[%s] getgeo device\n",
3089 bdev
->bd_disk
->disk_name
, current
->comm
);
3091 if (skdev
->read_cap_is_valid
) {
3092 capacity
= get_capacity(skdev
->disk
);
3095 geo
->cylinders
= (capacity
) / (255 * 64);
3102 static int skd_bdev_attach(struct device
*parent
, struct skd_device
*skdev
)
3104 dev_dbg(&skdev
->pdev
->dev
, "add_disk\n");
3105 device_add_disk(parent
, skdev
->disk
);
3109 static const struct block_device_operations skd_blockdev_ops
= {
3110 .owner
= THIS_MODULE
,
3111 .getgeo
= skd_bdev_getgeo
,
3115 *****************************************************************************
3117 *****************************************************************************
3120 static const struct pci_device_id skd_pci_tbl
[] = {
3121 { PCI_VENDOR_ID_STEC
, PCI_DEVICE_ID_S1120
,
3122 PCI_ANY_ID
, PCI_ANY_ID
, 0, 0, },
3123 { 0 } /* terminate list */
3126 MODULE_DEVICE_TABLE(pci
, skd_pci_tbl
);
3128 static char *skd_pci_info(struct skd_device
*skdev
, char *str
)
3132 strcpy(str
, "PCIe (");
3133 pcie_reg
= pci_find_capability(skdev
->pdev
, PCI_CAP_ID_EXP
);
3138 uint16_t pcie_lstat
, lspeed
, lwidth
;
3141 pci_read_config_word(skdev
->pdev
, pcie_reg
, &pcie_lstat
);
3142 lspeed
= pcie_lstat
& (0xF);
3143 lwidth
= (pcie_lstat
& 0x3F0) >> 4;
3146 strcat(str
, "2.5GT/s ");
3147 else if (lspeed
== 2)
3148 strcat(str
, "5.0GT/s ");
3150 strcat(str
, "<unknown> ");
3151 snprintf(lwstr
, sizeof(lwstr
), "%dX)", lwidth
);
3157 static int skd_pci_probe(struct pci_dev
*pdev
, const struct pci_device_id
*ent
)
3162 struct skd_device
*skdev
;
3164 dev_dbg(&pdev
->dev
, "vendor=%04X device=%04x\n", pdev
->vendor
,
3167 rc
= pci_enable_device(pdev
);
3170 rc
= pci_request_regions(pdev
, DRV_NAME
);
3173 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(64));
3175 if (pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64))) {
3176 dev_err(&pdev
->dev
, "consistent DMA mask error %d\n",
3180 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
3182 dev_err(&pdev
->dev
, "DMA mask error %d\n", rc
);
3183 goto err_out_regions
;
3188 rc
= register_blkdev(0, DRV_NAME
);
3190 goto err_out_regions
;
3195 skdev
= skd_construct(pdev
);
3196 if (skdev
== NULL
) {
3198 goto err_out_regions
;
3201 skd_pci_info(skdev
, pci_str
);
3202 dev_info(&pdev
->dev
, "%s 64bit\n", pci_str
);
3204 pci_set_master(pdev
);
3205 rc
= pci_enable_pcie_error_reporting(pdev
);
3208 "bad enable of PCIe error reporting rc=%d\n", rc
);
3209 skdev
->pcie_error_reporting_is_enabled
= 0;
3211 skdev
->pcie_error_reporting_is_enabled
= 1;
3213 pci_set_drvdata(pdev
, skdev
);
3215 for (i
= 0; i
< SKD_MAX_BARS
; i
++) {
3216 skdev
->mem_phys
[i
] = pci_resource_start(pdev
, i
);
3217 skdev
->mem_size
[i
] = (u32
)pci_resource_len(pdev
, i
);
3218 skdev
->mem_map
[i
] = ioremap(skdev
->mem_phys
[i
],
3219 skdev
->mem_size
[i
]);
3220 if (!skdev
->mem_map
[i
]) {
3222 "Unable to map adapter memory!\n");
3224 goto err_out_iounmap
;
3226 dev_dbg(&pdev
->dev
, "mem_map=%p, phyd=%016llx, size=%d\n",
3227 skdev
->mem_map
[i
], (uint64_t)skdev
->mem_phys
[i
],
3228 skdev
->mem_size
[i
]);
3231 rc
= skd_acquire_irq(skdev
);
3233 dev_err(&pdev
->dev
, "interrupt resource error %d\n", rc
);
3234 goto err_out_iounmap
;
3237 rc
= skd_start_timer(skdev
);
3241 init_waitqueue_head(&skdev
->waitq
);
3243 skd_start_device(skdev
);
3245 rc
= wait_event_interruptible_timeout(skdev
->waitq
,
3246 (skdev
->gendisk_on
),
3247 (SKD_START_WAIT_SECONDS
* HZ
));
3248 if (skdev
->gendisk_on
> 0) {
3249 /* device came on-line after reset */
3250 skd_bdev_attach(&pdev
->dev
, skdev
);
3253 /* we timed out, something is wrong with the device,
3254 don't add the disk structure */
3255 dev_err(&pdev
->dev
, "error: waiting for s1120 timed out %d!\n",
3257 /* in case of no error; we timeout with ENXIO */
3266 skd_stop_device(skdev
);
3267 skd_release_irq(skdev
);
3270 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3271 if (skdev
->mem_map
[i
])
3272 iounmap(skdev
->mem_map
[i
]);
3274 if (skdev
->pcie_error_reporting_is_enabled
)
3275 pci_disable_pcie_error_reporting(pdev
);
3277 skd_destruct(skdev
);
3280 pci_release_regions(pdev
);
3283 pci_disable_device(pdev
);
3284 pci_set_drvdata(pdev
, NULL
);
3288 static void skd_pci_remove(struct pci_dev
*pdev
)
3291 struct skd_device
*skdev
;
3293 skdev
= pci_get_drvdata(pdev
);
3295 dev_err(&pdev
->dev
, "no device data for PCI\n");
3298 skd_stop_device(skdev
);
3299 skd_release_irq(skdev
);
3301 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3302 if (skdev
->mem_map
[i
])
3303 iounmap(skdev
->mem_map
[i
]);
3305 if (skdev
->pcie_error_reporting_is_enabled
)
3306 pci_disable_pcie_error_reporting(pdev
);
3308 skd_destruct(skdev
);
3310 pci_release_regions(pdev
);
3311 pci_disable_device(pdev
);
3312 pci_set_drvdata(pdev
, NULL
);
3317 static int skd_pci_suspend(struct pci_dev
*pdev
, pm_message_t state
)
3320 struct skd_device
*skdev
;
3322 skdev
= pci_get_drvdata(pdev
);
3324 dev_err(&pdev
->dev
, "no device data for PCI\n");
3328 skd_stop_device(skdev
);
3330 skd_release_irq(skdev
);
3332 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3333 if (skdev
->mem_map
[i
])
3334 iounmap(skdev
->mem_map
[i
]);
3336 if (skdev
->pcie_error_reporting_is_enabled
)
3337 pci_disable_pcie_error_reporting(pdev
);
3339 pci_release_regions(pdev
);
3340 pci_save_state(pdev
);
3341 pci_disable_device(pdev
);
3342 pci_set_power_state(pdev
, pci_choose_state(pdev
, state
));
3346 static int skd_pci_resume(struct pci_dev
*pdev
)
3350 struct skd_device
*skdev
;
3352 skdev
= pci_get_drvdata(pdev
);
3354 dev_err(&pdev
->dev
, "no device data for PCI\n");
3358 pci_set_power_state(pdev
, PCI_D0
);
3359 pci_enable_wake(pdev
, PCI_D0
, 0);
3360 pci_restore_state(pdev
);
3362 rc
= pci_enable_device(pdev
);
3365 rc
= pci_request_regions(pdev
, DRV_NAME
);
3368 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(64));
3370 if (pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64))) {
3372 dev_err(&pdev
->dev
, "consistent DMA mask error %d\n",
3376 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
3379 dev_err(&pdev
->dev
, "DMA mask error %d\n", rc
);
3380 goto err_out_regions
;
3384 pci_set_master(pdev
);
3385 rc
= pci_enable_pcie_error_reporting(pdev
);
3388 "bad enable of PCIe error reporting rc=%d\n", rc
);
3389 skdev
->pcie_error_reporting_is_enabled
= 0;
3391 skdev
->pcie_error_reporting_is_enabled
= 1;
3393 for (i
= 0; i
< SKD_MAX_BARS
; i
++) {
3395 skdev
->mem_phys
[i
] = pci_resource_start(pdev
, i
);
3396 skdev
->mem_size
[i
] = (u32
)pci_resource_len(pdev
, i
);
3397 skdev
->mem_map
[i
] = ioremap(skdev
->mem_phys
[i
],
3398 skdev
->mem_size
[i
]);
3399 if (!skdev
->mem_map
[i
]) {
3400 dev_err(&pdev
->dev
, "Unable to map adapter memory!\n");
3402 goto err_out_iounmap
;
3404 dev_dbg(&pdev
->dev
, "mem_map=%p, phyd=%016llx, size=%d\n",
3405 skdev
->mem_map
[i
], (uint64_t)skdev
->mem_phys
[i
],
3406 skdev
->mem_size
[i
]);
3408 rc
= skd_acquire_irq(skdev
);
3410 dev_err(&pdev
->dev
, "interrupt resource error %d\n", rc
);
3411 goto err_out_iounmap
;
3414 rc
= skd_start_timer(skdev
);
3418 init_waitqueue_head(&skdev
->waitq
);
3420 skd_start_device(skdev
);
3425 skd_stop_device(skdev
);
3426 skd_release_irq(skdev
);
3429 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3430 if (skdev
->mem_map
[i
])
3431 iounmap(skdev
->mem_map
[i
]);
3433 if (skdev
->pcie_error_reporting_is_enabled
)
3434 pci_disable_pcie_error_reporting(pdev
);
3437 pci_release_regions(pdev
);
3440 pci_disable_device(pdev
);
3444 static void skd_pci_shutdown(struct pci_dev
*pdev
)
3446 struct skd_device
*skdev
;
3448 dev_err(&pdev
->dev
, "%s called\n", __func__
);
3450 skdev
= pci_get_drvdata(pdev
);
3452 dev_err(&pdev
->dev
, "no device data for PCI\n");
3456 dev_err(&pdev
->dev
, "calling stop\n");
3457 skd_stop_device(skdev
);
3460 static struct pci_driver skd_driver
= {
3462 .id_table
= skd_pci_tbl
,
3463 .probe
= skd_pci_probe
,
3464 .remove
= skd_pci_remove
,
3465 .suspend
= skd_pci_suspend
,
3466 .resume
= skd_pci_resume
,
3467 .shutdown
= skd_pci_shutdown
,
3471 *****************************************************************************
3473 *****************************************************************************
3476 const char *skd_drive_state_to_str(int state
)
3479 case FIT_SR_DRIVE_OFFLINE
:
3481 case FIT_SR_DRIVE_INIT
:
3483 case FIT_SR_DRIVE_ONLINE
:
3485 case FIT_SR_DRIVE_BUSY
:
3487 case FIT_SR_DRIVE_FAULT
:
3489 case FIT_SR_DRIVE_DEGRADED
:
3491 case FIT_SR_PCIE_LINK_DOWN
:
3493 case FIT_SR_DRIVE_SOFT_RESET
:
3494 return "SOFT_RESET";
3495 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD
:
3497 case FIT_SR_DRIVE_INIT_FAULT
:
3498 return "INIT_FAULT";
3499 case FIT_SR_DRIVE_BUSY_SANITIZE
:
3500 return "BUSY_SANITIZE";
3501 case FIT_SR_DRIVE_BUSY_ERASE
:
3502 return "BUSY_ERASE";
3503 case FIT_SR_DRIVE_FW_BOOTING
:
3504 return "FW_BOOTING";
3510 const char *skd_skdev_state_to_str(enum skd_drvr_state state
)
3513 case SKD_DRVR_STATE_LOAD
:
3515 case SKD_DRVR_STATE_IDLE
:
3517 case SKD_DRVR_STATE_BUSY
:
3519 case SKD_DRVR_STATE_STARTING
:
3521 case SKD_DRVR_STATE_ONLINE
:
3523 case SKD_DRVR_STATE_PAUSING
:
3525 case SKD_DRVR_STATE_PAUSED
:
3527 case SKD_DRVR_STATE_RESTARTING
:
3528 return "RESTARTING";
3529 case SKD_DRVR_STATE_RESUMING
:
3531 case SKD_DRVR_STATE_STOPPING
:
3533 case SKD_DRVR_STATE_SYNCING
:
3535 case SKD_DRVR_STATE_FAULT
:
3537 case SKD_DRVR_STATE_DISAPPEARED
:
3538 return "DISAPPEARED";
3539 case SKD_DRVR_STATE_BUSY_ERASE
:
3540 return "BUSY_ERASE";
3541 case SKD_DRVR_STATE_BUSY_SANITIZE
:
3542 return "BUSY_SANITIZE";
3543 case SKD_DRVR_STATE_BUSY_IMMINENT
:
3544 return "BUSY_IMMINENT";
3545 case SKD_DRVR_STATE_WAIT_BOOT
:
3553 static const char *skd_skreq_state_to_str(enum skd_req_state state
)
3556 case SKD_REQ_STATE_IDLE
:
3558 case SKD_REQ_STATE_SETUP
:
3560 case SKD_REQ_STATE_BUSY
:
3562 case SKD_REQ_STATE_COMPLETED
:
3564 case SKD_REQ_STATE_TIMEOUT
:
3571 static void skd_log_skdev(struct skd_device
*skdev
, const char *event
)
3573 dev_dbg(&skdev
->pdev
->dev
, "skdev=%p event='%s'\n", skdev
, event
);
3574 dev_dbg(&skdev
->pdev
->dev
, " drive_state=%s(%d) driver_state=%s(%d)\n",
3575 skd_drive_state_to_str(skdev
->drive_state
), skdev
->drive_state
,
3576 skd_skdev_state_to_str(skdev
->state
), skdev
->state
);
3577 dev_dbg(&skdev
->pdev
->dev
, " busy=%d limit=%d dev=%d lowat=%d\n",
3578 skd_in_flight(skdev
), skdev
->cur_max_queue_depth
,
3579 skdev
->dev_max_queue_depth
, skdev
->queue_low_water_mark
);
3580 dev_dbg(&skdev
->pdev
->dev
, " cycle=%d cycle_ix=%d\n",
3581 skdev
->skcomp_cycle
, skdev
->skcomp_ix
);
3584 static void skd_log_skreq(struct skd_device
*skdev
,
3585 struct skd_request_context
*skreq
, const char *event
)
3587 struct request
*req
= blk_mq_rq_from_pdu(skreq
);
3588 u32 lba
= blk_rq_pos(req
);
3589 u32 count
= blk_rq_sectors(req
);
3591 dev_dbg(&skdev
->pdev
->dev
, "skreq=%p event='%s'\n", skreq
, event
);
3592 dev_dbg(&skdev
->pdev
->dev
, " state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
3593 skd_skreq_state_to_str(skreq
->state
), skreq
->state
, skreq
->id
,
3595 dev_dbg(&skdev
->pdev
->dev
, " sg_dir=%d n_sg=%d\n",
3596 skreq
->data_dir
, skreq
->n_sg
);
3598 dev_dbg(&skdev
->pdev
->dev
,
3599 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n", req
, lba
, lba
,
3600 count
, count
, (int)rq_data_dir(req
));
3604 *****************************************************************************
3606 *****************************************************************************
3609 static int __init
skd_init(void)
3611 BUILD_BUG_ON(sizeof(struct fit_completion_entry_v1
) != 8);
3612 BUILD_BUG_ON(sizeof(struct fit_comp_error_info
) != 32);
3613 BUILD_BUG_ON(sizeof(struct skd_command_header
) != 16);
3614 BUILD_BUG_ON(sizeof(struct skd_scsi_request
) != 32);
3615 BUILD_BUG_ON(sizeof(struct driver_inquiry_data
) != 44);
3616 BUILD_BUG_ON(offsetof(struct skd_msg_buf
, fmh
) != 0);
3617 BUILD_BUG_ON(offsetof(struct skd_msg_buf
, scsi
) != 64);
3618 BUILD_BUG_ON(sizeof(struct skd_msg_buf
) != SKD_N_FITMSG_BYTES
);
3620 switch (skd_isr_type
) {
3621 case SKD_IRQ_LEGACY
:
3626 pr_err(PFX
"skd_isr_type %d invalid, re-set to %d\n",
3627 skd_isr_type
, SKD_IRQ_DEFAULT
);
3628 skd_isr_type
= SKD_IRQ_DEFAULT
;
3631 if (skd_max_queue_depth
< 1 ||
3632 skd_max_queue_depth
> SKD_MAX_QUEUE_DEPTH
) {
3633 pr_err(PFX
"skd_max_queue_depth %d invalid, re-set to %d\n",
3634 skd_max_queue_depth
, SKD_MAX_QUEUE_DEPTH_DEFAULT
);
3635 skd_max_queue_depth
= SKD_MAX_QUEUE_DEPTH_DEFAULT
;
3638 if (skd_max_req_per_msg
< 1 ||
3639 skd_max_req_per_msg
> SKD_MAX_REQ_PER_MSG
) {
3640 pr_err(PFX
"skd_max_req_per_msg %d invalid, re-set to %d\n",
3641 skd_max_req_per_msg
, SKD_MAX_REQ_PER_MSG_DEFAULT
);
3642 skd_max_req_per_msg
= SKD_MAX_REQ_PER_MSG_DEFAULT
;
3645 if (skd_sgs_per_request
< 1 || skd_sgs_per_request
> 4096) {
3646 pr_err(PFX
"skd_sg_per_request %d invalid, re-set to %d\n",
3647 skd_sgs_per_request
, SKD_N_SG_PER_REQ_DEFAULT
);
3648 skd_sgs_per_request
= SKD_N_SG_PER_REQ_DEFAULT
;
3651 if (skd_dbg_level
< 0 || skd_dbg_level
> 2) {
3652 pr_err(PFX
"skd_dbg_level %d invalid, re-set to %d\n",
3657 if (skd_isr_comp_limit
< 0) {
3658 pr_err(PFX
"skd_isr_comp_limit %d invalid, set to %d\n",
3659 skd_isr_comp_limit
, 0);
3660 skd_isr_comp_limit
= 0;
3663 return pci_register_driver(&skd_driver
);
3666 static void __exit
skd_exit(void)
3668 pci_unregister_driver(&skd_driver
);
3671 unregister_blkdev(skd_major
, DRV_NAME
);
3674 module_init(skd_init
);
3675 module_exit(skd_exit
);