]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/block/skd_main.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
skd: Reduce memory usage
[mirror_ubuntu-hirsute-kernel.git] / drivers / block / skd_main.c
CommitLineData
bec9e8ac
BVA		 1/*
2 * Driver for sTec s1120 PCIe SSDs. sTec was acquired in 2013 by HGST and HGST
3 * was acquired by Western Digital in 2012.
e67f86b3 4 *
bec9e8ac
BVA		 5 * Copyright 2012 sTec, Inc.
6 * Copyright (c) 2017 Western Digital Corporation or its affiliates.
7 *
8 * This file is part of the Linux kernel, and is made available under
9 * the terms of the GNU General Public License version 2.
e67f86b3
AB		 10 */
11
12#include <linux/kernel.h>
13#include <linux/module.h>
14#include <linux/init.h>
15#include <linux/pci.h>
16#include <linux/slab.h>
17#include <linux/spinlock.h>
18#include <linux/blkdev.h>
f18c17c8 19#include <linux/blk-mq.h>
e67f86b3
AB		 20#include <linux/sched.h>
21#include <linux/interrupt.h>
22#include <linux/compiler.h>
23#include <linux/workqueue.h>
e67f86b3
AB		 24#include <linux/delay.h>
25#include <linux/time.h>
26#include <linux/hdreg.h>
27#include <linux/dma-mapping.h>
28#include <linux/completion.h>
29#include <linux/scatterlist.h>
30#include <linux/version.h>
31#include <linux/err.h>
e67f86b3 32#include <linux/aer.h>
e67f86b3 33#include <linux/wait.h>
2da7b403 34#include <linux/stringify.h>
a3db102d 35#include <linux/slab_def.h>
e67f86b3 36#include <scsi/scsi.h>
e67f86b3
AB		 37#include <scsi/sg.h>
38#include <linux/io.h>
39#include <linux/uaccess.h>
4ca90b53 40#include <asm/unaligned.h>
e67f86b3
AB		 41
42#include "skd_s1120.h"
43
44static int skd_dbg_level;
45static int skd_isr_comp_limit = 4;
46
e67f86b3
AB		 47enum {
48 SKD_FLUSH_INITIALIZER,
49 SKD_FLUSH_ZERO_SIZE_FIRST,
50 SKD_FLUSH_DATA_SECOND,
51};
52
e67f86b3
AB		 53#define SKD_ASSERT(expr) \
54 do { \
55 if (unlikely(!(expr))) { \
56 pr_err("Assertion failed! %s,%s,%s,line=%d\n", \
57 # expr, __FILE__, __func__, __LINE__); \
58 } \
59 } while (0)
60
e67f86b3
AB		 61#define DRV_NAME "skd"
62#define DRV_VERSION "2.2.1"
63#define DRV_BUILD_ID "0260"
64#define PFX DRV_NAME ": "
e67f86b3 65
bec9e8ac 66MODULE_LICENSE("GPL");
e67f86b3 67
38d4a1bb 68MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver (b" DRV_BUILD_ID ")");
e67f86b3
AB		 69MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
70
71#define PCI_VENDOR_ID_STEC 0x1B39
72#define PCI_DEVICE_ID_S1120 0x0001
73
74#define SKD_FUA_NV (1 << 1)
75#define SKD_MINORS_PER_DEVICE 16
76
77#define SKD_MAX_QUEUE_DEPTH 200u
78
79#define SKD_PAUSE_TIMEOUT (5 * 1000)
80
81#define SKD_N_FITMSG_BYTES (512u)
2da7b403 82#define SKD_MAX_REQ_PER_MSG 14
e67f86b3 83
e67f86b3
AB		 84#define SKD_N_SPECIAL_FITMSG_BYTES (128u)
85
86/* SG elements are 32 bytes, so we can make this 4096 and still be under the
87 * 128KB limit. That allows 4096*4K = 16M xfer size
88 */
89#define SKD_N_SG_PER_REQ_DEFAULT 256u
e67f86b3
AB		 90
91#define SKD_N_COMPLETION_ENTRY 256u
92#define SKD_N_READ_CAP_BYTES (8u)
93
94#define SKD_N_INTERNAL_BYTES (512u)
95
6f7c7675
BVA		 96#define SKD_SKCOMP_SIZE \
97 ((sizeof(struct fit_completion_entry_v1) + \
98 sizeof(struct fit_comp_error_info)) * SKD_N_COMPLETION_ENTRY)
99
e67f86b3
AB		 100/* 5 bits of uniqifier, 0xF800 */
101#define SKD_ID_INCR (0x400)
102#define SKD_ID_TABLE_MASK (3u << 8u)
103#define SKD_ID_RW_REQUEST (0u << 8u)
104#define SKD_ID_INTERNAL (1u << 8u)
e67f86b3
AB		 105#define SKD_ID_FIT_MSG (3u << 8u)
106#define SKD_ID_SLOT_MASK 0x00FFu
107#define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
108
e67f86b3
AB		 109#define SKD_N_MAX_SECTORS 2048u
110
111#define SKD_MAX_RETRIES 2u
112
113#define SKD_TIMER_SECONDS(seconds) (seconds)
114#define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
115
116#define INQ_STD_NBYTES 36
e67f86b3
AB		 117
118enum skd_drvr_state {
119 SKD_DRVR_STATE_LOAD,
120 SKD_DRVR_STATE_IDLE,
121 SKD_DRVR_STATE_BUSY,
122 SKD_DRVR_STATE_STARTING,
123 SKD_DRVR_STATE_ONLINE,
124 SKD_DRVR_STATE_PAUSING,
125 SKD_DRVR_STATE_PAUSED,
e67f86b3
AB		 126 SKD_DRVR_STATE_RESTARTING,
127 SKD_DRVR_STATE_RESUMING,
128 SKD_DRVR_STATE_STOPPING,
129 SKD_DRVR_STATE_FAULT,
130 SKD_DRVR_STATE_DISAPPEARED,
131 SKD_DRVR_STATE_PROTOCOL_MISMATCH,
132 SKD_DRVR_STATE_BUSY_ERASE,
133 SKD_DRVR_STATE_BUSY_SANITIZE,
134 SKD_DRVR_STATE_BUSY_IMMINENT,
135 SKD_DRVR_STATE_WAIT_BOOT,
136 SKD_DRVR_STATE_SYNCING,
137};
138
139#define SKD_WAIT_BOOT_TIMO SKD_TIMER_SECONDS(90u)
140#define SKD_STARTING_TIMO SKD_TIMER_SECONDS(8u)
141#define SKD_RESTARTING_TIMO SKD_TIMER_MINUTES(4u)
e67f86b3
AB		 142#define SKD_BUSY_TIMO SKD_TIMER_MINUTES(20u)
143#define SKD_STARTED_BUSY_TIMO SKD_TIMER_SECONDS(60u)
144#define SKD_START_WAIT_SECONDS 90u
145
146enum skd_req_state {
147 SKD_REQ_STATE_IDLE,
148 SKD_REQ_STATE_SETUP,
149 SKD_REQ_STATE_BUSY,
150 SKD_REQ_STATE_COMPLETED,
151 SKD_REQ_STATE_TIMEOUT,
e67f86b3
AB		 152};
153
e67f86b3
AB		 154enum skd_check_status_action {
155 SKD_CHECK_STATUS_REPORT_GOOD,
156 SKD_CHECK_STATUS_REPORT_SMART_ALERT,
157 SKD_CHECK_STATUS_REQUEUE_REQUEST,
158 SKD_CHECK_STATUS_REPORT_ERROR,
159 SKD_CHECK_STATUS_BUSY_IMMINENT,
160};
161
d891fe60
BVA		 162struct skd_msg_buf {
163 struct fit_msg_hdr fmh;
164 struct skd_scsi_request scsi[SKD_MAX_REQ_PER_MSG];
165};
166
e67f86b3 167struct skd_fitmsg_context {
e67f86b3 168 u32 id;
e67f86b3
AB		 169
170 u32 length;
e67f86b3 171
d891fe60 172 struct skd_msg_buf *msg_buf;
e67f86b3
AB		 173 dma_addr_t mb_dma_address;
174};
175
176struct skd_request_context {
177 enum skd_req_state state;
178
e67f86b3
AB		 179 u16 id;
180 u32 fitmsg_id;
181
e67f86b3 182 u8 flush_cmd;
e67f86b3 183
b1824eef 184 enum dma_data_direction data_dir;
e67f86b3
AB		 185 struct scatterlist *sg;
186 u32 n_sg;
187 u32 sg_byte_count;
188
189 struct fit_sg_descriptor *sksg_list;
190 dma_addr_t sksg_dma_address;
191
192 struct fit_completion_entry_v1 completion;
193
194 struct fit_comp_error_info err_info;
195
196};
e67f86b3
AB		 197
198struct skd_special_context {
199 struct skd_request_context req;
200
e67f86b3
AB		 201 void *data_buf;
202 dma_addr_t db_dma_address;
203
d891fe60 204 struct skd_msg_buf *msg_buf;
e67f86b3
AB		 205 dma_addr_t mb_dma_address;
206};
207
e67f86b3
AB		 208typedef enum skd_irq_type {
209 SKD_IRQ_LEGACY,
210 SKD_IRQ_MSI,
211 SKD_IRQ_MSIX
212} skd_irq_type_t;
213
214#define SKD_MAX_BARS 2
215
216struct skd_device {
85e34112 217 void __iomem *mem_map[SKD_MAX_BARS];
e67f86b3
AB		 218 resource_size_t mem_phys[SKD_MAX_BARS];
219 u32 mem_size[SKD_MAX_BARS];
220
e67f86b3
AB		 221 struct skd_msix_entry *msix_entries;
222
223 struct pci_dev *pdev;
224 int pcie_error_reporting_is_enabled;
225
226 spinlock_t lock;
227 struct gendisk *disk;
ca33dd92 228 struct blk_mq_tag_set tag_set;
e67f86b3 229 struct request_queue *queue;
91f85da4 230 struct skd_fitmsg_context *skmsg;
e67f86b3
AB		 231 struct device *class_dev;
232 int gendisk_on;
233 int sync_done;
234
e67f86b3
AB		 235 u32 devno;
236 u32 major;
e67f86b3
AB		 237 char isr_name[30];
238
239 enum skd_drvr_state state;
240 u32 drive_state;
241
e67f86b3
AB		 242 u32 cur_max_queue_depth;
243 u32 queue_low_water_mark;
244 u32 dev_max_queue_depth;
245
246 u32 num_fitmsg_context;
247 u32 num_req_context;
248
e67f86b3
AB		 249 struct skd_fitmsg_context *skmsg_table;
250
e67f86b3
AB		 251 struct skd_special_context internal_skspcl;
252 u32 read_cap_blocksize;
253 u32 read_cap_last_lba;
254 int read_cap_is_valid;
255 int inquiry_is_valid;
256 u8 inq_serial_num[13]; /*12 chars plus null term */
e67f86b3
AB		 257
258 u8 skcomp_cycle;
259 u32 skcomp_ix;
a3db102d
BVA		 260 struct kmem_cache *msgbuf_cache;
261 struct kmem_cache *sglist_cache;
262 struct kmem_cache *databuf_cache;
e67f86b3
AB		 263 struct fit_completion_entry_v1 *skcomp_table;
264 struct fit_comp_error_info *skerr_table;
265 dma_addr_t cq_dma_address;
266
267 wait_queue_head_t waitq;
268
269 struct timer_list timer;
270 u32 timer_countdown;
271 u32 timer_substate;
272
e67f86b3
AB		 273 int sgs_per_request;
274 u32 last_mtd;
275
276 u32 proto_ver;
277
278 int dbg_level;
279 u32 connect_time_stamp;
280 int connect_retries;
281#define SKD_MAX_CONNECT_RETRIES 16
282 u32 drive_jiffies;
283
284 u32 timo_slot;
285
ca33dd92 286 struct work_struct start_queue;
38d4a1bb 287 struct work_struct completion_worker;
e67f86b3
AB		 288};
289
290#define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
291#define SKD_READL(DEV, OFF) skd_reg_read32(DEV, OFF)
292#define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
293
294static inline u32 skd_reg_read32(struct skd_device *skdev, u32 offset)
295{
14262a4b 296 u32 val = readl(skdev->mem_map[1] + offset);
e67f86b3 297
14262a4b 298 if (unlikely(skdev->dbg_level >= 2))
f98806d6 299 dev_dbg(&skdev->pdev->dev, "offset %x = %x\n", offset, val);
14262a4b 300 return val;
e67f86b3
AB		 301}
302
303static inline void skd_reg_write32(struct skd_device *skdev, u32 val,
304 u32 offset)
305{
14262a4b
BVA		 306 writel(val, skdev->mem_map[1] + offset);
307 if (unlikely(skdev->dbg_level >= 2))
f98806d6 308 dev_dbg(&skdev->pdev->dev, "offset %x = %x\n", offset, val);
e67f86b3
AB		 309}
310
311static inline void skd_reg_write64(struct skd_device *skdev, u64 val,
312 u32 offset)
313{
14262a4b
BVA		 314 writeq(val, skdev->mem_map[1] + offset);
315 if (unlikely(skdev->dbg_level >= 2))
f98806d6
BVA		 316 dev_dbg(&skdev->pdev->dev, "offset %x = %016llx\n", offset,
317 val);
e67f86b3
AB		 318}
319
320
321#define SKD_IRQ_DEFAULT SKD_IRQ_MSI
322static int skd_isr_type = SKD_IRQ_DEFAULT;
323
324module_param(skd_isr_type, int, 0444);
325MODULE_PARM_DESC(skd_isr_type, "Interrupt type capability."
326 " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
327
328#define SKD_MAX_REQ_PER_MSG_DEFAULT 1
329static int skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
330
331module_param(skd_max_req_per_msg, int, 0444);
332MODULE_PARM_DESC(skd_max_req_per_msg,
333 "Maximum SCSI requests packed in a single message."
2da7b403 334 " (1-" __stringify(SKD_MAX_REQ_PER_MSG) ", default==1)");
e67f86b3
AB		 335
336#define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
337#define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
338static int skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
339
340module_param(skd_max_queue_depth, int, 0444);
341MODULE_PARM_DESC(skd_max_queue_depth,
342 "Maximum SCSI requests issued to s1120."
343 " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR ")");
344
345static int skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
346module_param(skd_sgs_per_request, int, 0444);
347MODULE_PARM_DESC(skd_sgs_per_request,
348 "Maximum SG elements per block request."
349 " (1-4096, default==256)");
350
63214121 351static int skd_max_pass_thru = 1;
e67f86b3
AB		 352module_param(skd_max_pass_thru, int, 0444);
353MODULE_PARM_DESC(skd_max_pass_thru,
63214121 354 "Maximum SCSI pass-thru at a time. IGNORED");
e67f86b3
AB		 355
356module_param(skd_dbg_level, int, 0444);
357MODULE_PARM_DESC(skd_dbg_level, "s1120 debug level (0,1,2)");
358
359module_param(skd_isr_comp_limit, int, 0444);
360MODULE_PARM_DESC(skd_isr_comp_limit, "s1120 isr comp limit (0=none) default=4");
361
e67f86b3
AB		 362/* Major device number dynamically assigned. */
363static u32 skd_major;
364
e67f86b3
AB		 365static void skd_destruct(struct skd_device *skdev);
366static const struct block_device_operations skd_blockdev_ops;
367static void skd_send_fitmsg(struct skd_device *skdev,
368 struct skd_fitmsg_context *skmsg);
369static void skd_send_special_fitmsg(struct skd_device *skdev,
370 struct skd_special_context *skspcl);
f18c17c8
BVA		 371static void skd_end_request(struct skd_device *skdev, struct request *req,
372 blk_status_t status);
2a842aca 373static bool skd_preop_sg_list(struct skd_device *skdev,
e67f86b3
AB		 374 struct skd_request_context *skreq);
375static void skd_postop_sg_list(struct skd_device *skdev,
376 struct skd_request_context *skreq);
377
378static void skd_restart_device(struct skd_device *skdev);
379static int skd_quiesce_dev(struct skd_device *skdev);
380static int skd_unquiesce_dev(struct skd_device *skdev);
e67f86b3
AB		 381static void skd_disable_interrupts(struct skd_device *skdev);
382static void skd_isr_fwstate(struct skd_device *skdev);
79ce12a8 383static void skd_recover_requests(struct skd_device *skdev);
e67f86b3
AB		 384static void skd_soft_reset(struct skd_device *skdev);
385
e67f86b3
AB		 386const char *skd_drive_state_to_str(int state);
387const char *skd_skdev_state_to_str(enum skd_drvr_state state);
388static void skd_log_skdev(struct skd_device *skdev, const char *event);
e67f86b3
AB		 389static void skd_log_skreq(struct skd_device *skdev,
390 struct skd_request_context *skreq, const char *event);
391
e67f86b3
AB		 392/*
393 *****************************************************************************
394 * READ/WRITE REQUESTS
395 *****************************************************************************
396 */
d4d0f5fc
BVA		 397static void skd_inc_in_flight(struct request *rq, void *data, bool reserved)
398{
399 int *count = data;
400
401 count++;
402}
403
404static int skd_in_flight(struct skd_device *skdev)
405{
406 int count = 0;
407
408 blk_mq_tagset_busy_iter(&skdev->tag_set, skd_inc_in_flight, &count);
409
410 return count;
411}
412
e67f86b3
AB		 413static void
414skd_prep_rw_cdb(struct skd_scsi_request *scsi_req,
415 int data_dir, unsigned lba,
416 unsigned count)
417{
418 if (data_dir == READ)
fb4844b8 419 scsi_req->cdb[0] = READ_10;
e67f86b3 420 else
fb4844b8 421 scsi_req->cdb[0] = WRITE_10;
e67f86b3
AB		 422
423 scsi_req->cdb[1] = 0;
424 scsi_req->cdb[2] = (lba & 0xff000000) >> 24;
425 scsi_req->cdb[3] = (lba & 0xff0000) >> 16;
426 scsi_req->cdb[4] = (lba & 0xff00) >> 8;
427 scsi_req->cdb[5] = (lba & 0xff);
428 scsi_req->cdb[6] = 0;
429 scsi_req->cdb[7] = (count & 0xff00) >> 8;
430 scsi_req->cdb[8] = count & 0xff;
431 scsi_req->cdb[9] = 0;
432}
433
434static void
435skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
38d4a1bb 436 struct skd_request_context *skreq)
e67f86b3
AB		 437{
438 skreq->flush_cmd = 1;
439
fb4844b8 440 scsi_req->cdb[0] = SYNCHRONIZE_CACHE;
e67f86b3
AB		 441 scsi_req->cdb[1] = 0;
442 scsi_req->cdb[2] = 0;
443 scsi_req->cdb[3] = 0;
444 scsi_req->cdb[4] = 0;
445 scsi_req->cdb[5] = 0;
446 scsi_req->cdb[6] = 0;
447 scsi_req->cdb[7] = 0;
448 scsi_req->cdb[8] = 0;
449 scsi_req->cdb[9] = 0;
450}
451
3d17a679
BVA		 452/*
453 * Return true if and only if all pending requests should be failed.
454 */
455static bool skd_fail_all(struct request_queue *q)
cb6981b9
BVA		 456{
457 struct skd_device *skdev = q->queuedata;
458
459 SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
460
461 skd_log_skdev(skdev, "req_not_online");
462 switch (skdev->state) {
463 case SKD_DRVR_STATE_PAUSING:
464 case SKD_DRVR_STATE_PAUSED:
465 case SKD_DRVR_STATE_STARTING:
466 case SKD_DRVR_STATE_RESTARTING:
467 case SKD_DRVR_STATE_WAIT_BOOT:
468 /* In case of starting, we haven't started the queue,
469 * so we can't get here... but requests are
470 * possibly hanging out waiting for us because we
471 * reported the dev/skd0 already. They'll wait
472 * forever if connect doesn't complete.
473 * What to do??? delay dev/skd0 ??
474 */
475 case SKD_DRVR_STATE_BUSY:
476 case SKD_DRVR_STATE_BUSY_IMMINENT:
477 case SKD_DRVR_STATE_BUSY_ERASE:
3d17a679 478 return false;
cb6981b9
BVA		 479
480 case SKD_DRVR_STATE_BUSY_SANITIZE:
481 case SKD_DRVR_STATE_STOPPING:
482 case SKD_DRVR_STATE_SYNCING:
483 case SKD_DRVR_STATE_FAULT:
484 case SKD_DRVR_STATE_DISAPPEARED:
485 default:
3d17a679 486 return true;
cb6981b9 487 }
cb6981b9 488}
e67f86b3 489
ca33dd92 490static void skd_process_request(struct request *req, bool last)
e67f86b3 491{
91f85da4 492 struct request_queue *const q = req->q;
e67f86b3 493 struct skd_device *skdev = q->queuedata;
91f85da4
BVA		 494 struct skd_fitmsg_context *skmsg;
495 struct fit_msg_hdr *fmh;
496 const u32 tag = blk_mq_unique_tag(req);
e7278a8b 497 struct skd_request_context *const skreq = blk_mq_rq_to_pdu(req);
e67f86b3 498 struct skd_scsi_request *scsi_req;
ca33dd92 499 unsigned long flags;
e67f86b3 500 unsigned long io_flags;
e67f86b3
AB		 501 u32 lba;
502 u32 count;
503 int data_dir;
4854afe3 504 __be64 be_dmaa;
e67f86b3 505 int flush, fua;
91f85da4
BVA		 506
507 WARN_ONCE(tag >= skd_max_queue_depth, "%#x > %#x (nr_requests = %lu)\n",
508 tag, skd_max_queue_depth, q->nr_requests);
509
510 SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
511
512 flush = fua = 0;
513
514 lba = (u32)blk_rq_pos(req);
515 count = blk_rq_sectors(req);
516 data_dir = rq_data_dir(req);
517 io_flags = req->cmd_flags;
518
519 if (req_op(req) == REQ_OP_FLUSH)
520 flush++;
521
522 if (io_flags & REQ_FUA)
523 fua++;
524
525 dev_dbg(&skdev->pdev->dev,
526 "new req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n", req, lba,
527 lba, count, count, data_dir);
528
529 skreq->id = tag + SKD_ID_RW_REQUEST;
530 skreq->flush_cmd = 0;
531 skreq->n_sg = 0;
532 skreq->sg_byte_count = 0;
533
91f85da4
BVA		 534 skreq->fitmsg_id = 0;
535
536 skreq->data_dir = data_dir == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
537
538 if (req->bio && !skd_preop_sg_list(skdev, skreq)) {
539 dev_dbg(&skdev->pdev->dev, "error Out\n");
e7278a8b
BVA		 540 skd_end_request(skdev, blk_mq_rq_from_pdu(skreq),
541 BLK_STS_RESOURCE);
91f85da4
BVA		 542 return;
543 }
544
a3db102d
BVA		 545 dma_sync_single_for_device(&skdev->pdev->dev, skreq->sksg_dma_address,
546 skreq->n_sg *
547 sizeof(struct fit_sg_descriptor),
548 DMA_TO_DEVICE);
549
ca33dd92 550 spin_lock_irqsave(&skdev->lock, flags);
91f85da4
BVA		 551 /* Either a FIT msg is in progress or we have to start one. */
552 skmsg = skdev->skmsg;
553 if (!skmsg) {
554 skmsg = &skdev->skmsg_table[tag];
555 skdev->skmsg = skmsg;
556
557 /* Initialize the FIT msg header */
558 fmh = &skmsg->msg_buf->fmh;
559 memset(fmh, 0, sizeof(*fmh));
560 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
561 skmsg->length = sizeof(*fmh);
562 } else {
563 fmh = &skmsg->msg_buf->fmh;
564 }
565
566 skreq->fitmsg_id = skmsg->id;
567
568 scsi_req = &skmsg->msg_buf->scsi[fmh->num_protocol_cmds_coalesced];
569 memset(scsi_req, 0, sizeof(*scsi_req));
570
571 be_dmaa = cpu_to_be64(skreq->sksg_dma_address);
572
573 scsi_req->hdr.tag = skreq->id;
574 scsi_req->hdr.sg_list_dma_address = be_dmaa;
575
576 if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
577 skd_prep_zerosize_flush_cdb(scsi_req, skreq);
578 SKD_ASSERT(skreq->flush_cmd == 1);
579 } else {
580 skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
581 }
582
583 if (fua)
584 scsi_req->cdb[1] |= SKD_FUA_NV;
585
586 scsi_req->hdr.sg_list_len_bytes = cpu_to_be32(skreq->sg_byte_count);
587
588 /* Complete resource allocations. */
589 skreq->state = SKD_REQ_STATE_BUSY;
590
591 skmsg->length += sizeof(struct skd_scsi_request);
592 fmh->num_protocol_cmds_coalesced++;
593
91f85da4 594 dev_dbg(&skdev->pdev->dev, "req=0x%x busy=%d\n", skreq->id,
d4d0f5fc 595 skd_in_flight(skdev));
91f85da4
BVA		 596
597 /*
598 * If the FIT msg buffer is full send it.
599 */
ca33dd92 600 if (last || fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
91f85da4
BVA		 601 skd_send_fitmsg(skdev, skmsg);
602 skdev->skmsg = NULL;
603 }
ca33dd92 604 spin_unlock_irqrestore(&skdev->lock, flags);
91f85da4
BVA		 605}
606
ca33dd92
BVA		 607static blk_status_t skd_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
608 const struct blk_mq_queue_data *mqd)
91f85da4 609{
ca33dd92
BVA		 610 struct request *req = mqd->rq;
611 struct request_queue *q = req->q;
91f85da4 612 struct skd_device *skdev = q->queuedata;
e67f86b3 613
ca33dd92
BVA		 614 if (skdev->state == SKD_DRVR_STATE_ONLINE) {
615 blk_mq_start_request(req);
616 skd_process_request(req, mqd->last);
91f85da4 617
ca33dd92
BVA		 618 return BLK_STS_OK;
619 } else {
620 return skd_fail_all(q) ? BLK_STS_IOERR : BLK_STS_RESOURCE;
e67f86b3
AB		 621 }
622
ca33dd92 623 return BLK_STS_OK;
e67f86b3
AB		 624}
625
a74d5b76
BVA		 626static enum blk_eh_timer_return skd_timed_out(struct request *req)
627{
628 struct skd_device *skdev = req->q->queuedata;
629
630 dev_err(&skdev->pdev->dev, "request with tag %#x timed out\n",
631 blk_mq_unique_tag(req));
632
633 return BLK_EH_HANDLED;
634}
635
f18c17c8
BVA		 636static void skd_end_request(struct skd_device *skdev, struct request *req,
637 blk_status_t error)
e67f86b3 638{
e67f86b3 639 if (unlikely(error)) {
e67f86b3
AB		 640 char *cmd = (rq_data_dir(req) == READ) ? "read" : "write";
641 u32 lba = (u32)blk_rq_pos(req);
642 u32 count = blk_rq_sectors(req);
643
f98806d6
BVA		 644 dev_err(&skdev->pdev->dev,
645 "Error cmd=%s sect=%u count=%u id=0x%x\n", cmd, lba,
f18c17c8 646 count, req->tag);
e67f86b3 647 } else
f18c17c8 648 dev_dbg(&skdev->pdev->dev, "id=0x%x error=%d\n", req->tag,
f98806d6 649 error);
e67f86b3 650
ca33dd92 651 blk_mq_end_request(req, error);
e67f86b3
AB		 652}
653
a74d5b76
BVA		 654/* Only called in case of a request timeout */
655static void skd_softirq_done(struct request *req)
656{
657 struct skd_device *skdev = req->q->queuedata;
658 struct skd_request_context *skreq = blk_mq_rq_to_pdu(req);
659 unsigned long flags;
660
661 spin_lock_irqsave(&skdev->lock, flags);
662 skd_end_request(skdev, blk_mq_rq_from_pdu(skreq), BLK_STS_TIMEOUT);
663 spin_unlock_irqrestore(&skdev->lock, flags);
664}
665
2a842aca 666static bool skd_preop_sg_list(struct skd_device *skdev,
38d4a1bb 667 struct skd_request_context *skreq)
e67f86b3 668{
e7278a8b 669 struct request *req = blk_mq_rq_from_pdu(skreq);
06f824c4 670 struct scatterlist *sgl = &skreq->sg[0], *sg;
e67f86b3
AB		 671 int n_sg;
672 int i;
673
674 skreq->sg_byte_count = 0;
675
b1824eef
BVA		 676 WARN_ON_ONCE(skreq->data_dir != DMA_TO_DEVICE &&
677 skreq->data_dir != DMA_FROM_DEVICE);
e67f86b3 678
06f824c4 679 n_sg = blk_rq_map_sg(skdev->queue, req, sgl);
e67f86b3 680 if (n_sg <= 0)
2a842aca 681 return false;
e67f86b3
AB		 682
683 /*
684 * Map scatterlist to PCI bus addresses.
685 * Note PCI might change the number of entries.
686 */
06f824c4 687 n_sg = pci_map_sg(skdev->pdev, sgl, n_sg, skreq->data_dir);
e67f86b3 688 if (n_sg <= 0)
2a842aca 689 return false;
e67f86b3
AB		 690
691 SKD_ASSERT(n_sg <= skdev->sgs_per_request);
692
693 skreq->n_sg = n_sg;
694
06f824c4 695 for_each_sg(sgl, sg, n_sg, i) {
e67f86b3 696 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
06f824c4
BVA		 697 u32 cnt = sg_dma_len(sg);
698 uint64_t dma_addr = sg_dma_address(sg);
e67f86b3
AB		 699
700 sgd->control = FIT_SGD_CONTROL_NOT_LAST;
701 sgd->byte_count = cnt;
702 skreq->sg_byte_count += cnt;
703 sgd->host_side_addr = dma_addr;
704 sgd->dev_side_addr = 0;
705 }
706
707 skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;
708 skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
709
710 if (unlikely(skdev->dbg_level > 1)) {
f98806d6
BVA		 711 dev_dbg(&skdev->pdev->dev,
712 "skreq=%x sksg_list=%p sksg_dma=%llx\n",
713 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 714 for (i = 0; i < n_sg; i++) {
715 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
f98806d6
BVA		 716
717 dev_dbg(&skdev->pdev->dev,
718 " sg[%d] count=%u ctrl=0x%x addr=0x%llx next=0x%llx\n",
719 i, sgd->byte_count, sgd->control,
720 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 721 }
722 }
723
2a842aca 724 return true;
e67f86b3
AB		 725}
726
fcd37eb3 727static void skd_postop_sg_list(struct skd_device *skdev,
38d4a1bb 728 struct skd_request_context *skreq)
e67f86b3 729{
e67f86b3
AB		 730 /*
731 * restore the next ptr for next IO request so we
732 * don't have to set it every time.
733 */
734 skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
735 skreq->sksg_dma_address +
736 ((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
b1824eef 737 pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, skreq->data_dir);
e67f86b3
AB		 738}
739
e67f86b3
AB		 740/*
741 *****************************************************************************
742 * TIMER
743 *****************************************************************************
744 */
745
746static void skd_timer_tick_not_online(struct skd_device *skdev);
747
ca33dd92
BVA		 748static void skd_start_queue(struct work_struct *work)
749{
750 struct skd_device *skdev = container_of(work, typeof(*skdev),
751 start_queue);
752
753 /*
754 * Although it is safe to call blk_start_queue() from interrupt
755 * context, blk_mq_start_hw_queues() must not be called from
756 * interrupt context.
757 */
758 blk_mq_start_hw_queues(skdev->queue);
759}
760
e67f86b3
AB		 761static void skd_timer_tick(ulong arg)
762{
763 struct skd_device *skdev = (struct skd_device *)arg;
e67f86b3
AB		 764 unsigned long reqflags;
765 u32 state;
766
767 if (skdev->state == SKD_DRVR_STATE_FAULT)
768 /* The driver has declared fault, and we want it to
769 * stay that way until driver is reloaded.
770 */
771 return;
772
773 spin_lock_irqsave(&skdev->lock, reqflags);
774
775 state = SKD_READL(skdev, FIT_STATUS);
776 state &= FIT_SR_DRIVE_STATE_MASK;
777 if (state != skdev->drive_state)
778 skd_isr_fwstate(skdev);
779
a74d5b76 780 if (skdev->state != SKD_DRVR_STATE_ONLINE)
e67f86b3 781 skd_timer_tick_not_online(skdev);
e67f86b3 782
e67f86b3
AB		 783 mod_timer(&skdev->timer, (jiffies + HZ));
784
785 spin_unlock_irqrestore(&skdev->lock, reqflags);
786}
787
788static void skd_timer_tick_not_online(struct skd_device *skdev)
789{
790 switch (skdev->state) {
791 case SKD_DRVR_STATE_IDLE:
792 case SKD_DRVR_STATE_LOAD:
793 break;
794 case SKD_DRVR_STATE_BUSY_SANITIZE:
f98806d6
BVA		 795 dev_dbg(&skdev->pdev->dev,
796 "drive busy sanitize[%x], driver[%x]\n",
797 skdev->drive_state, skdev->state);
e67f86b3
AB		 798 /* If we've been in sanitize for 3 seconds, we figure we're not
799 * going to get anymore completions, so recover requests now
800 */
801 if (skdev->timer_countdown > 0) {
802 skdev->timer_countdown--;
803 return;
804 }
79ce12a8 805 skd_recover_requests(skdev);
e67f86b3
AB		 806 break;
807
808 case SKD_DRVR_STATE_BUSY:
809 case SKD_DRVR_STATE_BUSY_IMMINENT:
810 case SKD_DRVR_STATE_BUSY_ERASE:
f98806d6
BVA		 811 dev_dbg(&skdev->pdev->dev, "busy[%x], countdown=%d\n",
812 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 813 if (skdev->timer_countdown > 0) {
814 skdev->timer_countdown--;
815 return;
816 }
f98806d6
BVA		 817 dev_dbg(&skdev->pdev->dev,
818 "busy[%x], timedout=%d, restarting device.",
819 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 820 skd_restart_device(skdev);
821 break;
822
823 case SKD_DRVR_STATE_WAIT_BOOT:
824 case SKD_DRVR_STATE_STARTING:
825 if (skdev->timer_countdown > 0) {
826 skdev->timer_countdown--;
827 return;
828 }
829 /* For now, we fault the drive. Could attempt resets to
830 * revcover at some point. */
831 skdev->state = SKD_DRVR_STATE_FAULT;
832
f98806d6
BVA		 833 dev_err(&skdev->pdev->dev, "DriveFault Connect Timeout (%x)\n",
834 skdev->drive_state);
e67f86b3
AB		 835
836 /*start the queue so we can respond with error to requests */
837 /* wakeup anyone waiting for startup complete */
ca33dd92 838 schedule_work(&skdev->start_queue);
e67f86b3
AB		 839 skdev->gendisk_on = -1;
840 wake_up_interruptible(&skdev->waitq);
841 break;
842
843 case SKD_DRVR_STATE_ONLINE:
844 /* shouldn't get here. */
845 break;
846
847 case SKD_DRVR_STATE_PAUSING:
848 case SKD_DRVR_STATE_PAUSED:
849 break;
850
e67f86b3
AB		 851 case SKD_DRVR_STATE_RESTARTING:
852 if (skdev->timer_countdown > 0) {
853 skdev->timer_countdown--;
854 return;
855 }
856 /* For now, we fault the drive. Could attempt resets to
857 * revcover at some point. */
858 skdev->state = SKD_DRVR_STATE_FAULT;
f98806d6
BVA		 859 dev_err(&skdev->pdev->dev,
860 "DriveFault Reconnect Timeout (%x)\n",
861 skdev->drive_state);
e67f86b3
AB		 862
863 /*
864 * Recovering does two things:
865 * 1. completes IO with error
866 * 2. reclaims dma resources
867 * When is it safe to recover requests?
868 * - if the drive state is faulted
869 * - if the state is still soft reset after out timeout
870 * - if the drive registers are dead (state = FF)
871 * If it is "unsafe", we still need to recover, so we will
872 * disable pci bus mastering and disable our interrupts.
873 */
874
875 if ((skdev->drive_state == FIT_SR_DRIVE_SOFT_RESET) ||
876 (skdev->drive_state == FIT_SR_DRIVE_FAULT) ||
877 (skdev->drive_state == FIT_SR_DRIVE_STATE_MASK))
878 /* It never came out of soft reset. Try to
879 * recover the requests and then let them
880 * fail. This is to mitigate hung processes. */
79ce12a8 881 skd_recover_requests(skdev);
e67f86b3 882 else {
f98806d6
BVA		 883 dev_err(&skdev->pdev->dev, "Disable BusMaster (%x)\n",
884 skdev->drive_state);
e67f86b3
AB		 885 pci_disable_device(skdev->pdev);
886 skd_disable_interrupts(skdev);
79ce12a8 887 skd_recover_requests(skdev);
e67f86b3
AB		 888 }
889
890 /*start the queue so we can respond with error to requests */
891 /* wakeup anyone waiting for startup complete */
ca33dd92 892 schedule_work(&skdev->start_queue);
e67f86b3
AB		 893 skdev->gendisk_on = -1;
894 wake_up_interruptible(&skdev->waitq);
895 break;
896
897 case SKD_DRVR_STATE_RESUMING:
898 case SKD_DRVR_STATE_STOPPING:
899 case SKD_DRVR_STATE_SYNCING:
900 case SKD_DRVR_STATE_FAULT:
901 case SKD_DRVR_STATE_DISAPPEARED:
902 default:
903 break;
904 }
905}
906
907static int skd_start_timer(struct skd_device *skdev)
908{
909 int rc;
910
e67f86b3
AB		 911 setup_timer(&skdev->timer, skd_timer_tick, (ulong)skdev);
912
913 rc = mod_timer(&skdev->timer, (jiffies + HZ));
914 if (rc)
f98806d6 915 dev_err(&skdev->pdev->dev, "failed to start timer %d\n", rc);
e67f86b3
AB		 916 return rc;
917}
918
919static void skd_kill_timer(struct skd_device *skdev)
920{
921 del_timer_sync(&skdev->timer);
922}
923
e67f86b3
AB		 924/*
925 *****************************************************************************
926 * INTERNAL REQUESTS -- generated by driver itself
927 *****************************************************************************
928 */
929
930static int skd_format_internal_skspcl(struct skd_device *skdev)
931{
932 struct skd_special_context *skspcl = &skdev->internal_skspcl;
933 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
934 struct fit_msg_hdr *fmh;
935 uint64_t dma_address;
936 struct skd_scsi_request *scsi;
937
d891fe60 938 fmh = &skspcl->msg_buf->fmh;
e67f86b3
AB		 939 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
940 fmh->num_protocol_cmds_coalesced = 1;
941
d891fe60 942 scsi = &skspcl->msg_buf->scsi[0];
e67f86b3
AB		 943 memset(scsi, 0, sizeof(*scsi));
944 dma_address = skspcl->req.sksg_dma_address;
945 scsi->hdr.sg_list_dma_address = cpu_to_be64(dma_address);
32494df9 946 skspcl->req.n_sg = 1;
e67f86b3
AB		 947 sgd->control = FIT_SGD_CONTROL_LAST;
948 sgd->byte_count = 0;
949 sgd->host_side_addr = skspcl->db_dma_address;
950 sgd->dev_side_addr = 0;
951 sgd->next_desc_ptr = 0LL;
952
953 return 1;
954}
955
956#define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
957
958static void skd_send_internal_skspcl(struct skd_device *skdev,
959 struct skd_special_context *skspcl,
960 u8 opcode)
961{
962 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
963 struct skd_scsi_request *scsi;
964 unsigned char *buf = skspcl->data_buf;
965 int i;
966
967 if (skspcl->req.state != SKD_REQ_STATE_IDLE)
968 /*
969 * A refresh is already in progress.
970 * Just wait for it to finish.
971 */
972 return;
973
974 SKD_ASSERT((skspcl->req.id & SKD_ID_INCR) == 0);
975 skspcl->req.state = SKD_REQ_STATE_BUSY;
976 skspcl->req.id += SKD_ID_INCR;
977
d891fe60 978 scsi = &skspcl->msg_buf->scsi[0];
e67f86b3
AB		 979 scsi->hdr.tag = skspcl->req.id;
980
981 memset(scsi->cdb, 0, sizeof(scsi->cdb));
982
983 switch (opcode) {
984 case TEST_UNIT_READY:
985 scsi->cdb[0] = TEST_UNIT_READY;
986 sgd->byte_count = 0;
987 scsi->hdr.sg_list_len_bytes = 0;
988 break;
989
990 case READ_CAPACITY:
991 scsi->cdb[0] = READ_CAPACITY;
992 sgd->byte_count = SKD_N_READ_CAP_BYTES;
993 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
994 break;
995
996 case INQUIRY:
997 scsi->cdb[0] = INQUIRY;
998 scsi->cdb[1] = 0x01; /* evpd */
999 scsi->cdb[2] = 0x80; /* serial number page */
1000 scsi->cdb[4] = 0x10;
1001 sgd->byte_count = 16;
1002 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1003 break;
1004
1005 case SYNCHRONIZE_CACHE:
1006 scsi->cdb[0] = SYNCHRONIZE_CACHE;
1007 sgd->byte_count = 0;
1008 scsi->hdr.sg_list_len_bytes = 0;
1009 break;
1010
1011 case WRITE_BUFFER:
1012 scsi->cdb[0] = WRITE_BUFFER;
1013 scsi->cdb[1] = 0x02;
1014 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1015 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1016 sgd->byte_count = WR_BUF_SIZE;
1017 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1018 /* fill incrementing byte pattern */
1019 for (i = 0; i < sgd->byte_count; i++)
1020 buf[i] = i & 0xFF;
1021 break;
1022
1023 case READ_BUFFER:
1024 scsi->cdb[0] = READ_BUFFER;
1025 scsi->cdb[1] = 0x02;
1026 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1027 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1028 sgd->byte_count = WR_BUF_SIZE;
1029 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1030 memset(skspcl->data_buf, 0, sgd->byte_count);
1031 break;
1032
1033 default:
1034 SKD_ASSERT("Don't know what to send");
1035 return;
1036
1037 }
1038 skd_send_special_fitmsg(skdev, skspcl);
1039}
1040
1041static void skd_refresh_device_data(struct skd_device *skdev)
1042{
1043 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1044
1045 skd_send_internal_skspcl(skdev, skspcl, TEST_UNIT_READY);
1046}
1047
1048static int skd_chk_read_buf(struct skd_device *skdev,
1049 struct skd_special_context *skspcl)
1050{
1051 unsigned char *buf = skspcl->data_buf;
1052 int i;
1053
1054 /* check for incrementing byte pattern */
1055 for (i = 0; i < WR_BUF_SIZE; i++)
1056 if (buf[i] != (i & 0xFF))
1057 return 1;
1058
1059 return 0;
1060}
1061
1062static void skd_log_check_status(struct skd_device *skdev, u8 status, u8 key,
1063 u8 code, u8 qual, u8 fruc)
1064{
1065 /* If the check condition is of special interest, log a message */
1066 if ((status == SAM_STAT_CHECK_CONDITION) && (key == 0x02)
1067 && (code == 0x04) && (qual == 0x06)) {
f98806d6
BVA		 1068 dev_err(&skdev->pdev->dev,
1069 "*** LOST_WRITE_DATA ERROR *** key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
1070 key, code, qual, fruc);
e67f86b3
AB		 1071 }
1072}
1073
1074static void skd_complete_internal(struct skd_device *skdev,
85e34112
BVA		 1075 struct fit_completion_entry_v1 *skcomp,
1076 struct fit_comp_error_info *skerr,
e67f86b3
AB		 1077 struct skd_special_context *skspcl)
1078{
1079 u8 *buf = skspcl->data_buf;
1080 u8 status;
1081 int i;
d891fe60 1082 struct skd_scsi_request *scsi = &skspcl->msg_buf->scsi[0];
e67f86b3 1083
760b48ca
BVA		 1084 lockdep_assert_held(&skdev->lock);
1085
e67f86b3
AB		 1086 SKD_ASSERT(skspcl == &skdev->internal_skspcl);
1087
f98806d6 1088 dev_dbg(&skdev->pdev->dev, "complete internal %x\n", scsi->cdb[0]);
e67f86b3 1089
a3db102d
BVA		 1090 dma_sync_single_for_cpu(&skdev->pdev->dev,
1091 skspcl->db_dma_address,
1092 skspcl->req.sksg_list[0].byte_count,
1093 DMA_BIDIRECTIONAL);
1094
e67f86b3
AB		 1095 skspcl->req.completion = *skcomp;
1096 skspcl->req.state = SKD_REQ_STATE_IDLE;
1097 skspcl->req.id += SKD_ID_INCR;
1098
1099 status = skspcl->req.completion.status;
1100
1101 skd_log_check_status(skdev, status, skerr->key, skerr->code,
1102 skerr->qual, skerr->fruc);
1103
1104 switch (scsi->cdb[0]) {
1105 case TEST_UNIT_READY:
1106 if (status == SAM_STAT_GOOD)
1107 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
1108 else if ((status == SAM_STAT_CHECK_CONDITION) &&
1109 (skerr->key == MEDIUM_ERROR))
1110 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
1111 else {
1112 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
f98806d6
BVA		 1113 dev_dbg(&skdev->pdev->dev,
1114 "TUR failed, don't send anymore state 0x%x\n",
1115 skdev->state);
e67f86b3
AB		 1116 return;
1117 }
f98806d6
BVA		 1118 dev_dbg(&skdev->pdev->dev,
1119 "**** TUR failed, retry skerr\n");
fb4844b8
BVA		 1120 skd_send_internal_skspcl(skdev, skspcl,
1121 TEST_UNIT_READY);
e67f86b3
AB		 1122 }
1123 break;
1124
1125 case WRITE_BUFFER:
1126 if (status == SAM_STAT_GOOD)
1127 skd_send_internal_skspcl(skdev, skspcl, READ_BUFFER);
1128 else {
1129 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
f98806d6
BVA		 1130 dev_dbg(&skdev->pdev->dev,
1131 "write buffer failed, don't send anymore state 0x%x\n",
1132 skdev->state);
e67f86b3
AB		 1133 return;
1134 }
f98806d6
BVA		 1135 dev_dbg(&skdev->pdev->dev,
1136 "**** write buffer failed, retry skerr\n");
fb4844b8
BVA		 1137 skd_send_internal_skspcl(skdev, skspcl,
1138 TEST_UNIT_READY);
e67f86b3
AB		 1139 }
1140 break;
1141
1142 case READ_BUFFER:
1143 if (status == SAM_STAT_GOOD) {
1144 if (skd_chk_read_buf(skdev, skspcl) == 0)
1145 skd_send_internal_skspcl(skdev, skspcl,
1146 READ_CAPACITY);
1147 else {
f98806d6
BVA		 1148 dev_err(&skdev->pdev->dev,
1149 "*** W/R Buffer mismatch %d ***\n",
1150 skdev->connect_retries);
e67f86b3
AB		 1151 if (skdev->connect_retries <
1152 SKD_MAX_CONNECT_RETRIES) {
1153 skdev->connect_retries++;
1154 skd_soft_reset(skdev);
1155 } else {
f98806d6
BVA		 1156 dev_err(&skdev->pdev->dev,
1157 "W/R Buffer Connect Error\n");
e67f86b3
AB		 1158 return;
1159 }
1160 }
1161
1162 } else {
1163 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
f98806d6
BVA		 1164 dev_dbg(&skdev->pdev->dev,
1165 "read buffer failed, don't send anymore state 0x%x\n",
1166 skdev->state);
e67f86b3
AB		 1167 return;
1168 }
f98806d6
BVA		 1169 dev_dbg(&skdev->pdev->dev,
1170 "**** read buffer failed, retry skerr\n");
fb4844b8
BVA		 1171 skd_send_internal_skspcl(skdev, skspcl,
1172 TEST_UNIT_READY);
e67f86b3
AB		 1173 }
1174 break;
1175
1176 case READ_CAPACITY:
1177 skdev->read_cap_is_valid = 0;
1178 if (status == SAM_STAT_GOOD) {
1179 skdev->read_cap_last_lba =
1180 (buf[0] << 24) | (buf[1] << 16) |
1181 (buf[2] << 8) | buf[3];
1182 skdev->read_cap_blocksize =
1183 (buf[4] << 24) | (buf[5] << 16) |
1184 (buf[6] << 8) | buf[7];
1185
f98806d6
BVA		 1186 dev_dbg(&skdev->pdev->dev, "last lba %d, bs %d\n",
1187 skdev->read_cap_last_lba,
1188 skdev->read_cap_blocksize);
e67f86b3
AB		 1189
1190 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
1191
1192 skdev->read_cap_is_valid = 1;
1193
1194 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
1195 } else if ((status == SAM_STAT_CHECK_CONDITION) &&
1196 (skerr->key == MEDIUM_ERROR)) {
1197 skdev->read_cap_last_lba = ~0;
1198 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
f98806d6 1199 dev_dbg(&skdev->pdev->dev, "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n");
e67f86b3
AB		 1200 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
1201 } else {
f98806d6 1202 dev_dbg(&skdev->pdev->dev, "**** READCAP failed, retry TUR\n");
e67f86b3
AB		 1203 skd_send_internal_skspcl(skdev, skspcl,
1204 TEST_UNIT_READY);
1205 }
1206 break;
1207
1208 case INQUIRY:
1209 skdev->inquiry_is_valid = 0;
1210 if (status == SAM_STAT_GOOD) {
1211 skdev->inquiry_is_valid = 1;
1212
1213 for (i = 0; i < 12; i++)
1214 skdev->inq_serial_num[i] = buf[i + 4];
1215 skdev->inq_serial_num[12] = 0;
1216 }
1217
1218 if (skd_unquiesce_dev(skdev) < 0)
f98806d6 1219 dev_dbg(&skdev->pdev->dev, "**** failed, to ONLINE device\n");
e67f86b3
AB		 1220 /* connection is complete */
1221 skdev->connect_retries = 0;
1222 break;
1223
1224 case SYNCHRONIZE_CACHE:
1225 if (status == SAM_STAT_GOOD)
1226 skdev->sync_done = 1;
1227 else
1228 skdev->sync_done = -1;
1229 wake_up_interruptible(&skdev->waitq);
1230 break;
1231
1232 default:
1233 SKD_ASSERT("we didn't send this");
1234 }
1235}
1236
1237/*
1238 *****************************************************************************
1239 * FIT MESSAGES
1240 *****************************************************************************
1241 */
1242
1243static void skd_send_fitmsg(struct skd_device *skdev,
1244 struct skd_fitmsg_context *skmsg)
1245{
1246 u64 qcmd;
e67f86b3 1247
f98806d6 1248 dev_dbg(&skdev->pdev->dev, "dma address 0x%llx, busy=%d\n",
d4d0f5fc 1249 skmsg->mb_dma_address, skd_in_flight(skdev));
6507f436 1250 dev_dbg(&skdev->pdev->dev, "msg_buf %p\n", skmsg->msg_buf);
e67f86b3
AB		 1251
1252 qcmd = skmsg->mb_dma_address;
1253 qcmd |= FIT_QCMD_QID_NORMAL;
1254
e67f86b3
AB		 1255 if (unlikely(skdev->dbg_level > 1)) {
1256 u8 *bp = (u8 *)skmsg->msg_buf;
1257 int i;
1258 for (i = 0; i < skmsg->length; i += 8) {
f98806d6
BVA		 1259 dev_dbg(&skdev->pdev->dev, "msg[%2d] %8ph\n", i,
1260 &bp[i]);
e67f86b3
AB		 1261 if (i == 0)
1262 i = 64 - 8;
1263 }
1264 }
1265
1266 if (skmsg->length > 256)
1267 qcmd |= FIT_QCMD_MSGSIZE_512;
1268 else if (skmsg->length > 128)
1269 qcmd |= FIT_QCMD_MSGSIZE_256;
1270 else if (skmsg->length > 64)
1271 qcmd |= FIT_QCMD_MSGSIZE_128;
1272 else
1273 /*
1274 * This makes no sense because the FIT msg header is
1275 * 64 bytes. If the msg is only 64 bytes long it has
1276 * no payload.
1277 */
1278 qcmd |= FIT_QCMD_MSGSIZE_64;
1279
a3db102d
BVA		 1280 dma_sync_single_for_device(&skdev->pdev->dev, skmsg->mb_dma_address,
1281 skmsg->length, DMA_TO_DEVICE);
1282
5fbd545c
BVA		 1283 /* Make sure skd_msg_buf is written before the doorbell is triggered. */
1284 smp_wmb();
1285
e67f86b3 1286 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
e67f86b3
AB		 1287}
1288
1289static void skd_send_special_fitmsg(struct skd_device *skdev,
1290 struct skd_special_context *skspcl)
1291{
1292 u64 qcmd;
1293
a3db102d
BVA		 1294 WARN_ON_ONCE(skspcl->req.n_sg != 1);
1295
e67f86b3
AB		 1296 if (unlikely(skdev->dbg_level > 1)) {
1297 u8 *bp = (u8 *)skspcl->msg_buf;
1298 int i;
1299
1300 for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
f98806d6
BVA		 1301 dev_dbg(&skdev->pdev->dev, " spcl[%2d] %8ph\n", i,
1302 &bp[i]);
e67f86b3
AB		 1303 if (i == 0)
1304 i = 64 - 8;
1305 }
1306
f98806d6
BVA		 1307 dev_dbg(&skdev->pdev->dev,
1308 "skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
1309 skspcl, skspcl->req.id, skspcl->req.sksg_list,
1310 skspcl->req.sksg_dma_address);
e67f86b3
AB		 1311 for (i = 0; i < skspcl->req.n_sg; i++) {
1312 struct fit_sg_descriptor *sgd =
1313 &skspcl->req.sksg_list[i];
1314
f98806d6
BVA		 1315 dev_dbg(&skdev->pdev->dev,
1316 " sg[%d] count=%u ctrl=0x%x addr=0x%llx next=0x%llx\n",
1317 i, sgd->byte_count, sgd->control,
1318 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 1319 }
1320 }
1321
1322 /*
1323 * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
1324 * and one 64-byte SSDI command.
1325 */
1326 qcmd = skspcl->mb_dma_address;
1327 qcmd |= FIT_QCMD_QID_NORMAL + FIT_QCMD_MSGSIZE_128;
1328
a3db102d
BVA		 1329 dma_sync_single_for_device(&skdev->pdev->dev, skspcl->mb_dma_address,
1330 SKD_N_SPECIAL_FITMSG_BYTES, DMA_TO_DEVICE);
1331 dma_sync_single_for_device(&skdev->pdev->dev,
1332 skspcl->req.sksg_dma_address,
1333 1 * sizeof(struct fit_sg_descriptor),
1334 DMA_TO_DEVICE);
1335 dma_sync_single_for_device(&skdev->pdev->dev,
1336 skspcl->db_dma_address,
1337 skspcl->req.sksg_list[0].byte_count,
1338 DMA_BIDIRECTIONAL);
1339
5fbd545c
BVA		 1340 /* Make sure skd_msg_buf is written before the doorbell is triggered. */
1341 smp_wmb();
1342
e67f86b3
AB		 1343 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
1344}
1345
1346/*
1347 *****************************************************************************
1348 * COMPLETION QUEUE
1349 *****************************************************************************
1350 */
1351
1352static void skd_complete_other(struct skd_device *skdev,
85e34112
BVA		 1353 struct fit_completion_entry_v1 *skcomp,
1354 struct fit_comp_error_info *skerr);
e67f86b3 1355
e67f86b3
AB		 1356struct sns_info {
1357 u8 type;
1358 u8 stat;
1359 u8 key;
1360 u8 asc;
1361 u8 ascq;
1362 u8 mask;
1363 enum skd_check_status_action action;
1364};
1365
1366static struct sns_info skd_chkstat_table[] = {
1367 /* Good */
1368 { 0x70, 0x02, RECOVERED_ERROR, 0, 0, 0x1c,
1369 SKD_CHECK_STATUS_REPORT_GOOD },
1370
1371 /* Smart alerts */
1372 { 0x70, 0x02, NO_SENSE, 0x0B, 0x00, 0x1E, /* warnings */
1373 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
1374 { 0x70, 0x02, NO_SENSE, 0x5D, 0x00, 0x1E, /* thresholds */
1375 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
1376 { 0x70, 0x02, RECOVERED_ERROR, 0x0B, 0x01, 0x1F, /* temperature over trigger */
1377 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
1378
1379 /* Retry (with limits) */
1380 { 0x70, 0x02, 0x0B, 0, 0, 0x1C, /* This one is for DMA ERROR */
1381 SKD_CHECK_STATUS_REQUEUE_REQUEST },
1382 { 0x70, 0x02, 0x06, 0x0B, 0x00, 0x1E, /* warnings */
1383 SKD_CHECK_STATUS_REQUEUE_REQUEST },
1384 { 0x70, 0x02, 0x06, 0x5D, 0x00, 0x1E, /* thresholds */
1385 SKD_CHECK_STATUS_REQUEUE_REQUEST },
1386 { 0x70, 0x02, 0x06, 0x80, 0x30, 0x1F, /* backup power */
1387 SKD_CHECK_STATUS_REQUEUE_REQUEST },
1388
1389 /* Busy (or about to be) */
1390 { 0x70, 0x02, 0x06, 0x3f, 0x01, 0x1F, /* fw changed */
1391 SKD_CHECK_STATUS_BUSY_IMMINENT },
1392};
1393
1394/*
1395 * Look up status and sense data to decide how to handle the error
1396 * from the device.
1397 * mask says which fields must match e.g., mask=0x18 means check
1398 * type and stat, ignore key, asc, ascq.
1399 */
1400
38d4a1bb
MS		 1401static enum skd_check_status_action
1402skd_check_status(struct skd_device *skdev,
85e34112 1403 u8 cmp_status, struct fit_comp_error_info *skerr)
e67f86b3 1404{
0b2e0c07 1405 int i;
e67f86b3 1406
f98806d6
BVA		 1407 dev_err(&skdev->pdev->dev, "key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
1408 skerr->key, skerr->code, skerr->qual, skerr->fruc);
e67f86b3 1409
f98806d6
BVA		 1410 dev_dbg(&skdev->pdev->dev,
1411 "stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
1412 skerr->type, cmp_status, skerr->key, skerr->code, skerr->qual,
1413 skerr->fruc);
e67f86b3
AB		 1414
1415 /* Does the info match an entry in the good category? */
0b2e0c07 1416 for (i = 0; i < ARRAY_SIZE(skd_chkstat_table); i++) {
e67f86b3
AB		 1417 struct sns_info *sns = &skd_chkstat_table[i];
1418
1419 if (sns->mask & 0x10)
1420 if (skerr->type != sns->type)
1421 continue;
1422
1423 if (sns->mask & 0x08)
1424 if (cmp_status != sns->stat)
1425 continue;
1426
1427 if (sns->mask & 0x04)
1428 if (skerr->key != sns->key)
1429 continue;
1430
1431 if (sns->mask & 0x02)
1432 if (skerr->code != sns->asc)
1433 continue;
1434
1435 if (sns->mask & 0x01)
1436 if (skerr->qual != sns->ascq)
1437 continue;
1438
1439 if (sns->action == SKD_CHECK_STATUS_REPORT_SMART_ALERT) {
f98806d6
BVA		 1440 dev_err(&skdev->pdev->dev,
1441 "SMART Alert: sense key/asc/ascq %02x/%02x/%02x\n",
1442 skerr->key, skerr->code, skerr->qual);
e67f86b3
AB		 1443 }
1444 return sns->action;
1445 }
1446
1447 /* No other match, so nonzero status means error,
1448 * zero status means good
1449 */
1450 if (cmp_status) {
f98806d6 1451 dev_dbg(&skdev->pdev->dev, "status check: error\n");
e67f86b3
AB		 1452 return SKD_CHECK_STATUS_REPORT_ERROR;
1453 }
1454
f98806d6 1455 dev_dbg(&skdev->pdev->dev, "status check good default\n");
e67f86b3
AB		 1456 return SKD_CHECK_STATUS_REPORT_GOOD;
1457}
1458
1459static void skd_resolve_req_exception(struct skd_device *skdev,
f18c17c8
BVA		 1460 struct skd_request_context *skreq,
1461 struct request *req)
e67f86b3
AB		 1462{
1463 u8 cmp_status = skreq->completion.status;
1464
1465 switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {
1466 case SKD_CHECK_STATUS_REPORT_GOOD:
1467 case SKD_CHECK_STATUS_REPORT_SMART_ALERT:
f18c17c8 1468 skd_end_request(skdev, req, BLK_STS_OK);
e67f86b3
AB		 1469 break;
1470
1471 case SKD_CHECK_STATUS_BUSY_IMMINENT:
1472 skd_log_skreq(skdev, skreq, "retry(busy)");
f18c17c8 1473 blk_requeue_request(skdev->queue, req);
f98806d6 1474 dev_info(&skdev->pdev->dev, "drive BUSY imminent\n");
e67f86b3
AB		 1475 skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT;
1476 skdev->timer_countdown = SKD_TIMER_MINUTES(20);
1477 skd_quiesce_dev(skdev);
1478 break;
1479
1480 case SKD_CHECK_STATUS_REQUEUE_REQUEST:
f18c17c8 1481 if ((unsigned long) ++req->special < SKD_MAX_RETRIES) {
fcd37eb3 1482 skd_log_skreq(skdev, skreq, "retry");
f18c17c8 1483 blk_requeue_request(skdev->queue, req);
fcd37eb3 1484 break;
e67f86b3 1485 }
ce6882ba 1486 /* fall through */
e67f86b3
AB		 1487
1488 case SKD_CHECK_STATUS_REPORT_ERROR:
1489 default:
f18c17c8 1490 skd_end_request(skdev, req, BLK_STS_IOERR);
e67f86b3
AB		 1491 break;
1492 }
1493}
1494
e67f86b3
AB		 1495static void skd_release_skreq(struct skd_device *skdev,
1496 struct skd_request_context *skreq)
1497{
e67f86b3
AB		 1498 /*
1499 * Reclaim the skd_request_context
1500 */
1501 skreq->state = SKD_REQ_STATE_IDLE;
1502 skreq->id += SKD_ID_INCR;
f18c17c8
BVA		 1503}
1504
e67f86b3
AB		 1505static int skd_isr_completion_posted(struct skd_device *skdev,
1506 int limit, int *enqueued)
1507{
85e34112
BVA		 1508 struct fit_completion_entry_v1 *skcmp;
1509 struct fit_comp_error_info *skerr;
e67f86b3 1510 u16 req_id;
f18c17c8 1511 u32 tag;
ca33dd92 1512 u16 hwq = 0;
f18c17c8 1513 struct request *rq;
e67f86b3 1514 struct skd_request_context *skreq;
c830da8c
BVA		 1515 u16 cmp_cntxt;
1516 u8 cmp_status;
1517 u8 cmp_cycle;
1518 u32 cmp_bytes;
1519 int rc;
e67f86b3 1520 int processed = 0;
e67f86b3 1521
760b48ca
BVA		 1522 lockdep_assert_held(&skdev->lock);
1523
e67f86b3
AB		 1524 for (;; ) {
1525 SKD_ASSERT(skdev->skcomp_ix < SKD_N_COMPLETION_ENTRY);
1526
1527 skcmp = &skdev->skcomp_table[skdev->skcomp_ix];
1528 cmp_cycle = skcmp->cycle;
1529 cmp_cntxt = skcmp->tag;
1530 cmp_status = skcmp->status;
1531 cmp_bytes = be32_to_cpu(skcmp->num_returned_bytes);
1532
1533 skerr = &skdev->skerr_table[skdev->skcomp_ix];
1534
f98806d6
BVA		 1535 dev_dbg(&skdev->pdev->dev,
1536 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d busy=%d rbytes=0x%x proto=%d\n",
1537 skdev->skcomp_cycle, skdev->skcomp_ix, cmp_cycle,
d4d0f5fc 1538 cmp_cntxt, cmp_status, skd_in_flight(skdev),
6fbb2de5 1539 cmp_bytes, skdev->proto_ver);
e67f86b3
AB		 1540
1541 if (cmp_cycle != skdev->skcomp_cycle) {
f98806d6 1542 dev_dbg(&skdev->pdev->dev, "end of completions\n");
e67f86b3
AB		 1543 break;
1544 }
1545 /*
1546 * Update the completion queue head index and possibly
1547 * the completion cycle count. 8-bit wrap-around.
1548 */
1549 skdev->skcomp_ix++;
1550 if (skdev->skcomp_ix >= SKD_N_COMPLETION_ENTRY) {
1551 skdev->skcomp_ix = 0;
1552 skdev->skcomp_cycle++;
1553 }
1554
1555 /*
1556 * The command context is a unique 32-bit ID. The low order
1557 * bits help locate the request. The request is usually a
1558 * r/w request (see skd_start() above) or a special request.
1559 */
1560 req_id = cmp_cntxt;
f18c17c8 1561 tag = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
e67f86b3
AB		 1562
1563 /* Is this other than a r/w request? */
f18c17c8 1564 if (tag >= skdev->num_req_context) {
e67f86b3
AB		 1565 /*
1566 * This is not a completion for a r/w request.
1567 */
ca33dd92
BVA		 1568 WARN_ON_ONCE(blk_mq_tag_to_rq(skdev->tag_set.tags[hwq],
1569 tag));
e67f86b3
AB		 1570 skd_complete_other(skdev, skcmp, skerr);
1571 continue;
1572 }
1573
ca33dd92 1574 rq = blk_mq_tag_to_rq(skdev->tag_set.tags[hwq], tag);
f18c17c8
BVA		 1575 if (WARN(!rq, "No request for tag %#x -> %#x\n", cmp_cntxt,
1576 tag))
1577 continue;
e7278a8b 1578 skreq = blk_mq_rq_to_pdu(rq);
e67f86b3
AB		 1579
1580 /*
1581 * Make sure the request ID for the slot matches.
1582 */
1583 if (skreq->id != req_id) {
f98806d6
BVA		 1584 dev_dbg(&skdev->pdev->dev,
1585 "mismatch comp_id=0x%x req_id=0x%x\n", req_id,
1586 skreq->id);
e67f86b3
AB		 1587 {
1588 u16 new_id = cmp_cntxt;
f98806d6
BVA		 1589 dev_err(&skdev->pdev->dev,
1590 "Completion mismatch comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
1591 req_id, skreq->id, new_id);
e67f86b3
AB		 1592
1593 continue;
1594 }
1595 }
1596
1597 SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);
1598
e67f86b3
AB		 1599 skreq->completion = *skcmp;
1600 if (unlikely(cmp_status == SAM_STAT_CHECK_CONDITION)) {
1601 skreq->err_info = *skerr;
1602 skd_log_check_status(skdev, cmp_status, skerr->key,
1603 skerr->code, skerr->qual,
1604 skerr->fruc);
1605 }
1606 /* Release DMA resources for the request. */
1607 if (skreq->n_sg > 0)
1608 skd_postop_sg_list(skdev, skreq);
1609
f18c17c8 1610 skd_release_skreq(skdev, skreq);
e67f86b3
AB		 1611
1612 /*
f18c17c8 1613 * Capture the outcome and post it back to the native request.
e67f86b3 1614 */
f18c17c8
BVA		 1615 if (likely(cmp_status == SAM_STAT_GOOD))
1616 skd_end_request(skdev, rq, BLK_STS_OK);
1617 else
1618 skd_resolve_req_exception(skdev, skreq, rq);
e67f86b3
AB		 1619
1620 /* skd_isr_comp_limit equal zero means no limit */
1621 if (limit) {
1622 if (++processed >= limit) {
1623 rc = 1;
1624 break;
1625 }
1626 }
1627 }
1628
6fbb2de5 1629 if (skdev->state == SKD_DRVR_STATE_PAUSING &&
d4d0f5fc 1630 skd_in_flight(skdev) == 0) {
e67f86b3
AB		 1631 skdev->state = SKD_DRVR_STATE_PAUSED;
1632 wake_up_interruptible(&skdev->waitq);
1633 }
1634
1635 return rc;
1636}
1637
1638static void skd_complete_other(struct skd_device *skdev,
85e34112
BVA		 1639 struct fit_completion_entry_v1 *skcomp,
1640 struct fit_comp_error_info *skerr)
e67f86b3
AB		 1641{
1642 u32 req_id = 0;
1643 u32 req_table;
1644 u32 req_slot;
1645 struct skd_special_context *skspcl;
1646
760b48ca
BVA		 1647 lockdep_assert_held(&skdev->lock);
1648
e67f86b3
AB		 1649 req_id = skcomp->tag;
1650 req_table = req_id & SKD_ID_TABLE_MASK;
1651 req_slot = req_id & SKD_ID_SLOT_MASK;
1652
f98806d6
BVA		 1653 dev_dbg(&skdev->pdev->dev, "table=0x%x id=0x%x slot=%d\n", req_table,
1654 req_id, req_slot);
e67f86b3
AB		 1655
1656 /*
1657 * Based on the request id, determine how to dispatch this completion.
1658 * This swich/case is finding the good cases and forwarding the
1659 * completion entry. Errors are reported below the switch.
1660 */
1661 switch (req_table) {
1662 case SKD_ID_RW_REQUEST:
1663 /*
e1d06f2d 1664 * The caller, skd_isr_completion_posted() above,
e67f86b3
AB		 1665 * handles r/w requests. The only way we get here
1666 * is if the req_slot is out of bounds.
1667 */
1668 break;
1669
e67f86b3
AB		 1670 case SKD_ID_INTERNAL:
1671 if (req_slot == 0) {
1672 skspcl = &skdev->internal_skspcl;
1673 if (skspcl->req.id == req_id &&
1674 skspcl->req.state == SKD_REQ_STATE_BUSY) {
1675 skd_complete_internal(skdev,
1676 skcomp, skerr, skspcl);
1677 return;
1678 }
1679 }
1680 break;
1681
1682 case SKD_ID_FIT_MSG:
1683 /*
1684 * These id's should never appear in a completion record.
1685 */
1686 break;
1687
1688 default:
1689 /*
1690 * These id's should never appear anywhere;
1691 */
1692 break;
1693 }
1694
1695 /*
1696 * If we get here it is a bad or stale id.
1697 */
1698}
1699
e67f86b3
AB		 1700static void skd_reset_skcomp(struct skd_device *skdev)
1701{
6f7c7675 1702 memset(skdev->skcomp_table, 0, SKD_SKCOMP_SIZE);
e67f86b3
AB		 1703
1704 skdev->skcomp_ix = 0;
1705 skdev->skcomp_cycle = 1;
1706}
1707
1708/*
1709 *****************************************************************************
1710 * INTERRUPTS
1711 *****************************************************************************
1712 */
1713static void skd_completion_worker(struct work_struct *work)
1714{
1715 struct skd_device *skdev =
1716 container_of(work, struct skd_device, completion_worker);
1717 unsigned long flags;
1718 int flush_enqueued = 0;
1719
1720 spin_lock_irqsave(&skdev->lock, flags);
1721
1722 /*
1723 * pass in limit=0, which means no limit..
1724 * process everything in compq
1725 */
1726 skd_isr_completion_posted(skdev, 0, &flush_enqueued);
ca33dd92 1727 schedule_work(&skdev->start_queue);
e67f86b3
AB		 1728
1729 spin_unlock_irqrestore(&skdev->lock, flags);
1730}
1731
1732static void skd_isr_msg_from_dev(struct skd_device *skdev);
1733
41c9499b
AB		 1734static irqreturn_t
1735skd_isr(int irq, void *ptr)
e67f86b3 1736{
1cd3c1ab 1737 struct skd_device *skdev = ptr;
e67f86b3
AB		 1738 u32 intstat;
1739 u32 ack;
1740 int rc = 0;
1741 int deferred = 0;
1742 int flush_enqueued = 0;
1743
e67f86b3
AB		 1744 spin_lock(&skdev->lock);
1745
1746 for (;; ) {
1747 intstat = SKD_READL(skdev, FIT_INT_STATUS_HOST);
1748
1749 ack = FIT_INT_DEF_MASK;
1750 ack &= intstat;
1751
f98806d6
BVA		 1752 dev_dbg(&skdev->pdev->dev, "intstat=0x%x ack=0x%x\n", intstat,
1753 ack);
e67f86b3
AB		 1754
1755 /* As long as there is an int pending on device, keep
1756 * running loop. When none, get out, but if we've never
1757 * done any processing, call completion handler?
1758 */
1759 if (ack == 0) {
1760 /* No interrupts on device, but run the completion
1761 * processor anyway?
1762 */
1763 if (rc == 0)
1764 if (likely (skdev->state
1765 == SKD_DRVR_STATE_ONLINE))
1766 deferred = 1;
1767 break;
1768 }
1769
1770 rc = IRQ_HANDLED;
1771
1772 SKD_WRITEL(skdev, ack, FIT_INT_STATUS_HOST);
1773
1774 if (likely((skdev->state != SKD_DRVR_STATE_LOAD) &&
1775 (skdev->state != SKD_DRVR_STATE_STOPPING))) {
1776 if (intstat & FIT_ISH_COMPLETION_POSTED) {
1777 /*
1778 * If we have already deferred completion
1779 * processing, don't bother running it again
1780 */
1781 if (deferred == 0)
1782 deferred =
1783 skd_isr_completion_posted(skdev,
1784 skd_isr_comp_limit, &flush_enqueued);
1785 }
1786
1787 if (intstat & FIT_ISH_FW_STATE_CHANGE) {
1788 skd_isr_fwstate(skdev);
1789 if (skdev->state == SKD_DRVR_STATE_FAULT ||
1790 skdev->state ==
1791 SKD_DRVR_STATE_DISAPPEARED) {
1792 spin_unlock(&skdev->lock);
1793 return rc;
1794 }
1795 }
1796
1797 if (intstat & FIT_ISH_MSG_FROM_DEV)
1798 skd_isr_msg_from_dev(skdev);
1799 }
1800 }
1801
1802 if (unlikely(flush_enqueued))
ca33dd92 1803 schedule_work(&skdev->start_queue);
e67f86b3
AB		 1804
1805 if (deferred)
1806 schedule_work(&skdev->completion_worker);
1807 else if (!flush_enqueued)
ca33dd92 1808 schedule_work(&skdev->start_queue);
e67f86b3
AB		 1809
1810 spin_unlock(&skdev->lock);
1811
1812 return rc;
1813}
1814
e67f86b3
AB		 1815static void skd_drive_fault(struct skd_device *skdev)
1816{
1817 skdev->state = SKD_DRVR_STATE_FAULT;
f98806d6 1818 dev_err(&skdev->pdev->dev, "Drive FAULT\n");
e67f86b3
AB		 1819}
1820
1821static void skd_drive_disappeared(struct skd_device *skdev)
1822{
1823 skdev->state = SKD_DRVR_STATE_DISAPPEARED;
f98806d6 1824 dev_err(&skdev->pdev->dev, "Drive DISAPPEARED\n");
e67f86b3
AB		 1825}
1826
1827static void skd_isr_fwstate(struct skd_device *skdev)
1828{
1829 u32 sense;
1830 u32 state;
1831 u32 mtd;
1832 int prev_driver_state = skdev->state;
1833
1834 sense = SKD_READL(skdev, FIT_STATUS);
1835 state = sense & FIT_SR_DRIVE_STATE_MASK;
1836
f98806d6
BVA		 1837 dev_err(&skdev->pdev->dev, "s1120 state %s(%d)=>%s(%d)\n",
1838 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
1839 skd_drive_state_to_str(state), state);
e67f86b3
AB		 1840
1841 skdev->drive_state = state;
1842
1843 switch (skdev->drive_state) {
1844 case FIT_SR_DRIVE_INIT:
1845 if (skdev->state == SKD_DRVR_STATE_PROTOCOL_MISMATCH) {
1846 skd_disable_interrupts(skdev);
1847 break;
1848 }
1849 if (skdev->state == SKD_DRVR_STATE_RESTARTING)
79ce12a8 1850 skd_recover_requests(skdev);
e67f86b3
AB		 1851 if (skdev->state == SKD_DRVR_STATE_WAIT_BOOT) {
1852 skdev->timer_countdown = SKD_STARTING_TIMO;
1853 skdev->state = SKD_DRVR_STATE_STARTING;
1854 skd_soft_reset(skdev);
1855 break;
1856 }
1857 mtd = FIT_MXD_CONS(FIT_MTD_FITFW_INIT, 0, 0);
1858 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
1859 skdev->last_mtd = mtd;
1860 break;
1861
1862 case FIT_SR_DRIVE_ONLINE:
1863 skdev->cur_max_queue_depth = skd_max_queue_depth;
1864 if (skdev->cur_max_queue_depth > skdev->dev_max_queue_depth)
1865 skdev->cur_max_queue_depth = skdev->dev_max_queue_depth;
1866
1867 skdev->queue_low_water_mark =
1868 skdev->cur_max_queue_depth * 2 / 3 + 1;
1869 if (skdev->queue_low_water_mark < 1)
1870 skdev->queue_low_water_mark = 1;
f98806d6
BVA		 1871 dev_info(&skdev->pdev->dev,
1872 "Queue depth limit=%d dev=%d lowat=%d\n",
1873 skdev->cur_max_queue_depth,
1874 skdev->dev_max_queue_depth,
1875 skdev->queue_low_water_mark);
e67f86b3
AB		 1876
1877 skd_refresh_device_data(skdev);
1878 break;
1879
1880 case FIT_SR_DRIVE_BUSY:
1881 skdev->state = SKD_DRVR_STATE_BUSY;
1882 skdev->timer_countdown = SKD_BUSY_TIMO;
1883 skd_quiesce_dev(skdev);
1884 break;
1885 case FIT_SR_DRIVE_BUSY_SANITIZE:
1886 /* set timer for 3 seconds, we'll abort any unfinished
1887 * commands after that expires
1888 */
1889 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
1890 skdev->timer_countdown = SKD_TIMER_SECONDS(3);
ca33dd92 1891 schedule_work(&skdev->start_queue);
e67f86b3
AB		 1892 break;
1893 case FIT_SR_DRIVE_BUSY_ERASE:
1894 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
1895 skdev->timer_countdown = SKD_BUSY_TIMO;
1896 break;
1897 case FIT_SR_DRIVE_OFFLINE:
1898 skdev->state = SKD_DRVR_STATE_IDLE;
1899 break;
1900 case FIT_SR_DRIVE_SOFT_RESET:
1901 switch (skdev->state) {
1902 case SKD_DRVR_STATE_STARTING:
1903 case SKD_DRVR_STATE_RESTARTING:
1904 /* Expected by a caller of skd_soft_reset() */
1905 break;
1906 default:
1907 skdev->state = SKD_DRVR_STATE_RESTARTING;
1908 break;
1909 }
1910 break;
1911 case FIT_SR_DRIVE_FW_BOOTING:
f98806d6 1912 dev_dbg(&skdev->pdev->dev, "ISR FIT_SR_DRIVE_FW_BOOTING\n");
e67f86b3
AB		 1913 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
1914 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
1915 break;
1916
1917 case FIT_SR_DRIVE_DEGRADED:
1918 case FIT_SR_PCIE_LINK_DOWN:
1919 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
1920 break;
1921
1922 case FIT_SR_DRIVE_FAULT:
1923 skd_drive_fault(skdev);
79ce12a8 1924 skd_recover_requests(skdev);
ca33dd92 1925 schedule_work(&skdev->start_queue);
e67f86b3
AB		 1926 break;
1927
1928 /* PCIe bus returned all Fs? */
1929 case 0xFF:
f98806d6
BVA		 1930 dev_info(&skdev->pdev->dev, "state=0x%x sense=0x%x\n", state,
1931 sense);
e67f86b3 1932 skd_drive_disappeared(skdev);
79ce12a8 1933 skd_recover_requests(skdev);
ca33dd92 1934 schedule_work(&skdev->start_queue);
e67f86b3
AB		 1935 break;
1936 default:
1937 /*
1938 * Uknown FW State. Wait for a state we recognize.
1939 */
1940 break;
1941 }
f98806d6
BVA		 1942 dev_err(&skdev->pdev->dev, "Driver state %s(%d)=>%s(%d)\n",
1943 skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
1944 skd_skdev_state_to_str(skdev->state), skdev->state);
e67f86b3
AB		 1945}
1946
ca33dd92 1947static void skd_recover_request(struct request *req, void *data, bool reserved)
e67f86b3 1948{
ca33dd92
BVA		 1949 struct skd_device *const skdev = data;
1950 struct skd_request_context *skreq = blk_mq_rq_to_pdu(req);
e67f86b3 1951
4e54b849
BVA		 1952 if (skreq->state != SKD_REQ_STATE_BUSY)
1953 return;
e67f86b3 1954
4e54b849 1955 skd_log_skreq(skdev, skreq, "recover");
e67f86b3 1956
4e54b849
BVA		 1957 /* Release DMA resources for the request. */
1958 if (skreq->n_sg > 0)
1959 skd_postop_sg_list(skdev, skreq);
e67f86b3 1960
4e54b849 1961 skreq->state = SKD_REQ_STATE_IDLE;
e67f86b3 1962
4e54b849
BVA		 1963 skd_end_request(skdev, req, BLK_STS_IOERR);
1964}
e67f86b3 1965
4e54b849
BVA		 1966static void skd_recover_requests(struct skd_device *skdev)
1967{
ca33dd92 1968 blk_mq_tagset_busy_iter(&skdev->tag_set, skd_recover_request, skdev);
e67f86b3
AB		 1969}
1970
1971static void skd_isr_msg_from_dev(struct skd_device *skdev)
1972{
1973 u32 mfd;
1974 u32 mtd;
1975 u32 data;
1976
1977 mfd = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
1978
f98806d6
BVA		 1979 dev_dbg(&skdev->pdev->dev, "mfd=0x%x last_mtd=0x%x\n", mfd,
1980 skdev->last_mtd);
e67f86b3
AB		 1981
1982 /* ignore any mtd that is an ack for something we didn't send */
1983 if (FIT_MXD_TYPE(mfd) != FIT_MXD_TYPE(skdev->last_mtd))
1984 return;
1985
1986 switch (FIT_MXD_TYPE(mfd)) {
1987 case FIT_MTD_FITFW_INIT:
1988 skdev->proto_ver = FIT_PROTOCOL_MAJOR_VER(mfd);
1989
1990 if (skdev->proto_ver != FIT_PROTOCOL_VERSION_1) {
f98806d6
BVA		 1991 dev_err(&skdev->pdev->dev, "protocol mismatch\n");
1992 dev_err(&skdev->pdev->dev, " got=%d support=%d\n",
1993 skdev->proto_ver, FIT_PROTOCOL_VERSION_1);
1994 dev_err(&skdev->pdev->dev, " please upgrade driver\n");
e67f86b3
AB		 1995 skdev->state = SKD_DRVR_STATE_PROTOCOL_MISMATCH;
1996 skd_soft_reset(skdev);
1997 break;
1998 }
1999 mtd = FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH, 0, 0);
2000 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
2001 skdev->last_mtd = mtd;
2002 break;
2003
2004 case FIT_MTD_GET_CMDQ_DEPTH:
2005 skdev->dev_max_queue_depth = FIT_MXD_DATA(mfd);
2006 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH, 0,
2007 SKD_N_COMPLETION_ENTRY);
2008 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
2009 skdev->last_mtd = mtd;
2010 break;
2011
2012 case FIT_MTD_SET_COMPQ_DEPTH:
2013 SKD_WRITEQ(skdev, skdev->cq_dma_address, FIT_MSG_TO_DEVICE_ARG);
2014 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR, 0, 0);
2015 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
2016 skdev->last_mtd = mtd;
2017 break;
2018
2019 case FIT_MTD_SET_COMPQ_ADDR:
2020 skd_reset_skcomp(skdev);
2021 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID, 0, skdev->devno);
2022 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
2023 skdev->last_mtd = mtd;
2024 break;
2025
2026 case FIT_MTD_CMD_LOG_HOST_ID:
2027 skdev->connect_time_stamp = get_seconds();
2028 data = skdev->connect_time_stamp & 0xFFFF;
2029 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO, 0, data);
2030 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
2031 skdev->last_mtd = mtd;
2032 break;
2033
2034 case FIT_MTD_CMD_LOG_TIME_STAMP_LO:
2035 skdev->drive_jiffies = FIT_MXD_DATA(mfd);
2036 data = (skdev->connect_time_stamp >> 16) & 0xFFFF;
2037 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI, 0, data);
2038 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
2039 skdev->last_mtd = mtd;
2040 break;
2041
2042 case FIT_MTD_CMD_LOG_TIME_STAMP_HI:
2043 skdev->drive_jiffies |= (FIT_MXD_DATA(mfd) << 16);
2044 mtd = FIT_MXD_CONS(FIT_MTD_ARM_QUEUE, 0, 0);
2045 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
2046 skdev->last_mtd = mtd;
2047
f98806d6
BVA		 2048 dev_err(&skdev->pdev->dev, "Time sync driver=0x%x device=0x%x\n",
2049 skdev->connect_time_stamp, skdev->drive_jiffies);
e67f86b3
AB		 2050 break;
2051
2052 case FIT_MTD_ARM_QUEUE:
2053 skdev->last_mtd = 0;
2054 /*
2055 * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
2056 */
2057 break;
2058
2059 default:
2060 break;
2061 }
2062}
2063
2064static void skd_disable_interrupts(struct skd_device *skdev)
2065{
2066 u32 sense;
2067
2068 sense = SKD_READL(skdev, FIT_CONTROL);
2069 sense &= ~FIT_CR_ENABLE_INTERRUPTS;
2070 SKD_WRITEL(skdev, sense, FIT_CONTROL);
f98806d6 2071 dev_dbg(&skdev->pdev->dev, "sense 0x%x\n", sense);
e67f86b3
AB		 2072
2073 /* Note that the 1s is written. A 1-bit means
2074 * disable, a 0 means enable.
2075 */
2076 SKD_WRITEL(skdev, ~0, FIT_INT_MASK_HOST);
2077}
2078
2079static void skd_enable_interrupts(struct skd_device *skdev)
2080{
2081 u32 val;
2082
2083 /* unmask interrupts first */
2084 val = FIT_ISH_FW_STATE_CHANGE +
2085 FIT_ISH_COMPLETION_POSTED + FIT_ISH_MSG_FROM_DEV;
2086
2087 /* Note that the compliment of mask is written. A 1-bit means
2088 * disable, a 0 means enable. */
2089 SKD_WRITEL(skdev, ~val, FIT_INT_MASK_HOST);
f98806d6 2090 dev_dbg(&skdev->pdev->dev, "interrupt mask=0x%x\n", ~val);
e67f86b3
AB		 2091
2092 val = SKD_READL(skdev, FIT_CONTROL);
2093 val |= FIT_CR_ENABLE_INTERRUPTS;
f98806d6 2094 dev_dbg(&skdev->pdev->dev, "control=0x%x\n", val);
e67f86b3
AB		 2095 SKD_WRITEL(skdev, val, FIT_CONTROL);
2096}
2097
2098/*
2099 *****************************************************************************
2100 * START, STOP, RESTART, QUIESCE, UNQUIESCE
2101 *****************************************************************************
2102 */
2103
2104static void skd_soft_reset(struct skd_device *skdev)
2105{
2106 u32 val;
2107
2108 val = SKD_READL(skdev, FIT_CONTROL);
2109 val |= (FIT_CR_SOFT_RESET);
f98806d6 2110 dev_dbg(&skdev->pdev->dev, "control=0x%x\n", val);
e67f86b3
AB		 2111 SKD_WRITEL(skdev, val, FIT_CONTROL);
2112}
2113
2114static void skd_start_device(struct skd_device *skdev)
2115{
2116 unsigned long flags;
2117 u32 sense;
2118 u32 state;
2119
2120 spin_lock_irqsave(&skdev->lock, flags);
2121
2122 /* ack all ghost interrupts */
2123 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
2124
2125 sense = SKD_READL(skdev, FIT_STATUS);
2126
f98806d6 2127 dev_dbg(&skdev->pdev->dev, "initial status=0x%x\n", sense);
e67f86b3
AB		 2128
2129 state = sense & FIT_SR_DRIVE_STATE_MASK;
2130 skdev->drive_state = state;
2131 skdev->last_mtd = 0;
2132
2133 skdev->state = SKD_DRVR_STATE_STARTING;
2134 skdev->timer_countdown = SKD_STARTING_TIMO;
2135
2136 skd_enable_interrupts(skdev);
2137
2138 switch (skdev->drive_state) {
2139 case FIT_SR_DRIVE_OFFLINE:
f98806d6 2140 dev_err(&skdev->pdev->dev, "Drive offline...\n");
e67f86b3
AB		 2141 break;
2142
2143 case FIT_SR_DRIVE_FW_BOOTING:
f98806d6 2144 dev_dbg(&skdev->pdev->dev, "FIT_SR_DRIVE_FW_BOOTING\n");
e67f86b3
AB		 2145 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
2146 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
2147 break;
2148
2149 case FIT_SR_DRIVE_BUSY_SANITIZE:
f98806d6 2150 dev_info(&skdev->pdev->dev, "Start: BUSY_SANITIZE\n");
e67f86b3
AB		 2151 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
2152 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
2153 break;
2154
2155 case FIT_SR_DRIVE_BUSY_ERASE:
f98806d6 2156 dev_info(&skdev->pdev->dev, "Start: BUSY_ERASE\n");
e67f86b3
AB		 2157 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
2158 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
2159 break;
2160
2161 case FIT_SR_DRIVE_INIT:
2162 case FIT_SR_DRIVE_ONLINE:
2163 skd_soft_reset(skdev);
2164 break;
2165
2166 case FIT_SR_DRIVE_BUSY:
f98806d6 2167 dev_err(&skdev->pdev->dev, "Drive Busy...\n");
e67f86b3
AB		 2168 skdev->state = SKD_DRVR_STATE_BUSY;
2169 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
2170 break;
2171
2172 case FIT_SR_DRIVE_SOFT_RESET:
f98806d6 2173 dev_err(&skdev->pdev->dev, "drive soft reset in prog\n");
e67f86b3
AB		 2174 break;
2175
2176 case FIT_SR_DRIVE_FAULT:
2177 /* Fault state is bad...soft reset won't do it...
2178 * Hard reset, maybe, but does it work on device?
2179 * For now, just fault so the system doesn't hang.
2180 */
2181 skd_drive_fault(skdev);
2182 /*start the queue so we can respond with error to requests */
f98806d6 2183 dev_dbg(&skdev->pdev->dev, "starting queue\n");
ca33dd92 2184 schedule_work(&skdev->start_queue);
e67f86b3
AB		 2185 skdev->gendisk_on = -1;
2186 wake_up_interruptible(&skdev->waitq);
2187 break;
2188
2189 case 0xFF:
2190 /* Most likely the device isn't there or isn't responding
2191 * to the BAR1 addresses. */
2192 skd_drive_disappeared(skdev);
2193 /*start the queue so we can respond with error to requests */
f98806d6
BVA		 2194 dev_dbg(&skdev->pdev->dev,
2195 "starting queue to error-out reqs\n");
ca33dd92 2196 schedule_work(&skdev->start_queue);
e67f86b3
AB		 2197 skdev->gendisk_on = -1;
2198 wake_up_interruptible(&skdev->waitq);
2199 break;
2200
2201 default:
f98806d6
BVA		 2202 dev_err(&skdev->pdev->dev, "Start: unknown state %x\n",
2203 skdev->drive_state);
e67f86b3
AB		 2204 break;
2205 }
2206
2207 state = SKD_READL(skdev, FIT_CONTROL);
f98806d6 2208 dev_dbg(&skdev->pdev->dev, "FIT Control Status=0x%x\n", state);
e67f86b3
AB		 2209
2210 state = SKD_READL(skdev, FIT_INT_STATUS_HOST);
f98806d6 2211 dev_dbg(&skdev->pdev->dev, "Intr Status=0x%x\n", state);
e67f86b3
AB		 2212
2213 state = SKD_READL(skdev, FIT_INT_MASK_HOST);
f98806d6 2214 dev_dbg(&skdev->pdev->dev, "Intr Mask=0x%x\n", state);
e67f86b3
AB		 2215
2216 state = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
f98806d6 2217 dev_dbg(&skdev->pdev->dev, "Msg from Dev=0x%x\n", state);
e67f86b3
AB		 2218
2219 state = SKD_READL(skdev, FIT_HW_VERSION);
f98806d6 2220 dev_dbg(&skdev->pdev->dev, "HW version=0x%x\n", state);
e67f86b3
AB		 2221
2222 spin_unlock_irqrestore(&skdev->lock, flags);
2223}
2224
2225static void skd_stop_device(struct skd_device *skdev)
2226{
2227 unsigned long flags;
2228 struct skd_special_context *skspcl = &skdev->internal_skspcl;
2229 u32 dev_state;
2230 int i;
2231
2232 spin_lock_irqsave(&skdev->lock, flags);
2233
2234 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
f98806d6 2235 dev_err(&skdev->pdev->dev, "%s not online no sync\n", __func__);
e67f86b3
AB		 2236 goto stop_out;
2237 }
2238
2239 if (skspcl->req.state != SKD_REQ_STATE_IDLE) {
f98806d6 2240 dev_err(&skdev->pdev->dev, "%s no special\n", __func__);
e67f86b3
AB		 2241 goto stop_out;
2242 }
2243
2244 skdev->state = SKD_DRVR_STATE_SYNCING;
2245 skdev->sync_done = 0;
2246
2247 skd_send_internal_skspcl(skdev, skspcl, SYNCHRONIZE_CACHE);
2248
2249 spin_unlock_irqrestore(&skdev->lock, flags);
2250
2251 wait_event_interruptible_timeout(skdev->waitq,
2252 (skdev->sync_done), (10 * HZ));
2253
2254 spin_lock_irqsave(&skdev->lock, flags);
2255
2256 switch (skdev->sync_done) {
2257 case 0:
f98806d6 2258 dev_err(&skdev->pdev->dev, "%s no sync\n", __func__);
e67f86b3
AB		 2259 break;
2260 case 1:
f98806d6 2261 dev_err(&skdev->pdev->dev, "%s sync done\n", __func__);
e67f86b3
AB		 2262 break;
2263 default:
f98806d6 2264 dev_err(&skdev->pdev->dev, "%s sync error\n", __func__);
e67f86b3
AB		 2265 }
2266
2267stop_out:
2268 skdev->state = SKD_DRVR_STATE_STOPPING;
2269 spin_unlock_irqrestore(&skdev->lock, flags);
2270
2271 skd_kill_timer(skdev);
2272
2273 spin_lock_irqsave(&skdev->lock, flags);
2274 skd_disable_interrupts(skdev);
2275
2276 /* ensure all ints on device are cleared */
2277 /* soft reset the device to unload with a clean slate */
2278 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
2279 SKD_WRITEL(skdev, FIT_CR_SOFT_RESET, FIT_CONTROL);
2280
2281 spin_unlock_irqrestore(&skdev->lock, flags);
2282
2283 /* poll every 100ms, 1 second timeout */
2284 for (i = 0; i < 10; i++) {
2285 dev_state =
2286 SKD_READL(skdev, FIT_STATUS) & FIT_SR_DRIVE_STATE_MASK;
2287 if (dev_state == FIT_SR_DRIVE_INIT)
2288 break;
2289 set_current_state(TASK_INTERRUPTIBLE);
2290 schedule_timeout(msecs_to_jiffies(100));
2291 }
2292
2293 if (dev_state != FIT_SR_DRIVE_INIT)
f98806d6
BVA		 2294 dev_err(&skdev->pdev->dev, "%s state error 0x%02x\n", __func__,
2295 dev_state);
e67f86b3
AB		 2296}
2297
2298/* assume spinlock is held */
2299static void skd_restart_device(struct skd_device *skdev)
2300{
2301 u32 state;
2302
2303 /* ack all ghost interrupts */
2304 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
2305
2306 state = SKD_READL(skdev, FIT_STATUS);
2307
f98806d6 2308 dev_dbg(&skdev->pdev->dev, "drive status=0x%x\n", state);
e67f86b3
AB		 2309
2310 state &= FIT_SR_DRIVE_STATE_MASK;
2311 skdev->drive_state = state;
2312 skdev->last_mtd = 0;
2313
2314 skdev->state = SKD_DRVR_STATE_RESTARTING;
2315 skdev->timer_countdown = SKD_RESTARTING_TIMO;
2316
2317 skd_soft_reset(skdev);
2318}
2319
2320/* assume spinlock is held */
2321static int skd_quiesce_dev(struct skd_device *skdev)
2322{
2323 int rc = 0;
2324
2325 switch (skdev->state) {
2326 case SKD_DRVR_STATE_BUSY:
2327 case SKD_DRVR_STATE_BUSY_IMMINENT:
f98806d6 2328 dev_dbg(&skdev->pdev->dev, "stopping queue\n");
ca33dd92 2329 blk_mq_stop_hw_queues(skdev->queue);
e67f86b3
AB		 2330 break;
2331 case SKD_DRVR_STATE_ONLINE:
2332 case SKD_DRVR_STATE_STOPPING:
2333 case SKD_DRVR_STATE_SYNCING:
2334 case SKD_DRVR_STATE_PAUSING:
2335 case SKD_DRVR_STATE_PAUSED:
2336 case SKD_DRVR_STATE_STARTING:
2337 case SKD_DRVR_STATE_RESTARTING:
2338 case SKD_DRVR_STATE_RESUMING:
2339 default:
2340 rc = -EINVAL;
f98806d6
BVA		 2341 dev_dbg(&skdev->pdev->dev, "state [%d] not implemented\n",
2342 skdev->state);
e67f86b3
AB		 2343 }
2344 return rc;
2345}
2346
2347/* assume spinlock is held */
2348static int skd_unquiesce_dev(struct skd_device *skdev)
2349{
2350 int prev_driver_state = skdev->state;
2351
2352 skd_log_skdev(skdev, "unquiesce");
2353 if (skdev->state == SKD_DRVR_STATE_ONLINE) {
f98806d6 2354 dev_dbg(&skdev->pdev->dev, "**** device already ONLINE\n");
e67f86b3
AB		 2355 return 0;
2356 }
2357 if (skdev->drive_state != FIT_SR_DRIVE_ONLINE) {
2358 /*
2359 * If there has been an state change to other than
2360 * ONLINE, we will rely on controller state change
2361 * to come back online and restart the queue.
2362 * The BUSY state means that driver is ready to
2363 * continue normal processing but waiting for controller
2364 * to become available.
2365 */
2366 skdev->state = SKD_DRVR_STATE_BUSY;
f98806d6 2367 dev_dbg(&skdev->pdev->dev, "drive BUSY state\n");
e67f86b3
AB		 2368 return 0;
2369 }
2370
2371 /*
2372 * Drive has just come online, driver is either in startup,
2373 * paused performing a task, or bust waiting for hardware.
2374 */
2375 switch (skdev->state) {
2376 case SKD_DRVR_STATE_PAUSED:
2377 case SKD_DRVR_STATE_BUSY:
2378 case SKD_DRVR_STATE_BUSY_IMMINENT:
2379 case SKD_DRVR_STATE_BUSY_ERASE:
2380 case SKD_DRVR_STATE_STARTING:
2381 case SKD_DRVR_STATE_RESTARTING:
2382 case SKD_DRVR_STATE_FAULT:
2383 case SKD_DRVR_STATE_IDLE:
2384 case SKD_DRVR_STATE_LOAD:
2385 skdev->state = SKD_DRVR_STATE_ONLINE;
f98806d6
BVA		 2386 dev_err(&skdev->pdev->dev, "Driver state %s(%d)=>%s(%d)\n",
2387 skd_skdev_state_to_str(prev_driver_state),
2388 prev_driver_state, skd_skdev_state_to_str(skdev->state),
2389 skdev->state);
2390 dev_dbg(&skdev->pdev->dev,
2391 "**** device ONLINE...starting block queue\n");
2392 dev_dbg(&skdev->pdev->dev, "starting queue\n");
2393 dev_info(&skdev->pdev->dev, "STEC s1120 ONLINE\n");
ca33dd92 2394 schedule_work(&skdev->start_queue);
e67f86b3
AB		 2395 skdev->gendisk_on = 1;
2396 wake_up_interruptible(&skdev->waitq);
2397 break;
2398
2399 case SKD_DRVR_STATE_DISAPPEARED:
2400 default:
f98806d6
BVA		 2401 dev_dbg(&skdev->pdev->dev,
2402 "**** driver state %d, not implemented\n",
2403 skdev->state);
e67f86b3
AB		 2404 return -EBUSY;
2405 }
2406 return 0;
2407}
2408
2409/*
2410 *****************************************************************************
2411 * PCIe MSI/MSI-X INTERRUPT HANDLERS
2412 *****************************************************************************
2413 */
2414
2415static irqreturn_t skd_reserved_isr(int irq, void *skd_host_data)
2416{
2417 struct skd_device *skdev = skd_host_data;
2418 unsigned long flags;
2419
2420 spin_lock_irqsave(&skdev->lock, flags);
f98806d6
BVA		 2421 dev_dbg(&skdev->pdev->dev, "MSIX = 0x%x\n",
2422 SKD_READL(skdev, FIT_INT_STATUS_HOST));
2423 dev_err(&skdev->pdev->dev, "MSIX reserved irq %d = 0x%x\n", irq,
2424 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 2425 SKD_WRITEL(skdev, FIT_INT_RESERVED_MASK, FIT_INT_STATUS_HOST);
2426 spin_unlock_irqrestore(&skdev->lock, flags);
2427 return IRQ_HANDLED;
2428}
2429
2430static irqreturn_t skd_statec_isr(int irq, void *skd_host_data)
2431{
2432 struct skd_device *skdev = skd_host_data;
2433 unsigned long flags;
2434
2435 spin_lock_irqsave(&skdev->lock, flags);
f98806d6
BVA		 2436 dev_dbg(&skdev->pdev->dev, "MSIX = 0x%x\n",
2437 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 2438 SKD_WRITEL(skdev, FIT_ISH_FW_STATE_CHANGE, FIT_INT_STATUS_HOST);
2439 skd_isr_fwstate(skdev);
2440 spin_unlock_irqrestore(&skdev->lock, flags);
2441 return IRQ_HANDLED;
2442}
2443
2444static irqreturn_t skd_comp_q(int irq, void *skd_host_data)
2445{
2446 struct skd_device *skdev = skd_host_data;
2447 unsigned long flags;
2448 int flush_enqueued = 0;
2449 int deferred;
2450
2451 spin_lock_irqsave(&skdev->lock, flags);
f98806d6
BVA		 2452 dev_dbg(&skdev->pdev->dev, "MSIX = 0x%x\n",
2453 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 2454 SKD_WRITEL(skdev, FIT_ISH_COMPLETION_POSTED, FIT_INT_STATUS_HOST);
2455 deferred = skd_isr_completion_posted(skdev, skd_isr_comp_limit,
2456 &flush_enqueued);
e67f86b3 2457 if (flush_enqueued)
ca33dd92 2458 schedule_work(&skdev->start_queue);
e67f86b3
AB		 2459
2460 if (deferred)
2461 schedule_work(&skdev->completion_worker);
2462 else if (!flush_enqueued)
ca33dd92 2463 schedule_work(&skdev->start_queue);
e67f86b3
AB		 2464
2465 spin_unlock_irqrestore(&skdev->lock, flags);
2466
2467 return IRQ_HANDLED;
2468}
2469
2470static irqreturn_t skd_msg_isr(int irq, void *skd_host_data)
2471{
2472 struct skd_device *skdev = skd_host_data;
2473 unsigned long flags;
2474
2475 spin_lock_irqsave(&skdev->lock, flags);
f98806d6
BVA		 2476 dev_dbg(&skdev->pdev->dev, "MSIX = 0x%x\n",
2477 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 2478 SKD_WRITEL(skdev, FIT_ISH_MSG_FROM_DEV, FIT_INT_STATUS_HOST);
2479 skd_isr_msg_from_dev(skdev);
2480 spin_unlock_irqrestore(&skdev->lock, flags);
2481 return IRQ_HANDLED;
2482}
2483
2484static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
2485{
2486 struct skd_device *skdev = skd_host_data;
2487 unsigned long flags;
2488
2489 spin_lock_irqsave(&skdev->lock, flags);
f98806d6
BVA		 2490 dev_dbg(&skdev->pdev->dev, "MSIX = 0x%x\n",
2491 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 2492 SKD_WRITEL(skdev, FIT_INT_QUEUE_FULL, FIT_INT_STATUS_HOST);
2493 spin_unlock_irqrestore(&skdev->lock, flags);
2494 return IRQ_HANDLED;
2495}
2496
2497/*
2498 *****************************************************************************
2499 * PCIe MSI/MSI-X SETUP
2500 *****************************************************************************
2501 */
2502
2503struct skd_msix_entry {
e67f86b3
AB		 2504 char isr_name[30];
2505};
2506
2507struct skd_init_msix_entry {
2508 const char *name;
2509 irq_handler_t handler;
2510};
2511
2512#define SKD_MAX_MSIX_COUNT 13
2513#define SKD_MIN_MSIX_COUNT 7
2514#define SKD_BASE_MSIX_IRQ 4
2515
2516static struct skd_init_msix_entry msix_entries[SKD_MAX_MSIX_COUNT] = {
2517 { "(DMA 0)", skd_reserved_isr },
2518 { "(DMA 1)", skd_reserved_isr },
2519 { "(DMA 2)", skd_reserved_isr },
2520 { "(DMA 3)", skd_reserved_isr },
2521 { "(State Change)", skd_statec_isr },
2522 { "(COMPL_Q)", skd_comp_q },
2523 { "(MSG)", skd_msg_isr },
2524 { "(Reserved)", skd_reserved_isr },
2525 { "(Reserved)", skd_reserved_isr },
2526 { "(Queue Full 0)", skd_qfull_isr },
2527 { "(Queue Full 1)", skd_qfull_isr },
2528 { "(Queue Full 2)", skd_qfull_isr },
2529 { "(Queue Full 3)", skd_qfull_isr },
2530};
2531
e67f86b3
AB		 2532static int skd_acquire_msix(struct skd_device *skdev)
2533{
a9df8625 2534 int i, rc;
46817769 2535 struct pci_dev *pdev = skdev->pdev;
e67f86b3 2536
180b0ae7
CH		 2537 rc = pci_alloc_irq_vectors(pdev, SKD_MAX_MSIX_COUNT, SKD_MAX_MSIX_COUNT,
2538 PCI_IRQ_MSIX);
2539 if (rc < 0) {
f98806d6 2540 dev_err(&skdev->pdev->dev, "failed to enable MSI-X %d\n", rc);
3bc8492f 2541 goto out;
e67f86b3 2542 }
46817769 2543
180b0ae7
CH		 2544 skdev->msix_entries = kcalloc(SKD_MAX_MSIX_COUNT,
2545 sizeof(struct skd_msix_entry), GFP_KERNEL);
e67f86b3
AB		 2546 if (!skdev->msix_entries) {
2547 rc = -ENOMEM;
f98806d6 2548 dev_err(&skdev->pdev->dev, "msix table allocation error\n");
3bc8492f 2549 goto out;
e67f86b3
AB		 2550 }
2551
e67f86b3 2552 /* Enable MSI-X vectors for the base queue */
180b0ae7
CH		 2553 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
2554 struct skd_msix_entry *qentry = &skdev->msix_entries[i];
2555
e67f86b3
AB		 2556 snprintf(qentry->isr_name, sizeof(qentry->isr_name),
2557 "%s%d-msix %s", DRV_NAME, skdev->devno,
2558 msix_entries[i].name);
180b0ae7
CH		 2559
2560 rc = devm_request_irq(&skdev->pdev->dev,
2561 pci_irq_vector(skdev->pdev, i),
2562 msix_entries[i].handler, 0,
2563 qentry->isr_name, skdev);
e67f86b3 2564 if (rc) {
f98806d6
BVA		 2565 dev_err(&skdev->pdev->dev,
2566 "Unable to register(%d) MSI-X handler %d: %s\n",
2567 rc, i, qentry->isr_name);
e67f86b3 2568 goto msix_out;
e67f86b3
AB		 2569 }
2570 }
180b0ae7 2571
f98806d6
BVA		 2572 dev_dbg(&skdev->pdev->dev, "%d msix irq(s) enabled\n",
2573 SKD_MAX_MSIX_COUNT);
e67f86b3
AB		 2574 return 0;
2575
2576msix_out:
180b0ae7
CH		 2577 while (--i >= 0)
2578 devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), skdev);
3bc8492f 2579out:
180b0ae7
CH		 2580 kfree(skdev->msix_entries);
2581 skdev->msix_entries = NULL;
e67f86b3
AB		 2582 return rc;
2583}
2584
2585static int skd_acquire_irq(struct skd_device *skdev)
2586{
180b0ae7
CH		 2587 struct pci_dev *pdev = skdev->pdev;
2588 unsigned int irq_flag = PCI_IRQ_LEGACY;
e67f86b3 2589 int rc;
e67f86b3 2590
180b0ae7 2591 if (skd_isr_type == SKD_IRQ_MSIX) {
e67f86b3
AB		 2592 rc = skd_acquire_msix(skdev);
2593 if (!rc)
180b0ae7
CH		 2594 return 0;
2595
f98806d6
BVA		 2596 dev_err(&skdev->pdev->dev,
2597 "failed to enable MSI-X, re-trying with MSI %d\n", rc);
e67f86b3 2598 }
180b0ae7
CH		 2599
2600 snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d", DRV_NAME,
2601 skdev->devno);
2602
2603 if (skd_isr_type != SKD_IRQ_LEGACY)
2604 irq_flag |= PCI_IRQ_MSI;
2605 rc = pci_alloc_irq_vectors(pdev, 1, 1, irq_flag);
2606 if (rc < 0) {
f98806d6
BVA		 2607 dev_err(&skdev->pdev->dev,
2608 "failed to allocate the MSI interrupt %d\n", rc);
180b0ae7
CH		 2609 return rc;
2610 }
2611
2612 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
2613 pdev->msi_enabled ? 0 : IRQF_SHARED,
2614 skdev->isr_name, skdev);
2615 if (rc) {
2616 pci_free_irq_vectors(pdev);
f98806d6
BVA		 2617 dev_err(&skdev->pdev->dev, "failed to allocate interrupt %d\n",
2618 rc);
180b0ae7
CH		 2619 return rc;
2620 }
2621
2622 return 0;
e67f86b3
AB		 2623}
2624
2625static void skd_release_irq(struct skd_device *skdev)
2626{
180b0ae7
CH		 2627 struct pci_dev *pdev = skdev->pdev;
2628
2629 if (skdev->msix_entries) {
2630 int i;
2631
2632 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
2633 devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i),
2634 skdev);
2635 }
2636
2637 kfree(skdev->msix_entries);
2638 skdev->msix_entries = NULL;
2639 } else {
2640 devm_free_irq(&pdev->dev, pdev->irq, skdev);
e67f86b3 2641 }
180b0ae7
CH		 2642
2643 pci_free_irq_vectors(pdev);
e67f86b3
AB		 2644}
2645
2646/*
2647 *****************************************************************************
2648 * CONSTRUCT
2649 *****************************************************************************
2650 */
2651
a3db102d
BVA		 2652static void *skd_alloc_dma(struct skd_device *skdev, struct kmem_cache *s,
2653 dma_addr_t *dma_handle, gfp_t gfp,
2654 enum dma_data_direction dir)
2655{
2656 struct device *dev = &skdev->pdev->dev;
2657 void *buf;
2658
2659 buf = kmem_cache_alloc(s, gfp);
2660 if (!buf)
2661 return NULL;
2662 *dma_handle = dma_map_single(dev, buf, s->size, dir);
2663 if (dma_mapping_error(dev, *dma_handle)) {
2664 kfree(buf);
2665 buf = NULL;
2666 }
2667 return buf;
2668}
2669
2670static void skd_free_dma(struct skd_device *skdev, struct kmem_cache *s,
2671 void *vaddr, dma_addr_t dma_handle,
2672 enum dma_data_direction dir)
2673{
2674 if (!vaddr)
2675 return;
2676
2677 dma_unmap_single(&skdev->pdev->dev, dma_handle, s->size, dir);
2678 kmem_cache_free(s, vaddr);
2679}
2680
e67f86b3
AB		 2681static int skd_cons_skcomp(struct skd_device *skdev)
2682{
2683 int rc = 0;
2684 struct fit_completion_entry_v1 *skcomp;
e67f86b3 2685
f98806d6 2686 dev_dbg(&skdev->pdev->dev,
6f7c7675
BVA		 2687 "comp pci_alloc, total bytes %zd entries %d\n",
2688 SKD_SKCOMP_SIZE, SKD_N_COMPLETION_ENTRY);
e67f86b3 2689
6f7c7675 2690 skcomp = pci_zalloc_consistent(skdev->pdev, SKD_SKCOMP_SIZE,
a5bbf616 2691 &skdev->cq_dma_address);
e67f86b3
AB		 2692
2693 if (skcomp == NULL) {
2694 rc = -ENOMEM;
2695 goto err_out;
2696 }
2697
e67f86b3
AB		 2698 skdev->skcomp_table = skcomp;
2699 skdev->skerr_table = (struct fit_comp_error_info *)((char *)skcomp +
2700 sizeof(*skcomp) *
2701 SKD_N_COMPLETION_ENTRY);
2702
2703err_out:
2704 return rc;
2705}
2706
2707static int skd_cons_skmsg(struct skd_device *skdev)
2708{
2709 int rc = 0;
2710 u32 i;
2711
f98806d6 2712 dev_dbg(&skdev->pdev->dev,
01433d0d 2713 "skmsg_table kcalloc, struct %lu, count %u total %lu\n",
f98806d6
BVA		 2714 sizeof(struct skd_fitmsg_context), skdev->num_fitmsg_context,
2715 sizeof(struct skd_fitmsg_context) * skdev->num_fitmsg_context);
e67f86b3 2716
01433d0d
BVA		 2717 skdev->skmsg_table = kcalloc(skdev->num_fitmsg_context,
2718 sizeof(struct skd_fitmsg_context),
2719 GFP_KERNEL);
e67f86b3
AB		 2720 if (skdev->skmsg_table == NULL) {
2721 rc = -ENOMEM;
2722 goto err_out;
2723 }
2724
2725 for (i = 0; i < skdev->num_fitmsg_context; i++) {
2726 struct skd_fitmsg_context *skmsg;
2727
2728 skmsg = &skdev->skmsg_table[i];
2729
2730 skmsg->id = i + SKD_ID_FIT_MSG;
2731
e67f86b3 2732 skmsg->msg_buf = pci_alloc_consistent(skdev->pdev,
6507f436 2733 SKD_N_FITMSG_BYTES,
e67f86b3
AB		 2734 &skmsg->mb_dma_address);
2735
2736 if (skmsg->msg_buf == NULL) {
2737 rc = -ENOMEM;
2738 goto err_out;
2739 }
2740
6507f436
BVA		 2741 WARN(((uintptr_t)skmsg->msg_buf | skmsg->mb_dma_address) &
2742 (FIT_QCMD_ALIGN - 1),
2743 "not aligned: msg_buf %p mb_dma_address %#llx\n",
2744 skmsg->msg_buf, skmsg->mb_dma_address);
e67f86b3 2745 memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
e67f86b3
AB		 2746 }
2747
e67f86b3
AB		 2748err_out:
2749 return rc;
2750}
2751
542d7b00
BZ		 2752static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
2753 u32 n_sg,
2754 dma_addr_t *ret_dma_addr)
2755{
2756 struct fit_sg_descriptor *sg_list;
542d7b00 2757
a3db102d
BVA		 2758 sg_list = skd_alloc_dma(skdev, skdev->sglist_cache, ret_dma_addr,
2759 GFP_DMA | __GFP_ZERO, DMA_TO_DEVICE);
542d7b00
BZ		 2760
2761 if (sg_list != NULL) {
2762 uint64_t dma_address = *ret_dma_addr;
2763 u32 i;
2764
542d7b00
BZ		 2765 for (i = 0; i < n_sg - 1; i++) {
2766 uint64_t ndp_off;
2767 ndp_off = (i + 1) * sizeof(struct fit_sg_descriptor);
2768
2769 sg_list[i].next_desc_ptr = dma_address + ndp_off;
2770 }
2771 sg_list[i].next_desc_ptr = 0LL;
2772 }
2773
2774 return sg_list;
2775}
2776
5d003240 2777static void skd_free_sg_list(struct skd_device *skdev,
a3db102d 2778 struct fit_sg_descriptor *sg_list,
5d003240
BVA		 2779 dma_addr_t dma_addr)
2780{
5d003240
BVA		 2781 if (WARN_ON_ONCE(!sg_list))
2782 return;
2783
a3db102d
BVA		 2784 skd_free_dma(skdev, skdev->sglist_cache, sg_list, dma_addr,
2785 DMA_TO_DEVICE);
5d003240
BVA		 2786}
2787
ca33dd92
BVA		 2788static int skd_init_request(struct blk_mq_tag_set *set, struct request *rq,
2789 unsigned int hctx_idx, unsigned int numa_node)
e67f86b3 2790{
ca33dd92 2791 struct skd_device *skdev = set->driver_data;
e7278a8b 2792 struct skd_request_context *skreq = blk_mq_rq_to_pdu(rq);
e67f86b3 2793
e7278a8b
BVA		 2794 skreq->state = SKD_REQ_STATE_IDLE;
2795 skreq->sg = (void *)(skreq + 1);
2796 sg_init_table(skreq->sg, skd_sgs_per_request);
2797 skreq->sksg_list = skd_cons_sg_list(skdev, skd_sgs_per_request,
2798 &skreq->sksg_dma_address);
e67f86b3 2799
e7278a8b
BVA		 2800 return skreq->sksg_list ? 0 : -ENOMEM;
2801}
e67f86b3 2802
ca33dd92
BVA		 2803static void skd_exit_request(struct blk_mq_tag_set *set, struct request *rq,
2804 unsigned int hctx_idx)
e7278a8b 2805{
ca33dd92 2806 struct skd_device *skdev = set->driver_data;
e7278a8b 2807 struct skd_request_context *skreq = blk_mq_rq_to_pdu(rq);
e67f86b3 2808
a3db102d 2809 skd_free_sg_list(skdev, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 2810}
2811
e67f86b3
AB		 2812static int skd_cons_sksb(struct skd_device *skdev)
2813{
2814 int rc = 0;
2815 struct skd_special_context *skspcl;
e67f86b3
AB		 2816
2817 skspcl = &skdev->internal_skspcl;
2818
2819 skspcl->req.id = 0 + SKD_ID_INTERNAL;
2820 skspcl->req.state = SKD_REQ_STATE_IDLE;
2821
a3db102d
BVA		 2822 skspcl->data_buf = skd_alloc_dma(skdev, skdev->databuf_cache,
2823 &skspcl->db_dma_address,
2824 GFP_DMA | __GFP_ZERO,
2825 DMA_BIDIRECTIONAL);
e67f86b3
AB		 2826 if (skspcl->data_buf == NULL) {
2827 rc = -ENOMEM;
2828 goto err_out;
2829 }
2830
a3db102d
BVA		 2831 skspcl->msg_buf = skd_alloc_dma(skdev, skdev->msgbuf_cache,
2832 &skspcl->mb_dma_address,
2833 GFP_DMA | __GFP_ZERO, DMA_TO_DEVICE);
e67f86b3
AB		 2834 if (skspcl->msg_buf == NULL) {
2835 rc = -ENOMEM;
2836 goto err_out;
2837 }
2838
e67f86b3
AB		 2839 skspcl->req.sksg_list = skd_cons_sg_list(skdev, 1,
2840 &skspcl->req.sksg_dma_address);
2841 if (skspcl->req.sksg_list == NULL) {
2842 rc = -ENOMEM;
2843 goto err_out;
2844 }
2845
2846 if (!skd_format_internal_skspcl(skdev)) {
2847 rc = -EINVAL;
2848 goto err_out;
2849 }
2850
2851err_out:
2852 return rc;
2853}
2854
ca33dd92
BVA		 2855static const struct blk_mq_ops skd_mq_ops = {
2856 .queue_rq = skd_mq_queue_rq,
2857 .init_request = skd_init_request,
2858 .exit_request = skd_exit_request,
2859};
2860
e67f86b3
AB		 2861static int skd_cons_disk(struct skd_device *skdev)
2862{
2863 int rc = 0;
2864 struct gendisk *disk;
2865 struct request_queue *q;
2866 unsigned long flags;
2867
2868 disk = alloc_disk(SKD_MINORS_PER_DEVICE);
2869 if (!disk) {
2870 rc = -ENOMEM;
2871 goto err_out;
2872 }
2873
2874 skdev->disk = disk;
2875 sprintf(disk->disk_name, DRV_NAME "%u", skdev->devno);
2876
2877 disk->major = skdev->major;
2878 disk->first_minor = skdev->devno * SKD_MINORS_PER_DEVICE;
2879 disk->fops = &skd_blockdev_ops;
2880 disk->private_data = skdev;
2881
ca33dd92
BVA		 2882 q = NULL;
2883 memset(&skdev->tag_set, 0, sizeof(skdev->tag_set));
2884 skdev->tag_set.ops = &skd_mq_ops;
2885 skdev->tag_set.nr_hw_queues = 1;
2886 skdev->tag_set.queue_depth = skd_max_queue_depth;
2887 skdev->tag_set.cmd_size = sizeof(struct skd_request_context) +
2888 skdev->sgs_per_request * sizeof(struct scatterlist);
2889 skdev->tag_set.numa_node = NUMA_NO_NODE;
2890 skdev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE |
2891 BLK_MQ_F_SG_MERGE |
2892 BLK_ALLOC_POLICY_TO_MQ_FLAG(BLK_TAG_ALLOC_FIFO);
2893 skdev->tag_set.driver_data = skdev;
2894 if (blk_mq_alloc_tag_set(&skdev->tag_set) >= 0) {
2895 q = blk_mq_init_queue(&skdev->tag_set);
2896 if (!q)
2897 blk_mq_free_tag_set(&skdev->tag_set);
2898 }
e67f86b3
AB		 2899 if (!q) {
2900 rc = -ENOMEM;
2901 goto err_out;
2902 }
8fc45044 2903 blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH);
e7278a8b 2904 q->queuedata = skdev;
f18c17c8 2905 q->nr_requests = skd_max_queue_depth / 2;
e67f86b3
AB		 2906
2907 skdev->queue = q;
2908 disk->queue = q;
e67f86b3 2909
6975f732 2910 blk_queue_write_cache(q, true, true);
e67f86b3
AB		 2911 blk_queue_max_segments(q, skdev->sgs_per_request);
2912 blk_queue_max_hw_sectors(q, SKD_N_MAX_SECTORS);
2913
a5c5b392 2914 /* set optimal I/O size to 8KB */
e67f86b3
AB		 2915 blk_queue_io_opt(q, 8192);
2916
e67f86b3 2917 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
b277da0a 2918 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, q);
e67f86b3 2919
a74d5b76
BVA		 2920 blk_queue_rq_timeout(q, 8 * HZ);
2921 blk_queue_rq_timed_out(q, skd_timed_out);
2922 blk_queue_softirq_done(q, skd_softirq_done);
2923
e67f86b3 2924 spin_lock_irqsave(&skdev->lock, flags);
f98806d6 2925 dev_dbg(&skdev->pdev->dev, "stopping queue\n");
ca33dd92 2926 blk_mq_stop_hw_queues(skdev->queue);
e67f86b3
AB		 2927 spin_unlock_irqrestore(&skdev->lock, flags);
2928
2929err_out:
2930 return rc;
2931}
2932
542d7b00
BZ		 2933#define SKD_N_DEV_TABLE 16u
2934static u32 skd_next_devno;
e67f86b3 2935
542d7b00 2936static struct skd_device *skd_construct(struct pci_dev *pdev)
e67f86b3 2937{
542d7b00
BZ		 2938 struct skd_device *skdev;
2939 int blk_major = skd_major;
a3db102d 2940 size_t size;
542d7b00 2941 int rc;
e67f86b3 2942
542d7b00 2943 skdev = kzalloc(sizeof(*skdev), GFP_KERNEL);
e67f86b3 2944
542d7b00 2945 if (!skdev) {
f98806d6 2946 dev_err(&pdev->dev, "memory alloc failure\n");
542d7b00
BZ		 2947 return NULL;
2948 }
e67f86b3 2949
542d7b00
BZ		 2950 skdev->state = SKD_DRVR_STATE_LOAD;
2951 skdev->pdev = pdev;
2952 skdev->devno = skd_next_devno++;
2953 skdev->major = blk_major;
542d7b00 2954 skdev->dev_max_queue_depth = 0;
e67f86b3 2955
542d7b00
BZ		 2956 skdev->num_req_context = skd_max_queue_depth;
2957 skdev->num_fitmsg_context = skd_max_queue_depth;
542d7b00
BZ		 2958 skdev->cur_max_queue_depth = 1;
2959 skdev->queue_low_water_mark = 1;
2960 skdev->proto_ver = 99;
2961 skdev->sgs_per_request = skd_sgs_per_request;
2962 skdev->dbg_level = skd_dbg_level;
e67f86b3 2963
542d7b00
BZ		 2964 spin_lock_init(&skdev->lock);
2965
ca33dd92 2966 INIT_WORK(&skdev->start_queue, skd_start_queue);
542d7b00 2967 INIT_WORK(&skdev->completion_worker, skd_completion_worker);
e67f86b3 2968
a3db102d
BVA		 2969 size = max(SKD_N_FITMSG_BYTES, SKD_N_SPECIAL_FITMSG_BYTES);
2970 skdev->msgbuf_cache = kmem_cache_create("skd-msgbuf", size, 0,
2971 SLAB_HWCACHE_ALIGN, NULL);
2972 if (!skdev->msgbuf_cache)
2973 goto err_out;
2974 WARN_ONCE(kmem_cache_size(skdev->msgbuf_cache) < size,
2975 "skd-msgbuf: %d < %zd\n",
2976 kmem_cache_size(skdev->msgbuf_cache), size);
2977 size = skd_sgs_per_request * sizeof(struct fit_sg_descriptor);
2978 skdev->sglist_cache = kmem_cache_create("skd-sglist", size, 0,
2979 SLAB_HWCACHE_ALIGN, NULL);
2980 if (!skdev->sglist_cache)
2981 goto err_out;
2982 WARN_ONCE(kmem_cache_size(skdev->sglist_cache) < size,
2983 "skd-sglist: %d < %zd\n",
2984 kmem_cache_size(skdev->sglist_cache), size);
2985 size = SKD_N_INTERNAL_BYTES;
2986 skdev->databuf_cache = kmem_cache_create("skd-databuf", size, 0,
2987 SLAB_HWCACHE_ALIGN, NULL);
2988 if (!skdev->databuf_cache)
2989 goto err_out;
2990 WARN_ONCE(kmem_cache_size(skdev->databuf_cache) < size,
2991 "skd-databuf: %d < %zd\n",
2992 kmem_cache_size(skdev->databuf_cache), size);
2993
f98806d6 2994 dev_dbg(&skdev->pdev->dev, "skcomp\n");
542d7b00
BZ		 2995 rc = skd_cons_skcomp(skdev);
2996 if (rc < 0)
2997 goto err_out;
e67f86b3 2998
f98806d6 2999 dev_dbg(&skdev->pdev->dev, "skmsg\n");
542d7b00
BZ		 3000 rc = skd_cons_skmsg(skdev);
3001 if (rc < 0)
3002 goto err_out;
3003
f98806d6 3004 dev_dbg(&skdev->pdev->dev, "sksb\n");
542d7b00
BZ		 3005 rc = skd_cons_sksb(skdev);
3006 if (rc < 0)
3007 goto err_out;
3008
f98806d6 3009 dev_dbg(&skdev->pdev->dev, "disk\n");
542d7b00
BZ		 3010 rc = skd_cons_disk(skdev);
3011 if (rc < 0)
3012 goto err_out;
3013
f98806d6 3014 dev_dbg(&skdev->pdev->dev, "VICTORY\n");
542d7b00
BZ		 3015 return skdev;
3016
3017err_out:
f98806d6 3018 dev_dbg(&skdev->pdev->dev, "construct failed\n");
542d7b00
BZ		 3019 skd_destruct(skdev);
3020 return NULL;
e67f86b3
AB		 3021}
3022
542d7b00
BZ		 3023/*
3024 *****************************************************************************
3025 * DESTRUCT (FREE)
3026 *****************************************************************************
3027 */
3028
e67f86b3
AB		 3029static void skd_free_skcomp(struct skd_device *skdev)
3030{
7f13bdad
BVA		 3031 if (skdev->skcomp_table)
3032 pci_free_consistent(skdev->pdev, SKD_SKCOMP_SIZE,
e67f86b3 3033 skdev->skcomp_table, skdev->cq_dma_address);
e67f86b3
AB		 3034
3035 skdev->skcomp_table = NULL;
3036 skdev->cq_dma_address = 0;
3037}
3038
3039static void skd_free_skmsg(struct skd_device *skdev)
3040{
3041 u32 i;
3042
3043 if (skdev->skmsg_table == NULL)
3044 return;
3045
3046 for (i = 0; i < skdev->num_fitmsg_context; i++) {
3047 struct skd_fitmsg_context *skmsg;
3048
3049 skmsg = &skdev->skmsg_table[i];
3050
3051 if (skmsg->msg_buf != NULL) {
e67f86b3
AB		 3052 pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES,
3053 skmsg->msg_buf,
3054 skmsg->mb_dma_address);
3055 }
3056 skmsg->msg_buf = NULL;
3057 skmsg->mb_dma_address = 0;
3058 }
3059
3060 kfree(skdev->skmsg_table);
3061 skdev->skmsg_table = NULL;
3062}
3063
e67f86b3
AB		 3064static void skd_free_sksb(struct skd_device *skdev)
3065{
a3db102d 3066 struct skd_special_context *skspcl = &skdev->internal_skspcl;
e67f86b3 3067
a3db102d
BVA		 3068 skd_free_dma(skdev, skdev->databuf_cache, skspcl->data_buf,
3069 skspcl->db_dma_address, DMA_BIDIRECTIONAL);
e67f86b3
AB		 3070
3071 skspcl->data_buf = NULL;
3072 skspcl->db_dma_address = 0;
3073
a3db102d
BVA		 3074 skd_free_dma(skdev, skdev->msgbuf_cache, skspcl->msg_buf,
3075 skspcl->mb_dma_address, DMA_TO_DEVICE);
e67f86b3
AB		 3076
3077 skspcl->msg_buf = NULL;
3078 skspcl->mb_dma_address = 0;
3079
a3db102d 3080 skd_free_sg_list(skdev, skspcl->req.sksg_list,
e67f86b3
AB		 3081 skspcl->req.sksg_dma_address);
3082
3083 skspcl->req.sksg_list = NULL;
3084 skspcl->req.sksg_dma_address = 0;
3085}
3086
e67f86b3
AB		 3087static void skd_free_disk(struct skd_device *skdev)
3088{
3089 struct gendisk *disk = skdev->disk;
3090
7277cc67
BVA		 3091 if (disk && (disk->flags & GENHD_FL_UP))
3092 del_gendisk(disk);
3093
3094 if (skdev->queue) {
3095 blk_cleanup_queue(skdev->queue);
3096 skdev->queue = NULL;
3097 disk->queue = NULL;
e67f86b3 3098 }
7277cc67 3099
ca33dd92
BVA		 3100 if (skdev->tag_set.tags)
3101 blk_mq_free_tag_set(&skdev->tag_set);
3102
7277cc67 3103 put_disk(disk);
e67f86b3
AB		 3104 skdev->disk = NULL;
3105}
3106
542d7b00
BZ		 3107static void skd_destruct(struct skd_device *skdev)
3108{
3109 if (skdev == NULL)
3110 return;
3111
ca33dd92
BVA		 3112 cancel_work_sync(&skdev->start_queue);
3113
f98806d6 3114 dev_dbg(&skdev->pdev->dev, "disk\n");
542d7b00
BZ		 3115 skd_free_disk(skdev);
3116
f98806d6 3117 dev_dbg(&skdev->pdev->dev, "sksb\n");
542d7b00
BZ		 3118 skd_free_sksb(skdev);
3119
f98806d6 3120 dev_dbg(&skdev->pdev->dev, "skmsg\n");
542d7b00 3121 skd_free_skmsg(skdev);
e67f86b3 3122
f98806d6 3123 dev_dbg(&skdev->pdev->dev, "skcomp\n");
542d7b00
BZ		 3124 skd_free_skcomp(skdev);
3125
a3db102d
BVA		 3126 kmem_cache_destroy(skdev->databuf_cache);
3127 kmem_cache_destroy(skdev->sglist_cache);
3128 kmem_cache_destroy(skdev->msgbuf_cache);
3129
f98806d6 3130 dev_dbg(&skdev->pdev->dev, "skdev\n");
542d7b00
BZ		 3131 kfree(skdev);
3132}
e67f86b3
AB		 3133
3134/*
3135 *****************************************************************************
3136 * BLOCK DEVICE (BDEV) GLUE
3137 *****************************************************************************
3138 */
3139
3140static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
3141{
3142 struct skd_device *skdev;
3143 u64 capacity;
3144
3145 skdev = bdev->bd_disk->private_data;
3146
f98806d6
BVA		 3147 dev_dbg(&skdev->pdev->dev, "%s: CMD[%s] getgeo device\n",
3148 bdev->bd_disk->disk_name, current->comm);
e67f86b3
AB		 3149
3150 if (skdev->read_cap_is_valid) {
3151 capacity = get_capacity(skdev->disk);
3152 geo->heads = 64;
3153 geo->sectors = 255;
3154 geo->cylinders = (capacity) / (255 * 64);
3155
3156 return 0;
3157 }
3158 return -EIO;
3159}
3160
0d52c756 3161static int skd_bdev_attach(struct device *parent, struct skd_device *skdev)
e67f86b3 3162{
f98806d6 3163 dev_dbg(&skdev->pdev->dev, "add_disk\n");
0d52c756 3164 device_add_disk(parent, skdev->disk);
e67f86b3
AB		 3165 return 0;
3166}
3167
3168static const struct block_device_operations skd_blockdev_ops = {
3169 .owner = THIS_MODULE,
e67f86b3
AB		 3170 .getgeo = skd_bdev_getgeo,
3171};
3172
e67f86b3
AB		 3173/*
3174 *****************************************************************************
3175 * PCIe DRIVER GLUE
3176 *****************************************************************************
3177 */
3178
9baa3c34 3179static const struct pci_device_id skd_pci_tbl[] = {
e67f86b3
AB		 3180 { PCI_VENDOR_ID_STEC, PCI_DEVICE_ID_S1120,
3181 PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
3182 { 0 } /* terminate list */
3183};
3184
3185MODULE_DEVICE_TABLE(pci, skd_pci_tbl);
3186
3187static char *skd_pci_info(struct skd_device *skdev, char *str)
3188{
3189 int pcie_reg;
3190
3191 strcpy(str, "PCIe (");
3192 pcie_reg = pci_find_capability(skdev->pdev, PCI_CAP_ID_EXP);
3193
3194 if (pcie_reg) {
3195
3196 char lwstr[6];
3197 uint16_t pcie_lstat, lspeed, lwidth;
3198
3199 pcie_reg += 0x12;
3200 pci_read_config_word(skdev->pdev, pcie_reg, &pcie_lstat);
3201 lspeed = pcie_lstat & (0xF);
3202 lwidth = (pcie_lstat & 0x3F0) >> 4;
3203
3204 if (lspeed == 1)
3205 strcat(str, "2.5GT/s ");
3206 else if (lspeed == 2)
3207 strcat(str, "5.0GT/s ");
3208 else
3209 strcat(str, "<unknown> ");
3210 snprintf(lwstr, sizeof(lwstr), "%dX)", lwidth);
3211 strcat(str, lwstr);
3212 }
3213 return str;
3214}
3215
3216static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3217{
3218 int i;
3219 int rc = 0;
3220 char pci_str[32];
3221 struct skd_device *skdev;
3222
f98806d6
BVA		 3223 dev_info(&pdev->dev, "STEC s1120 Driver(%s) version %s-b%s\n",
3224 DRV_NAME, DRV_VERSION, DRV_BUILD_ID);
3225 dev_info(&pdev->dev, "vendor=%04X device=%04x\n", pdev->vendor,
3226 pdev->device);
e67f86b3
AB		 3227
3228 rc = pci_enable_device(pdev);
3229 if (rc)
3230 return rc;
3231 rc = pci_request_regions(pdev, DRV_NAME);
3232 if (rc)
3233 goto err_out;
3234 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
3235 if (!rc) {
3236 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
f98806d6
BVA		 3237 dev_err(&pdev->dev, "consistent DMA mask error %d\n",
3238 rc);
e67f86b3
AB		 3239 }
3240 } else {
f98806d6 3241 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
e67f86b3 3242 if (rc) {
f98806d6 3243 dev_err(&pdev->dev, "DMA mask error %d\n", rc);
e67f86b3
AB		 3244 goto err_out_regions;
3245 }
3246 }
3247
b8df6647
BZ		 3248 if (!skd_major) {
3249 rc = register_blkdev(0, DRV_NAME);
3250 if (rc < 0)
3251 goto err_out_regions;
3252 BUG_ON(!rc);
3253 skd_major = rc;
3254 }
3255
e67f86b3 3256 skdev = skd_construct(pdev);
1762b57f
WY		 3257 if (skdev == NULL) {
3258 rc = -ENOMEM;
e67f86b3 3259 goto err_out_regions;
1762b57f 3260 }
e67f86b3
AB		 3261
3262 skd_pci_info(skdev, pci_str);
f98806d6 3263 dev_info(&pdev->dev, "%s 64bit\n", pci_str);
e67f86b3
AB		 3264
3265 pci_set_master(pdev);
3266 rc = pci_enable_pcie_error_reporting(pdev);
3267 if (rc) {
f98806d6
BVA		 3268 dev_err(&pdev->dev,
3269 "bad enable of PCIe error reporting rc=%d\n", rc);
e67f86b3
AB		 3270 skdev->pcie_error_reporting_is_enabled = 0;
3271 } else
3272 skdev->pcie_error_reporting_is_enabled = 1;
3273
e67f86b3 3274 pci_set_drvdata(pdev, skdev);
ebedd16d 3275
e67f86b3
AB		 3276 for (i = 0; i < SKD_MAX_BARS; i++) {
3277 skdev->mem_phys[i] = pci_resource_start(pdev, i);
3278 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
3279 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
3280 skdev->mem_size[i]);
3281 if (!skdev->mem_map[i]) {
f98806d6
BVA		 3282 dev_err(&pdev->dev,
3283 "Unable to map adapter memory!\n");
e67f86b3
AB		 3284 rc = -ENODEV;
3285 goto err_out_iounmap;
3286 }
f98806d6
BVA		 3287 dev_dbg(&pdev->dev, "mem_map=%p, phyd=%016llx, size=%d\n",
3288 skdev->mem_map[i], (uint64_t)skdev->mem_phys[i],
3289 skdev->mem_size[i]);
e67f86b3
AB		 3290 }
3291
3292 rc = skd_acquire_irq(skdev);
3293 if (rc) {
f98806d6 3294 dev_err(&pdev->dev, "interrupt resource error %d\n", rc);
e67f86b3
AB		 3295 goto err_out_iounmap;
3296 }
3297
3298 rc = skd_start_timer(skdev);
3299 if (rc)
3300 goto err_out_timer;
3301
3302 init_waitqueue_head(&skdev->waitq);
3303
3304 skd_start_device(skdev);
3305
3306 rc = wait_event_interruptible_timeout(skdev->waitq,
3307 (skdev->gendisk_on),
3308 (SKD_START_WAIT_SECONDS * HZ));
3309 if (skdev->gendisk_on > 0) {
3310 /* device came on-line after reset */
0d52c756 3311 skd_bdev_attach(&pdev->dev, skdev);
e67f86b3
AB		 3312 rc = 0;
3313 } else {
3314 /* we timed out, something is wrong with the device,
3315 don't add the disk structure */
f98806d6
BVA		 3316 dev_err(&pdev->dev, "error: waiting for s1120 timed out %d!\n",
3317 rc);
e67f86b3
AB		 3318 /* in case of no error; we timeout with ENXIO */
3319 if (!rc)
3320 rc = -ENXIO;
3321 goto err_out_timer;
3322 }
3323
e67f86b3
AB		 3324 return rc;
3325
3326err_out_timer:
3327 skd_stop_device(skdev);
3328 skd_release_irq(skdev);
3329
3330err_out_iounmap:
3331 for (i = 0; i < SKD_MAX_BARS; i++)
3332 if (skdev->mem_map[i])
3333 iounmap(skdev->mem_map[i]);
3334
3335 if (skdev->pcie_error_reporting_is_enabled)
3336 pci_disable_pcie_error_reporting(pdev);
3337
3338 skd_destruct(skdev);
3339
3340err_out_regions:
3341 pci_release_regions(pdev);
3342
3343err_out:
3344 pci_disable_device(pdev);
3345 pci_set_drvdata(pdev, NULL);
3346 return rc;
3347}
3348
3349static void skd_pci_remove(struct pci_dev *pdev)
3350{
3351 int i;
3352 struct skd_device *skdev;
3353
3354 skdev = pci_get_drvdata(pdev);
3355 if (!skdev) {
f98806d6 3356 dev_err(&pdev->dev, "no device data for PCI\n");
e67f86b3
AB		 3357 return;
3358 }
3359 skd_stop_device(skdev);
3360 skd_release_irq(skdev);
3361
3362 for (i = 0; i < SKD_MAX_BARS; i++)
3363 if (skdev->mem_map[i])
4854afe3 3364 iounmap(skdev->mem_map[i]);
e67f86b3
AB		 3365
3366 if (skdev->pcie_error_reporting_is_enabled)
3367 pci_disable_pcie_error_reporting(pdev);
3368
3369 skd_destruct(skdev);
3370
3371 pci_release_regions(pdev);
3372 pci_disable_device(pdev);
3373 pci_set_drvdata(pdev, NULL);
3374
3375 return;
3376}
3377
3378static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
3379{
3380 int i;
3381 struct skd_device *skdev;
3382
3383 skdev = pci_get_drvdata(pdev);
3384 if (!skdev) {
f98806d6 3385 dev_err(&pdev->dev, "no device data for PCI\n");
e67f86b3
AB		 3386 return -EIO;
3387 }
3388
3389 skd_stop_device(skdev);
3390
3391 skd_release_irq(skdev);
3392
3393 for (i = 0; i < SKD_MAX_BARS; i++)
3394 if (skdev->mem_map[i])
4854afe3 3395 iounmap(skdev->mem_map[i]);
e67f86b3
AB		 3396
3397 if (skdev->pcie_error_reporting_is_enabled)
3398 pci_disable_pcie_error_reporting(pdev);
3399
3400 pci_release_regions(pdev);
3401 pci_save_state(pdev);
3402 pci_disable_device(pdev);
3403 pci_set_power_state(pdev, pci_choose_state(pdev, state));
3404 return 0;
3405}
3406
3407static int skd_pci_resume(struct pci_dev *pdev)
3408{
3409 int i;
3410 int rc = 0;
3411 struct skd_device *skdev;
3412
3413 skdev = pci_get_drvdata(pdev);
3414 if (!skdev) {
f98806d6 3415 dev_err(&pdev->dev, "no device data for PCI\n");
e67f86b3
AB		 3416 return -1;
3417 }
3418
3419 pci_set_power_state(pdev, PCI_D0);
3420 pci_enable_wake(pdev, PCI_D0, 0);
3421 pci_restore_state(pdev);
3422
3423 rc = pci_enable_device(pdev);
3424 if (rc)
3425 return rc;
3426 rc = pci_request_regions(pdev, DRV_NAME);
3427 if (rc)
3428 goto err_out;
3429 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
3430 if (!rc) {
3431 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
3432
f98806d6
BVA		 3433 dev_err(&pdev->dev, "consistent DMA mask error %d\n",
3434 rc);
e67f86b3
AB		 3435 }
3436 } else {
3437 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3438 if (rc) {
3439
f98806d6 3440 dev_err(&pdev->dev, "DMA mask error %d\n", rc);
e67f86b3
AB		 3441 goto err_out_regions;
3442 }
3443 }
3444
3445 pci_set_master(pdev);
3446 rc = pci_enable_pcie_error_reporting(pdev);
3447 if (rc) {
f98806d6
BVA		 3448 dev_err(&pdev->dev,
3449 "bad enable of PCIe error reporting rc=%d\n", rc);
e67f86b3
AB		 3450 skdev->pcie_error_reporting_is_enabled = 0;
3451 } else
3452 skdev->pcie_error_reporting_is_enabled = 1;
3453
3454 for (i = 0; i < SKD_MAX_BARS; i++) {
3455
3456 skdev->mem_phys[i] = pci_resource_start(pdev, i);
3457 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
3458 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
3459 skdev->mem_size[i]);
3460 if (!skdev->mem_map[i]) {
f98806d6 3461 dev_err(&pdev->dev, "Unable to map adapter memory!\n");
e67f86b3
AB		 3462 rc = -ENODEV;
3463 goto err_out_iounmap;
3464 }
f98806d6
BVA		 3465 dev_dbg(&pdev->dev, "mem_map=%p, phyd=%016llx, size=%d\n",
3466 skdev->mem_map[i], (uint64_t)skdev->mem_phys[i],
3467 skdev->mem_size[i]);
e67f86b3
AB		 3468 }
3469 rc = skd_acquire_irq(skdev);
3470 if (rc) {
f98806d6 3471 dev_err(&pdev->dev, "interrupt resource error %d\n", rc);
e67f86b3
AB		 3472 goto err_out_iounmap;
3473 }
3474
3475 rc = skd_start_timer(skdev);
3476 if (rc)
3477 goto err_out_timer;
3478
3479 init_waitqueue_head(&skdev->waitq);
3480
3481 skd_start_device(skdev);
3482
3483 return rc;
3484
3485err_out_timer:
3486 skd_stop_device(skdev);
3487 skd_release_irq(skdev);
3488
3489err_out_iounmap:
3490 for (i = 0; i < SKD_MAX_BARS; i++)
3491 if (skdev->mem_map[i])
3492 iounmap(skdev->mem_map[i]);
3493
3494 if (skdev->pcie_error_reporting_is_enabled)
3495 pci_disable_pcie_error_reporting(pdev);
3496
3497err_out_regions:
3498 pci_release_regions(pdev);
3499
3500err_out:
3501 pci_disable_device(pdev);
3502 return rc;
3503}
3504
3505static void skd_pci_shutdown(struct pci_dev *pdev)
3506{
3507 struct skd_device *skdev;
3508
f98806d6 3509 dev_err(&pdev->dev, "%s called\n", __func__);
e67f86b3
AB		 3510
3511 skdev = pci_get_drvdata(pdev);
3512 if (!skdev) {
f98806d6 3513 dev_err(&pdev->dev, "no device data for PCI\n");
e67f86b3
AB		 3514 return;
3515 }
3516
f98806d6 3517 dev_err(&pdev->dev, "calling stop\n");
e67f86b3
AB		 3518 skd_stop_device(skdev);
3519}
3520
3521static struct pci_driver skd_driver = {
3522 .name = DRV_NAME,
3523 .id_table = skd_pci_tbl,
3524 .probe = skd_pci_probe,
3525 .remove = skd_pci_remove,
3526 .suspend = skd_pci_suspend,
3527 .resume = skd_pci_resume,
3528 .shutdown = skd_pci_shutdown,
3529};
3530
3531/*
3532 *****************************************************************************
3533 * LOGGING SUPPORT
3534 *****************************************************************************
3535 */
3536
e67f86b3
AB		 3537const char *skd_drive_state_to_str(int state)
3538{
3539 switch (state) {
3540 case FIT_SR_DRIVE_OFFLINE:
3541 return "OFFLINE";
3542 case FIT_SR_DRIVE_INIT:
3543 return "INIT";
3544 case FIT_SR_DRIVE_ONLINE:
3545 return "ONLINE";
3546 case FIT_SR_DRIVE_BUSY:
3547 return "BUSY";
3548 case FIT_SR_DRIVE_FAULT:
3549 return "FAULT";
3550 case FIT_SR_DRIVE_DEGRADED:
3551 return "DEGRADED";
3552 case FIT_SR_PCIE_LINK_DOWN:
3553 return "INK_DOWN";
3554 case FIT_SR_DRIVE_SOFT_RESET:
3555 return "SOFT_RESET";
3556 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
3557 return "NEED_FW";
3558 case FIT_SR_DRIVE_INIT_FAULT:
3559 return "INIT_FAULT";
3560 case FIT_SR_DRIVE_BUSY_SANITIZE:
3561 return "BUSY_SANITIZE";
3562 case FIT_SR_DRIVE_BUSY_ERASE:
3563 return "BUSY_ERASE";
3564 case FIT_SR_DRIVE_FW_BOOTING:
3565 return "FW_BOOTING";
3566 default:
3567 return "???";
3568 }
3569}
3570
3571const char *skd_skdev_state_to_str(enum skd_drvr_state state)
3572{
3573 switch (state) {
3574 case SKD_DRVR_STATE_LOAD:
3575 return "LOAD";
3576 case SKD_DRVR_STATE_IDLE:
3577 return "IDLE";
3578 case SKD_DRVR_STATE_BUSY:
3579 return "BUSY";
3580 case SKD_DRVR_STATE_STARTING:
3581 return "STARTING";
3582 case SKD_DRVR_STATE_ONLINE:
3583 return "ONLINE";
3584 case SKD_DRVR_STATE_PAUSING:
3585 return "PAUSING";
3586 case SKD_DRVR_STATE_PAUSED:
3587 return "PAUSED";
e67f86b3
AB		 3588 case SKD_DRVR_STATE_RESTARTING:
3589 return "RESTARTING";
3590 case SKD_DRVR_STATE_RESUMING:
3591 return "RESUMING";
3592 case SKD_DRVR_STATE_STOPPING:
3593 return "STOPPING";
3594 case SKD_DRVR_STATE_SYNCING:
3595 return "SYNCING";
3596 case SKD_DRVR_STATE_FAULT:
3597 return "FAULT";
3598 case SKD_DRVR_STATE_DISAPPEARED:
3599 return "DISAPPEARED";
3600 case SKD_DRVR_STATE_BUSY_ERASE:
3601 return "BUSY_ERASE";
3602 case SKD_DRVR_STATE_BUSY_SANITIZE:
3603 return "BUSY_SANITIZE";
3604 case SKD_DRVR_STATE_BUSY_IMMINENT:
3605 return "BUSY_IMMINENT";
3606 case SKD_DRVR_STATE_WAIT_BOOT:
3607 return "WAIT_BOOT";
3608
3609 default:
3610 return "???";
3611 }
3612}
3613
a26ba7fa 3614static const char *skd_skreq_state_to_str(enum skd_req_state state)
e67f86b3
AB		 3615{
3616 switch (state) {
3617 case SKD_REQ_STATE_IDLE:
3618 return "IDLE";
3619 case SKD_REQ_STATE_SETUP:
3620 return "SETUP";
3621 case SKD_REQ_STATE_BUSY:
3622 return "BUSY";
3623 case SKD_REQ_STATE_COMPLETED:
3624 return "COMPLETED";
3625 case SKD_REQ_STATE_TIMEOUT:
3626 return "TIMEOUT";
e67f86b3
AB		 3627 default:
3628 return "???";
3629 }
3630}
3631
3632static void skd_log_skdev(struct skd_device *skdev, const char *event)
3633{
f98806d6
BVA		 3634 dev_dbg(&skdev->pdev->dev, "skdev=%p event='%s'\n", skdev, event);
3635 dev_dbg(&skdev->pdev->dev, " drive_state=%s(%d) driver_state=%s(%d)\n",
3636 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
3637 skd_skdev_state_to_str(skdev->state), skdev->state);
3638 dev_dbg(&skdev->pdev->dev, " busy=%d limit=%d dev=%d lowat=%d\n",
d4d0f5fc 3639 skd_in_flight(skdev), skdev->cur_max_queue_depth,
f98806d6 3640 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
a74d5b76
BVA		 3641 dev_dbg(&skdev->pdev->dev, " cycle=%d cycle_ix=%d\n",
3642 skdev->skcomp_cycle, skdev->skcomp_ix);
e67f86b3
AB		 3643}
3644
e67f86b3
AB		 3645static void skd_log_skreq(struct skd_device *skdev,
3646 struct skd_request_context *skreq, const char *event)
3647{
e7278a8b
BVA		 3648 struct request *req = blk_mq_rq_from_pdu(skreq);
3649 u32 lba = blk_rq_pos(req);
3650 u32 count = blk_rq_sectors(req);
3651
f98806d6
BVA		 3652 dev_dbg(&skdev->pdev->dev, "skreq=%p event='%s'\n", skreq, event);
3653 dev_dbg(&skdev->pdev->dev, " state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
3654 skd_skreq_state_to_str(skreq->state), skreq->state, skreq->id,
3655 skreq->fitmsg_id);
a74d5b76
BVA		 3656 dev_dbg(&skdev->pdev->dev, " sg_dir=%d n_sg=%d\n",
3657 skreq->data_dir, skreq->n_sg);
ca33dd92 3658
e7278a8b
BVA		 3659 dev_dbg(&skdev->pdev->dev,
3660 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n", req, lba, lba,
3661 count, count, (int)rq_data_dir(req));
e67f86b3
AB		 3662}
3663
3664/*
3665 *****************************************************************************
3666 * MODULE GLUE
3667 *****************************************************************************
3668 */
3669
3670static int __init skd_init(void)
3671{
16a70534
BVA		 3672 BUILD_BUG_ON(sizeof(struct fit_completion_entry_v1) != 8);
3673 BUILD_BUG_ON(sizeof(struct fit_comp_error_info) != 32);
3674 BUILD_BUG_ON(sizeof(struct skd_command_header) != 16);
3675 BUILD_BUG_ON(sizeof(struct skd_scsi_request) != 32);
3676 BUILD_BUG_ON(sizeof(struct driver_inquiry_data) != 44);
d891fe60
BVA		 3677 BUILD_BUG_ON(offsetof(struct skd_msg_buf, fmh) != 0);
3678 BUILD_BUG_ON(offsetof(struct skd_msg_buf, scsi) != 64);
3679 BUILD_BUG_ON(sizeof(struct skd_msg_buf) != SKD_N_FITMSG_BYTES);
2da7b403 3680
e67f86b3
AB		 3681 pr_info(PFX " v%s-b%s loaded\n", DRV_VERSION, DRV_BUILD_ID);
3682
3683 switch (skd_isr_type) {
3684 case SKD_IRQ_LEGACY:
3685 case SKD_IRQ_MSI:
3686 case SKD_IRQ_MSIX:
3687 break;
3688 default:
fbed149a 3689 pr_err(PFX "skd_isr_type %d invalid, re-set to %d\n",
e67f86b3
AB		 3690 skd_isr_type, SKD_IRQ_DEFAULT);
3691 skd_isr_type = SKD_IRQ_DEFAULT;
3692 }
3693
fbed149a
BZ		 3694 if (skd_max_queue_depth < 1 ||
3695 skd_max_queue_depth > SKD_MAX_QUEUE_DEPTH) {
3696 pr_err(PFX "skd_max_queue_depth %d invalid, re-set to %d\n",
e67f86b3
AB		 3697 skd_max_queue_depth, SKD_MAX_QUEUE_DEPTH_DEFAULT);
3698 skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
3699 }
3700
2da7b403
BVA		 3701 if (skd_max_req_per_msg < 1 ||
3702 skd_max_req_per_msg > SKD_MAX_REQ_PER_MSG) {
fbed149a 3703 pr_err(PFX "skd_max_req_per_msg %d invalid, re-set to %d\n",
e67f86b3
AB		 3704 skd_max_req_per_msg, SKD_MAX_REQ_PER_MSG_DEFAULT);
3705 skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
3706 }
3707
3708 if (skd_sgs_per_request < 1 || skd_sgs_per_request > 4096) {
fbed149a 3709 pr_err(PFX "skd_sg_per_request %d invalid, re-set to %d\n",
e67f86b3
AB		 3710 skd_sgs_per_request, SKD_N_SG_PER_REQ_DEFAULT);
3711 skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
3712 }
3713
3714 if (skd_dbg_level < 0 || skd_dbg_level > 2) {
fbed149a 3715 pr_err(PFX "skd_dbg_level %d invalid, re-set to %d\n",
e67f86b3
AB		 3716 skd_dbg_level, 0);
3717 skd_dbg_level = 0;
3718 }
3719
3720 if (skd_isr_comp_limit < 0) {
fbed149a 3721 pr_err(PFX "skd_isr_comp_limit %d invalid, set to %d\n",
e67f86b3
AB		 3722 skd_isr_comp_limit, 0);
3723 skd_isr_comp_limit = 0;
3724 }
3725
b8df6647 3726 return pci_register_driver(&skd_driver);
e67f86b3
AB		 3727}
3728
3729static void __exit skd_exit(void)
3730{
3731 pr_info(PFX " v%s-b%s unloading\n", DRV_VERSION, DRV_BUILD_ID);
3732
e67f86b3 3733 pci_unregister_driver(&skd_driver);
b8df6647
BZ		 3734
3735 if (skd_major)
3736 unregister_blkdev(skd_major, DRV_NAME);
e67f86b3
AB		 3737}
3738
e67f86b3
AB		 3739module_init(skd_init);
3740module_exit(skd_exit);
Ubuntu Hirsute Linux kernel mirror