]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/block/skd_main.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
skd: rip out bio path
[mirror_ubuntu-artful-kernel.git] / drivers / block / skd_main.c
CommitLineData
e67f86b3
AB		 1/* Copyright 2012 STEC, Inc.
2 *
3 * This file is licensed under the terms of the 3-clause
4 * BSD License (http://opensource.org/licenses/BSD-3-Clause)
5 * or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
6 * at your option. Both licenses are also available in the LICENSE file
7 * distributed with this project. This file may not be copied, modified,
8 * or distributed except in accordance with those terms.
9 * Gordoni Waidhofer <gwaidhofer@stec-inc.com>
10 * Initial Driver Design!
11 * Thomas Swann <tswann@stec-inc.com>
12 * Interrupt handling.
13 * Ramprasad Chinthekindi <rchinthekindi@stec-inc.com>
14 * biomode implementation.
15 * Akhil Bhansali <abhansali@stec-inc.com>
16 * Added support for DISCARD / FLUSH and FUA.
17 */
18
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/init.h>
22#include <linux/pci.h>
23#include <linux/slab.h>
24#include <linux/spinlock.h>
25#include <linux/blkdev.h>
26#include <linux/sched.h>
27#include <linux/interrupt.h>
28#include <linux/compiler.h>
29#include <linux/workqueue.h>
30#include <linux/bitops.h>
31#include <linux/delay.h>
32#include <linux/time.h>
33#include <linux/hdreg.h>
34#include <linux/dma-mapping.h>
35#include <linux/completion.h>
36#include <linux/scatterlist.h>
37#include <linux/version.h>
38#include <linux/err.h>
39#include <linux/scatterlist.h>
40#include <linux/aer.h>
41#include <linux/ctype.h>
42#include <linux/wait.h>
43#include <linux/uio.h>
44#include <scsi/scsi.h>
45#include <scsi/scsi_host.h>
46#include <scsi/scsi_tcq.h>
47#include <scsi/scsi_cmnd.h>
48#include <scsi/sg.h>
49#include <linux/io.h>
50#include <linux/uaccess.h>
51#include <asm-generic/unaligned.h>
52
53#include "skd_s1120.h"
54
55static int skd_dbg_level;
56static int skd_isr_comp_limit = 4;
57
58enum {
59 STEC_LINK_2_5GTS = 0,
60 STEC_LINK_5GTS = 1,
61 STEC_LINK_8GTS = 2,
62 STEC_LINK_UNKNOWN = 0xFF
63};
64
65enum {
66 SKD_FLUSH_INITIALIZER,
67 SKD_FLUSH_ZERO_SIZE_FIRST,
68 SKD_FLUSH_DATA_SECOND,
69};
70
e67f86b3
AB		 71#define SKD_ASSERT(expr) \
72 do { \
73 if (unlikely(!(expr))) { \
74 pr_err("Assertion failed! %s,%s,%s,line=%d\n", \
75 # expr, __FILE__, __func__, __LINE__); \
76 } \
77 } while (0)
78
e67f86b3
AB		 79#define DRV_NAME "skd"
80#define DRV_VERSION "2.2.1"
81#define DRV_BUILD_ID "0260"
82#define PFX DRV_NAME ": "
83#define DRV_BIN_VERSION 0x100
84#define DRV_VER_COMPL "2.2.1." DRV_BUILD_ID
85
86MODULE_AUTHOR("bug-reports: support@stec-inc.com");
87MODULE_LICENSE("Dual BSD/GPL");
88
89MODULE_DESCRIPTION("STEC s1120 PCIe SSD block/BIO driver (b" DRV_BUILD_ID ")");
90MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
91
92#define PCI_VENDOR_ID_STEC 0x1B39
93#define PCI_DEVICE_ID_S1120 0x0001
94
95#define SKD_FUA_NV (1 << 1)
96#define SKD_MINORS_PER_DEVICE 16
97
98#define SKD_MAX_QUEUE_DEPTH 200u
99
100#define SKD_PAUSE_TIMEOUT (5 * 1000)
101
102#define SKD_N_FITMSG_BYTES (512u)
103
104#define SKD_N_SPECIAL_CONTEXT 32u
105#define SKD_N_SPECIAL_FITMSG_BYTES (128u)
106
107/* SG elements are 32 bytes, so we can make this 4096 and still be under the
108 * 128KB limit. That allows 4096*4K = 16M xfer size
109 */
110#define SKD_N_SG_PER_REQ_DEFAULT 256u
111#define SKD_N_SG_PER_SPECIAL 256u
112
113#define SKD_N_COMPLETION_ENTRY 256u
114#define SKD_N_READ_CAP_BYTES (8u)
115
116#define SKD_N_INTERNAL_BYTES (512u)
117
118/* 5 bits of uniqifier, 0xF800 */
119#define SKD_ID_INCR (0x400)
120#define SKD_ID_TABLE_MASK (3u << 8u)
121#define SKD_ID_RW_REQUEST (0u << 8u)
122#define SKD_ID_INTERNAL (1u << 8u)
123#define SKD_ID_SPECIAL_REQUEST (2u << 8u)
124#define SKD_ID_FIT_MSG (3u << 8u)
125#define SKD_ID_SLOT_MASK 0x00FFu
126#define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
127
128#define SKD_N_TIMEOUT_SLOT 4u
129#define SKD_TIMEOUT_SLOT_MASK 3u
130
131#define SKD_N_MAX_SECTORS 2048u
132
133#define SKD_MAX_RETRIES 2u
134
135#define SKD_TIMER_SECONDS(seconds) (seconds)
136#define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
137
138#define INQ_STD_NBYTES 36
139#define SKD_DISCARD_CDB_LENGTH 24
140
141enum skd_drvr_state {
142 SKD_DRVR_STATE_LOAD,
143 SKD_DRVR_STATE_IDLE,
144 SKD_DRVR_STATE_BUSY,
145 SKD_DRVR_STATE_STARTING,
146 SKD_DRVR_STATE_ONLINE,
147 SKD_DRVR_STATE_PAUSING,
148 SKD_DRVR_STATE_PAUSED,
149 SKD_DRVR_STATE_DRAINING_TIMEOUT,
150 SKD_DRVR_STATE_RESTARTING,
151 SKD_DRVR_STATE_RESUMING,
152 SKD_DRVR_STATE_STOPPING,
153 SKD_DRVR_STATE_FAULT,
154 SKD_DRVR_STATE_DISAPPEARED,
155 SKD_DRVR_STATE_PROTOCOL_MISMATCH,
156 SKD_DRVR_STATE_BUSY_ERASE,
157 SKD_DRVR_STATE_BUSY_SANITIZE,
158 SKD_DRVR_STATE_BUSY_IMMINENT,
159 SKD_DRVR_STATE_WAIT_BOOT,
160 SKD_DRVR_STATE_SYNCING,
161};
162
163#define SKD_WAIT_BOOT_TIMO SKD_TIMER_SECONDS(90u)
164#define SKD_STARTING_TIMO SKD_TIMER_SECONDS(8u)
165#define SKD_RESTARTING_TIMO SKD_TIMER_MINUTES(4u)
166#define SKD_DRAINING_TIMO SKD_TIMER_SECONDS(6u)
167#define SKD_BUSY_TIMO SKD_TIMER_MINUTES(20u)
168#define SKD_STARTED_BUSY_TIMO SKD_TIMER_SECONDS(60u)
169#define SKD_START_WAIT_SECONDS 90u
170
171enum skd_req_state {
172 SKD_REQ_STATE_IDLE,
173 SKD_REQ_STATE_SETUP,
174 SKD_REQ_STATE_BUSY,
175 SKD_REQ_STATE_COMPLETED,
176 SKD_REQ_STATE_TIMEOUT,
177 SKD_REQ_STATE_ABORTED,
178};
179
180enum skd_fit_msg_state {
181 SKD_MSG_STATE_IDLE,
182 SKD_MSG_STATE_BUSY,
183};
184
185enum skd_check_status_action {
186 SKD_CHECK_STATUS_REPORT_GOOD,
187 SKD_CHECK_STATUS_REPORT_SMART_ALERT,
188 SKD_CHECK_STATUS_REQUEUE_REQUEST,
189 SKD_CHECK_STATUS_REPORT_ERROR,
190 SKD_CHECK_STATUS_BUSY_IMMINENT,
191};
192
193struct skd_fitmsg_context {
194 enum skd_fit_msg_state state;
195
196 struct skd_fitmsg_context *next;
197
198 u32 id;
199 u16 outstanding;
200
201 u32 length;
202 u32 offset;
203
204 u8 *msg_buf;
205 dma_addr_t mb_dma_address;
206};
207
208struct skd_request_context {
209 enum skd_req_state state;
210
211 struct skd_request_context *next;
212
213 u16 id;
214 u32 fitmsg_id;
215
216 struct request *req;
217 struct bio *bio;
218 unsigned long start_time;
219 u8 flush_cmd;
220 u8 discard_page;
221
222 u32 timeout_stamp;
223 u8 sg_data_dir;
224 struct scatterlist *sg;
225 u32 n_sg;
226 u32 sg_byte_count;
227
228 struct fit_sg_descriptor *sksg_list;
229 dma_addr_t sksg_dma_address;
230
231 struct fit_completion_entry_v1 completion;
232
233 struct fit_comp_error_info err_info;
234
235};
236#define SKD_DATA_DIR_HOST_TO_CARD 1
237#define SKD_DATA_DIR_CARD_TO_HOST 2
238#define SKD_DATA_DIR_NONE 3 /* especially for DISCARD requests. */
239
240struct skd_special_context {
241 struct skd_request_context req;
242
243 u8 orphaned;
244
245 void *data_buf;
246 dma_addr_t db_dma_address;
247
248 u8 *msg_buf;
249 dma_addr_t mb_dma_address;
250};
251
252struct skd_sg_io {
253 fmode_t mode;
254 void __user *argp;
255
256 struct sg_io_hdr sg;
257
258 u8 cdb[16];
259
260 u32 dxfer_len;
261 u32 iovcnt;
262 struct sg_iovec *iov;
263 struct sg_iovec no_iov_iov;
264
265 struct skd_special_context *skspcl;
266};
267
268typedef enum skd_irq_type {
269 SKD_IRQ_LEGACY,
270 SKD_IRQ_MSI,
271 SKD_IRQ_MSIX
272} skd_irq_type_t;
273
274#define SKD_MAX_BARS 2
275
276struct skd_device {
277 volatile void __iomem *mem_map[SKD_MAX_BARS];
278 resource_size_t mem_phys[SKD_MAX_BARS];
279 u32 mem_size[SKD_MAX_BARS];
280
281 skd_irq_type_t irq_type;
282 u32 msix_count;
283 struct skd_msix_entry *msix_entries;
284
285 struct pci_dev *pdev;
286 int pcie_error_reporting_is_enabled;
287
288 spinlock_t lock;
289 struct gendisk *disk;
290 struct request_queue *queue;
291 struct device *class_dev;
292 int gendisk_on;
293 int sync_done;
294
295 atomic_t device_count;
296 u32 devno;
297 u32 major;
298 char name[32];
299 char isr_name[30];
300
301 enum skd_drvr_state state;
302 u32 drive_state;
303
304 u32 in_flight;
305 u32 cur_max_queue_depth;
306 u32 queue_low_water_mark;
307 u32 dev_max_queue_depth;
308
309 u32 num_fitmsg_context;
310 u32 num_req_context;
311
312 u32 timeout_slot[SKD_N_TIMEOUT_SLOT];
313 u32 timeout_stamp;
314 struct skd_fitmsg_context *skmsg_free_list;
315 struct skd_fitmsg_context *skmsg_table;
316
317 struct skd_request_context *skreq_free_list;
318 struct skd_request_context *skreq_table;
319
320 struct skd_special_context *skspcl_free_list;
321 struct skd_special_context *skspcl_table;
322
323 struct skd_special_context internal_skspcl;
324 u32 read_cap_blocksize;
325 u32 read_cap_last_lba;
326 int read_cap_is_valid;
327 int inquiry_is_valid;
328 u8 inq_serial_num[13]; /*12 chars plus null term */
329 u8 id_str[80]; /* holds a composite name (pci + sernum) */
330
331 u8 skcomp_cycle;
332 u32 skcomp_ix;
333 struct fit_completion_entry_v1 *skcomp_table;
334 struct fit_comp_error_info *skerr_table;
335 dma_addr_t cq_dma_address;
336
337 wait_queue_head_t waitq;
338
339 struct timer_list timer;
340 u32 timer_countdown;
341 u32 timer_substate;
342
343 int n_special;
344 int sgs_per_request;
345 u32 last_mtd;
346
347 u32 proto_ver;
348
349 int dbg_level;
350 u32 connect_time_stamp;
351 int connect_retries;
352#define SKD_MAX_CONNECT_RETRIES 16
353 u32 drive_jiffies;
354
355 u32 timo_slot;
356
e67f86b3 357 struct work_struct completion_worker;
e67f86b3
AB		 358};
359
360#define SKD_FLUSH_JOB "skd-flush-jobs"
361struct kmem_cache *skd_flush_slab;
362
363/*
364 * These commands hold "nonzero size FLUSH bios",
365 * which are enqueud in skdev->flush_list during
366 * completion of "zero size FLUSH commands".
367 * It will be active in biomode.
368 */
369struct skd_flush_cmd {
370 void *cmd;
371 struct list_head flist;
372};
373
374#define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
375#define SKD_READL(DEV, OFF) skd_reg_read32(DEV, OFF)
376#define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
377
378static inline u32 skd_reg_read32(struct skd_device *skdev, u32 offset)
379{
380 u32 val;
381
382 if (likely(skdev->dbg_level < 2))
383 return readl(skdev->mem_map[1] + offset);
384 else {
385 barrier();
386 val = readl(skdev->mem_map[1] + offset);
387 barrier();
2e44b427 388 pr_debug("%s:%s:%d offset %x = %x\n",
389 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 390 return val;
391 }
392
393}
394
395static inline void skd_reg_write32(struct skd_device *skdev, u32 val,
396 u32 offset)
397{
398 if (likely(skdev->dbg_level < 2)) {
399 writel(val, skdev->mem_map[1] + offset);
400 barrier();
e67f86b3
AB		 401 } else {
402 barrier();
403 writel(val, skdev->mem_map[1] + offset);
404 barrier();
2e44b427 405 pr_debug("%s:%s:%d offset %x = %x\n",
406 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 407 }
408}
409
410static inline void skd_reg_write64(struct skd_device *skdev, u64 val,
411 u32 offset)
412{
413 if (likely(skdev->dbg_level < 2)) {
414 writeq(val, skdev->mem_map[1] + offset);
415 barrier();
e67f86b3
AB		 416 } else {
417 barrier();
418 writeq(val, skdev->mem_map[1] + offset);
419 barrier();
2e44b427 420 pr_debug("%s:%s:%d offset %x = %016llx\n",
421 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 422 }
423}
424
425
426#define SKD_IRQ_DEFAULT SKD_IRQ_MSI
427static int skd_isr_type = SKD_IRQ_DEFAULT;
428
429module_param(skd_isr_type, int, 0444);
430MODULE_PARM_DESC(skd_isr_type, "Interrupt type capability."
431 " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
432
433#define SKD_MAX_REQ_PER_MSG_DEFAULT 1
434static int skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
435
436module_param(skd_max_req_per_msg, int, 0444);
437MODULE_PARM_DESC(skd_max_req_per_msg,
438 "Maximum SCSI requests packed in a single message."
439 " (1-14, default==1)");
440
441#define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
442#define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
443static int skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
444
445module_param(skd_max_queue_depth, int, 0444);
446MODULE_PARM_DESC(skd_max_queue_depth,
447 "Maximum SCSI requests issued to s1120."
448 " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR ")");
449
450static int skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
451module_param(skd_sgs_per_request, int, 0444);
452MODULE_PARM_DESC(skd_sgs_per_request,
453 "Maximum SG elements per block request."
454 " (1-4096, default==256)");
455
456static int skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
457module_param(skd_max_pass_thru, int, 0444);
458MODULE_PARM_DESC(skd_max_pass_thru,
459 "Maximum SCSI pass-thru at a time." " (1-50, default==32)");
460
461module_param(skd_dbg_level, int, 0444);
462MODULE_PARM_DESC(skd_dbg_level, "s1120 debug level (0,1,2)");
463
464module_param(skd_isr_comp_limit, int, 0444);
465MODULE_PARM_DESC(skd_isr_comp_limit, "s1120 isr comp limit (0=none) default=4");
466
e67f86b3
AB		 467/* Major device number dynamically assigned. */
468static u32 skd_major;
469
470static struct skd_device *skd_construct(struct pci_dev *pdev);
471static void skd_destruct(struct skd_device *skdev);
472static const struct block_device_operations skd_blockdev_ops;
473static void skd_send_fitmsg(struct skd_device *skdev,
474 struct skd_fitmsg_context *skmsg);
475static void skd_send_special_fitmsg(struct skd_device *skdev,
476 struct skd_special_context *skspcl);
477static void skd_request_fn(struct request_queue *rq);
478static void skd_end_request(struct skd_device *skdev,
479 struct skd_request_context *skreq, int error);
480static int skd_preop_sg_list(struct skd_device *skdev,
481 struct skd_request_context *skreq);
482static void skd_postop_sg_list(struct skd_device *skdev,
483 struct skd_request_context *skreq);
484
485static void skd_restart_device(struct skd_device *skdev);
486static int skd_quiesce_dev(struct skd_device *skdev);
487static int skd_unquiesce_dev(struct skd_device *skdev);
488static void skd_release_special(struct skd_device *skdev,
489 struct skd_special_context *skspcl);
490static void skd_disable_interrupts(struct skd_device *skdev);
491static void skd_isr_fwstate(struct skd_device *skdev);
492static void skd_recover_requests(struct skd_device *skdev, int requeue);
493static void skd_soft_reset(struct skd_device *skdev);
494
495static const char *skd_name(struct skd_device *skdev);
496const char *skd_drive_state_to_str(int state);
497const char *skd_skdev_state_to_str(enum skd_drvr_state state);
498static void skd_log_skdev(struct skd_device *skdev, const char *event);
499static void skd_log_skmsg(struct skd_device *skdev,
500 struct skd_fitmsg_context *skmsg, const char *event);
501static void skd_log_skreq(struct skd_device *skdev,
502 struct skd_request_context *skreq, const char *event);
503
e67f86b3
AB		 504/*
505 *****************************************************************************
506 * READ/WRITE REQUESTS
507 *****************************************************************************
508 */
509static void skd_stop_queue(struct skd_device *skdev)
510{
fcd37eb3 511 blk_stop_queue(skdev->queue);
e67f86b3
AB		 512}
513
514static void skd_unstop_queue(struct skd_device *skdev)
515{
fcd37eb3 516 queue_flag_clear(QUEUE_FLAG_STOPPED, skdev->queue);
e67f86b3
AB		 517}
518
519static void skd_start_queue(struct skd_device *skdev)
520{
fcd37eb3 521 blk_start_queue(skdev->queue);
e67f86b3
AB		 522}
523
524static int skd_queue_stopped(struct skd_device *skdev)
525{
fcd37eb3 526 return blk_queue_stopped(skdev->queue);
e67f86b3
AB		 527}
528
fcd37eb3 529static void skd_fail_all_pending(struct skd_device *skdev)
e67f86b3
AB		 530{
531 struct request_queue *q = skdev->queue;
532 struct request *req;
533
534 for (;; ) {
535 req = blk_peek_request(q);
536 if (req == NULL)
537 break;
538 blk_start_request(req);
539 __blk_end_request_all(req, -EIO);
540 }
541}
542
e67f86b3
AB		 543static void
544skd_prep_rw_cdb(struct skd_scsi_request *scsi_req,
545 int data_dir, unsigned lba,
546 unsigned count)
547{
548 if (data_dir == READ)
549 scsi_req->cdb[0] = 0x28;
550 else
551 scsi_req->cdb[0] = 0x2a;
552
553 scsi_req->cdb[1] = 0;
554 scsi_req->cdb[2] = (lba & 0xff000000) >> 24;
555 scsi_req->cdb[3] = (lba & 0xff0000) >> 16;
556 scsi_req->cdb[4] = (lba & 0xff00) >> 8;
557 scsi_req->cdb[5] = (lba & 0xff);
558 scsi_req->cdb[6] = 0;
559 scsi_req->cdb[7] = (count & 0xff00) >> 8;
560 scsi_req->cdb[8] = count & 0xff;
561 scsi_req->cdb[9] = 0;
562}
563
564static void
565skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
566 struct skd_request_context *skreq)
567{
568 skreq->flush_cmd = 1;
569
570 scsi_req->cdb[0] = 0x35;
571 scsi_req->cdb[1] = 0;
572 scsi_req->cdb[2] = 0;
573 scsi_req->cdb[3] = 0;
574 scsi_req->cdb[4] = 0;
575 scsi_req->cdb[5] = 0;
576 scsi_req->cdb[6] = 0;
577 scsi_req->cdb[7] = 0;
578 scsi_req->cdb[8] = 0;
579 scsi_req->cdb[9] = 0;
580}
581
582static void
583skd_prep_discard_cdb(struct skd_scsi_request *scsi_req,
584 struct skd_request_context *skreq,
585 struct page *page,
586 u32 lba, u32 count)
587{
588 char *buf;
589 unsigned long len;
590 struct request *req;
591
592 buf = page_address(page);
593 len = SKD_DISCARD_CDB_LENGTH;
594
595 scsi_req->cdb[0] = UNMAP;
596 scsi_req->cdb[8] = len;
597
598 put_unaligned_be16(6 + 16, &buf[0]);
599 put_unaligned_be16(16, &buf[2]);
600 put_unaligned_be64(lba, &buf[8]);
601 put_unaligned_be32(count, &buf[16]);
602
fcd37eb3
JA		 603 req = skreq->req;
604 blk_add_request_payload(req, page, len);
605 req->buffer = buf;
e67f86b3
AB		 606}
607
608static void skd_request_fn_not_online(struct request_queue *q);
609
610static void skd_request_fn(struct request_queue *q)
611{
612 struct skd_device *skdev = q->queuedata;
613 struct skd_fitmsg_context *skmsg = NULL;
614 struct fit_msg_hdr *fmh = NULL;
615 struct skd_request_context *skreq;
616 struct request *req = NULL;
e67f86b3
AB		 617 struct skd_scsi_request *scsi_req;
618 struct page *page;
619 unsigned long io_flags;
620 int error;
621 u32 lba;
622 u32 count;
623 int data_dir;
624 u32 be_lba;
625 u32 be_count;
626 u64 be_dmaa;
627 u64 cmdctxt;
628 u32 timo_slot;
629 void *cmd_ptr;
630 int flush, fua;
631
632 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
633 skd_request_fn_not_online(q);
634 return;
635 }
636
637 if (skd_queue_stopped(skdev)) {
638 if (skdev->skmsg_free_list == NULL ||
639 skdev->skreq_free_list == NULL ||
640 skdev->in_flight >= skdev->queue_low_water_mark)
641 /* There is still some kind of shortage */
642 return;
643
644 skd_unstop_queue(skdev);
645 }
646
647 /*
648 * Stop conditions:
649 * - There are no more native requests
650 * - There are already the maximum number of requests in progress
651 * - There are no more skd_request_context entries
652 * - There are no more FIT msg buffers
653 */
654 for (;; ) {
655
656 flush = fua = 0;
657
fcd37eb3 658 req = blk_peek_request(q);
e67f86b3 659
fcd37eb3
JA		 660 /* Are there any native requests to start? */
661 if (req == NULL)
662 break;
e67f86b3 663
fcd37eb3
JA		 664 lba = (u32)blk_rq_pos(req);
665 count = blk_rq_sectors(req);
666 data_dir = rq_data_dir(req);
667 io_flags = req->cmd_flags;
e67f86b3 668
fcd37eb3
JA		 669 if (io_flags & REQ_FLUSH)
670 flush++;
e67f86b3 671
fcd37eb3
JA		 672 if (io_flags & REQ_FUA)
673 fua++;
e67f86b3 674
fcd37eb3
JA		 675 pr_debug("%s:%s:%d new req=%p lba=%u(0x%x) "
676 "count=%u(0x%x) dir=%d\n",
677 skdev->name, __func__, __LINE__,
678 req, lba, lba, count, count, data_dir);
e67f86b3
AB		 679
680 /* At this point we know there is a request
681 * (from our bio q or req q depending on the way
682 * the driver is built do checks for resources.
683 */
684
685 /* Are too many requets already in progress? */
686 if (skdev->in_flight >= skdev->cur_max_queue_depth) {
2e44b427 687 pr_debug("%s:%s:%d qdepth %d, limit %d\n",
688 skdev->name, __func__, __LINE__,
689 skdev->in_flight, skdev->cur_max_queue_depth);
e67f86b3
AB		 690 break;
691 }
692
693 /* Is a skd_request_context available? */
694 skreq = skdev->skreq_free_list;
695 if (skreq == NULL) {
2e44b427 696 pr_debug("%s:%s:%d Out of req=%p\n",
697 skdev->name, __func__, __LINE__, q);
e67f86b3
AB		 698 break;
699 }
700 SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
701 SKD_ASSERT((skreq->id & SKD_ID_INCR) == 0);
702
703 /* Now we check to see if we can get a fit msg */
704 if (skmsg == NULL) {
705 if (skdev->skmsg_free_list == NULL) {
2e44b427 706 pr_debug("%s:%s:%d Out of msg\n",
707 skdev->name, __func__, __LINE__);
e67f86b3
AB		 708 break;
709 }
710 }
711
712 skreq->flush_cmd = 0;
713 skreq->n_sg = 0;
714 skreq->sg_byte_count = 0;
715 skreq->discard_page = 0;
716
717 /*
718 * OK to now dequeue request from either bio or q.
719 *
720 * At this point we are comitted to either start or reject
721 * the native request. Note that skd_request_context is
722 * available but is still at the head of the free list.
723 */
fcd37eb3
JA		 724 blk_start_request(req);
725 skreq->req = req;
726 skreq->fitmsg_id = 0;
e67f86b3
AB		 727
728 /* Either a FIT msg is in progress or we have to start one. */
729 if (skmsg == NULL) {
730 /* Are there any FIT msg buffers available? */
731 skmsg = skdev->skmsg_free_list;
732 if (skmsg == NULL) {
2e44b427 733 pr_debug("%s:%s:%d Out of msg skdev=%p\n",
734 skdev->name, __func__, __LINE__,
735 skdev);
e67f86b3
AB		 736 break;
737 }
738 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_IDLE);
739 SKD_ASSERT((skmsg->id & SKD_ID_INCR) == 0);
740
741 skdev->skmsg_free_list = skmsg->next;
742
743 skmsg->state = SKD_MSG_STATE_BUSY;
744 skmsg->id += SKD_ID_INCR;
745
746 /* Initialize the FIT msg header */
747 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
748 memset(fmh, 0, sizeof(*fmh));
749 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
750 skmsg->length = sizeof(*fmh);
751 }
752
753 skreq->fitmsg_id = skmsg->id;
754
755 /*
756 * Note that a FIT msg may have just been started
757 * but contains no SoFIT requests yet.
758 */
759
760 /*
761 * Transcode the request, checking as we go. The outcome of
762 * the transcoding is represented by the error variable.
763 */
764 cmd_ptr = &skmsg->msg_buf[skmsg->length];
765 memset(cmd_ptr, 0, 32);
766
767 be_lba = cpu_to_be32(lba);
768 be_count = cpu_to_be32(count);
769 be_dmaa = cpu_to_be64((u64)skreq->sksg_dma_address);
770 cmdctxt = skreq->id + SKD_ID_INCR;
771
772 scsi_req = cmd_ptr;
773 scsi_req->hdr.tag = cmdctxt;
774 scsi_req->hdr.sg_list_dma_address = be_dmaa;
775
776 if (data_dir == READ)
777 skreq->sg_data_dir = SKD_DATA_DIR_CARD_TO_HOST;
778 else
779 skreq->sg_data_dir = SKD_DATA_DIR_HOST_TO_CARD;
780
781 if (io_flags & REQ_DISCARD) {
782 page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
783 if (!page) {
784 pr_err("request_fn:Page allocation failed.\n");
785 skd_end_request(skdev, skreq, -ENOMEM);
786 break;
787 }
788 skreq->discard_page = 1;
789 skd_prep_discard_cdb(scsi_req, skreq, page, lba, count);
790
791 } else if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
792 skd_prep_zerosize_flush_cdb(scsi_req, skreq);
793 SKD_ASSERT(skreq->flush_cmd == 1);
794
795 } else {
796 skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
797 }
798
799 if (fua)
800 scsi_req->cdb[1] |= SKD_FUA_NV;
801
fcd37eb3 802 if (!req->bio)
e67f86b3
AB		 803 goto skip_sg;
804
805 error = skd_preop_sg_list(skdev, skreq);
806
807 if (error != 0) {
808 /*
809 * Complete the native request with error.
810 * Note that the request context is still at the
811 * head of the free list, and that the SoFIT request
812 * was encoded into the FIT msg buffer but the FIT
813 * msg length has not been updated. In short, the
814 * only resource that has been allocated but might
815 * not be used is that the FIT msg could be empty.
816 */
2e44b427 817 pr_debug("%s:%s:%d error Out\n",
818 skdev->name, __func__, __LINE__);
e67f86b3
AB		 819 skd_end_request(skdev, skreq, error);
820 continue;
821 }
822
823skip_sg:
824 scsi_req->hdr.sg_list_len_bytes =
825 cpu_to_be32(skreq->sg_byte_count);
826
827 /* Complete resource allocations. */
828 skdev->skreq_free_list = skreq->next;
829 skreq->state = SKD_REQ_STATE_BUSY;
830 skreq->id += SKD_ID_INCR;
831
832 skmsg->length += sizeof(struct skd_scsi_request);
833 fmh->num_protocol_cmds_coalesced++;
834
835 /*
836 * Update the active request counts.
837 * Capture the timeout timestamp.
838 */
839 skreq->timeout_stamp = skdev->timeout_stamp;
840 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
841 skdev->timeout_slot[timo_slot]++;
842 skdev->in_flight++;
2e44b427 843 pr_debug("%s:%s:%d req=0x%x busy=%d\n",
844 skdev->name, __func__, __LINE__,
845 skreq->id, skdev->in_flight);
e67f86b3
AB		 846
847 /*
848 * If the FIT msg buffer is full send it.
849 */
850 if (skmsg->length >= SKD_N_FITMSG_BYTES ||
851 fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
852 skd_send_fitmsg(skdev, skmsg);
853 skmsg = NULL;
854 fmh = NULL;
855 }
856 }
857
858 /*
859 * Is a FIT msg in progress? If it is empty put the buffer back
860 * on the free list. If it is non-empty send what we got.
861 * This minimizes latency when there are fewer requests than
862 * what fits in a FIT msg.
863 */
864 if (skmsg != NULL) {
865 /* Bigger than just a FIT msg header? */
866 if (skmsg->length > sizeof(struct fit_msg_hdr)) {
2e44b427 867 pr_debug("%s:%s:%d sending msg=%p, len %d\n",
868 skdev->name, __func__, __LINE__,
869 skmsg, skmsg->length);
e67f86b3
AB		 870 skd_send_fitmsg(skdev, skmsg);
871 } else {
872 /*
873 * The FIT msg is empty. It means we got started
874 * on the msg, but the requests were rejected.
875 */
876 skmsg->state = SKD_MSG_STATE_IDLE;
877 skmsg->id += SKD_ID_INCR;
878 skmsg->next = skdev->skmsg_free_list;
879 skdev->skmsg_free_list = skmsg;
880 }
881 skmsg = NULL;
882 fmh = NULL;
883 }
884
885 /*
886 * If req is non-NULL it means there is something to do but
887 * we are out of a resource.
888 */
fcd37eb3 889 if (req)
e67f86b3
AB		 890 skd_stop_queue(skdev);
891}
892
893static void skd_end_request_blk(struct skd_device *skdev,
894 struct skd_request_context *skreq, int error)
895{
896 struct request *req = skreq->req;
897 unsigned int io_flags = req->cmd_flags;
898
899 if ((io_flags & REQ_DISCARD) &&
900 (skreq->discard_page == 1)) {
2e44b427 901 pr_debug("%s:%s:%d skd_end_request_blk, free the page!",
902 skdev->name, __func__, __LINE__);
e67f86b3
AB		 903 free_page((unsigned long)req->buffer);
904 req->buffer = NULL;
905 }
906
907 if (unlikely(error)) {
908 struct request *req = skreq->req;
909 char *cmd = (rq_data_dir(req) == READ) ? "read" : "write";
910 u32 lba = (u32)blk_rq_pos(req);
911 u32 count = blk_rq_sectors(req);
912
913 pr_err("(%s): Error cmd=%s sect=%u count=%u id=0x%x\n",
914 skd_name(skdev), cmd, lba, count, skreq->id);
915 } else
2e44b427 916 pr_debug("%s:%s:%d id=0x%x error=%d\n",
917 skdev->name, __func__, __LINE__, skreq->id, error);
e67f86b3
AB		 918
919 __blk_end_request_all(skreq->req, error);
920}
921
fcd37eb3 922static int skd_preop_sg_list(struct skd_device *skdev,
e67f86b3
AB		 923 struct skd_request_context *skreq)
924{
925 struct request *req = skreq->req;
926 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
927 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
928 struct scatterlist *sg = &skreq->sg[0];
929 int n_sg;
930 int i;
931
932 skreq->sg_byte_count = 0;
933
934 /* SKD_ASSERT(skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD ||
935 skreq->sg_data_dir == SKD_DATA_DIR_CARD_TO_HOST); */
936
937 n_sg = blk_rq_map_sg(skdev->queue, req, sg);
938 if (n_sg <= 0)
939 return -EINVAL;
940
941 /*
942 * Map scatterlist to PCI bus addresses.
943 * Note PCI might change the number of entries.
944 */
945 n_sg = pci_map_sg(skdev->pdev, sg, n_sg, pci_dir);
946 if (n_sg <= 0)
947 return -EINVAL;
948
949 SKD_ASSERT(n_sg <= skdev->sgs_per_request);
950
951 skreq->n_sg = n_sg;
952
953 for (i = 0; i < n_sg; i++) {
954 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
955 u32 cnt = sg_dma_len(&sg[i]);
956 uint64_t dma_addr = sg_dma_address(&sg[i]);
957
958 sgd->control = FIT_SGD_CONTROL_NOT_LAST;
959 sgd->byte_count = cnt;
960 skreq->sg_byte_count += cnt;
961 sgd->host_side_addr = dma_addr;
962 sgd->dev_side_addr = 0;
963 }
964
965 skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;
966 skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
967
968 if (unlikely(skdev->dbg_level > 1)) {
2e44b427 969 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
970 skdev->name, __func__, __LINE__,
971 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 972 for (i = 0; i < n_sg; i++) {
973 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
2e44b427 974 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
975 "addr=0x%llx next=0x%llx\n",
976 skdev->name, __func__, __LINE__,
977 i, sgd->byte_count, sgd->control,
978 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 979 }
980 }
981
982 return 0;
983}
984
fcd37eb3 985static void skd_postop_sg_list(struct skd_device *skdev,
e67f86b3
AB		 986 struct skd_request_context *skreq)
987{
988 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
989 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
990
991 /*
992 * restore the next ptr for next IO request so we
993 * don't have to set it every time.
994 */
995 skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
996 skreq->sksg_dma_address +
997 ((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
998 pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, pci_dir);
999}
1000
e67f86b3
AB		 1001static void skd_end_request(struct skd_device *skdev,
1002 struct skd_request_context *skreq, int error)
1003{
fcd37eb3 1004 skd_end_request_blk(skdev, skreq, error);
e67f86b3
AB		 1005}
1006
1007static void skd_request_fn_not_online(struct request_queue *q)
1008{
1009 struct skd_device *skdev = q->queuedata;
1010 int error;
1011
1012 SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
1013
1014 skd_log_skdev(skdev, "req_not_online");
1015 switch (skdev->state) {
1016 case SKD_DRVR_STATE_PAUSING:
1017 case SKD_DRVR_STATE_PAUSED:
1018 case SKD_DRVR_STATE_STARTING:
1019 case SKD_DRVR_STATE_RESTARTING:
1020 case SKD_DRVR_STATE_WAIT_BOOT:
1021 /* In case of starting, we haven't started the queue,
1022 * so we can't get here... but requests are
1023 * possibly hanging out waiting for us because we
1024 * reported the dev/skd0 already. They'll wait
1025 * forever if connect doesn't complete.
1026 * What to do??? delay dev/skd0 ??
1027 */
1028 case SKD_DRVR_STATE_BUSY:
1029 case SKD_DRVR_STATE_BUSY_IMMINENT:
1030 case SKD_DRVR_STATE_BUSY_ERASE:
1031 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
1032 return;
1033
1034 case SKD_DRVR_STATE_BUSY_SANITIZE:
1035 case SKD_DRVR_STATE_STOPPING:
1036 case SKD_DRVR_STATE_SYNCING:
1037 case SKD_DRVR_STATE_FAULT:
1038 case SKD_DRVR_STATE_DISAPPEARED:
1039 default:
1040 error = -EIO;
1041 break;
1042 }
1043
1044 /* If we get here, terminate all pending block requeusts
1045 * with EIO and any scsi pass thru with appropriate sense
1046 */
1047
1048 skd_fail_all_pending(skdev);
1049}
1050
1051/*
1052 *****************************************************************************
1053 * TIMER
1054 *****************************************************************************
1055 */
1056
1057static void skd_timer_tick_not_online(struct skd_device *skdev);
1058
1059static void skd_timer_tick(ulong arg)
1060{
1061 struct skd_device *skdev = (struct skd_device *)arg;
1062
1063 u32 timo_slot;
1064 u32 overdue_timestamp;
1065 unsigned long reqflags;
1066 u32 state;
1067
1068 if (skdev->state == SKD_DRVR_STATE_FAULT)
1069 /* The driver has declared fault, and we want it to
1070 * stay that way until driver is reloaded.
1071 */
1072 return;
1073
1074 spin_lock_irqsave(&skdev->lock, reqflags);
1075
1076 state = SKD_READL(skdev, FIT_STATUS);
1077 state &= FIT_SR_DRIVE_STATE_MASK;
1078 if (state != skdev->drive_state)
1079 skd_isr_fwstate(skdev);
1080
1081 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
1082 skd_timer_tick_not_online(skdev);
1083 goto timer_func_out;
1084 }
1085 skdev->timeout_stamp++;
1086 timo_slot = skdev->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
1087
1088 /*
1089 * All requests that happened during the previous use of
1090 * this slot should be done by now. The previous use was
1091 * over 7 seconds ago.
1092 */
1093 if (skdev->timeout_slot[timo_slot] == 0)
1094 goto timer_func_out;
1095
1096 /* Something is overdue */
1097 overdue_timestamp = skdev->timeout_stamp - SKD_N_TIMEOUT_SLOT;
1098
2e44b427 1099 pr_debug("%s:%s:%d found %d timeouts, draining busy=%d\n",
1100 skdev->name, __func__, __LINE__,
1101 skdev->timeout_slot[timo_slot], skdev->in_flight);
e67f86b3
AB		 1102 pr_err("(%s): Overdue IOs (%d), busy %d\n",
1103 skd_name(skdev), skdev->timeout_slot[timo_slot],
1104 skdev->in_flight);
1105
1106 skdev->timer_countdown = SKD_DRAINING_TIMO;
1107 skdev->state = SKD_DRVR_STATE_DRAINING_TIMEOUT;
1108 skdev->timo_slot = timo_slot;
1109 skd_stop_queue(skdev);
1110
1111timer_func_out:
1112 mod_timer(&skdev->timer, (jiffies + HZ));
1113
1114 spin_unlock_irqrestore(&skdev->lock, reqflags);
1115}
1116
1117static void skd_timer_tick_not_online(struct skd_device *skdev)
1118{
1119 switch (skdev->state) {
1120 case SKD_DRVR_STATE_IDLE:
1121 case SKD_DRVR_STATE_LOAD:
1122 break;
1123 case SKD_DRVR_STATE_BUSY_SANITIZE:
2e44b427 1124 pr_debug("%s:%s:%d drive busy sanitize[%x], driver[%x]\n",
1125 skdev->name, __func__, __LINE__,
1126 skdev->drive_state, skdev->state);
e67f86b3
AB		 1127 /* If we've been in sanitize for 3 seconds, we figure we're not
1128 * going to get anymore completions, so recover requests now
1129 */
1130 if (skdev->timer_countdown > 0) {
1131 skdev->timer_countdown--;
1132 return;
1133 }
1134 skd_recover_requests(skdev, 0);
1135 break;
1136
1137 case SKD_DRVR_STATE_BUSY:
1138 case SKD_DRVR_STATE_BUSY_IMMINENT:
1139 case SKD_DRVR_STATE_BUSY_ERASE:
2e44b427 1140 pr_debug("%s:%s:%d busy[%x], countdown=%d\n",
1141 skdev->name, __func__, __LINE__,
1142 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 1143 if (skdev->timer_countdown > 0) {
1144 skdev->timer_countdown--;
1145 return;
1146 }
2e44b427 1147 pr_debug("%s:%s:%d busy[%x], timedout=%d, restarting device.",
1148 skdev->name, __func__, __LINE__,
1149 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 1150 skd_restart_device(skdev);
1151 break;
1152
1153 case SKD_DRVR_STATE_WAIT_BOOT:
1154 case SKD_DRVR_STATE_STARTING:
1155 if (skdev->timer_countdown > 0) {
1156 skdev->timer_countdown--;
1157 return;
1158 }
1159 /* For now, we fault the drive. Could attempt resets to
1160 * revcover at some point. */
1161 skdev->state = SKD_DRVR_STATE_FAULT;
1162
1163 pr_err("(%s): DriveFault Connect Timeout (%x)\n",
1164 skd_name(skdev), skdev->drive_state);
1165
1166 /*start the queue so we can respond with error to requests */
1167 /* wakeup anyone waiting for startup complete */
1168 skd_start_queue(skdev);
1169 skdev->gendisk_on = -1;
1170 wake_up_interruptible(&skdev->waitq);
1171 break;
1172
1173 case SKD_DRVR_STATE_ONLINE:
1174 /* shouldn't get here. */
1175 break;
1176
1177 case SKD_DRVR_STATE_PAUSING:
1178 case SKD_DRVR_STATE_PAUSED:
1179 break;
1180
1181 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
2e44b427 1182 pr_debug("%s:%s:%d "
1183 "draining busy [%d] tick[%d] qdb[%d] tmls[%d]\n",
1184 skdev->name, __func__, __LINE__,
1185 skdev->timo_slot,
1186 skdev->timer_countdown,
1187 skdev->in_flight,
1188 skdev->timeout_slot[skdev->timo_slot]);
e67f86b3
AB		 1189 /* if the slot has cleared we can let the I/O continue */
1190 if (skdev->timeout_slot[skdev->timo_slot] == 0) {
2e44b427 1191 pr_debug("%s:%s:%d Slot drained, starting queue.\n",
1192 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1193 skdev->state = SKD_DRVR_STATE_ONLINE;
1194 skd_start_queue(skdev);
1195 return;
1196 }
1197 if (skdev->timer_countdown > 0) {
1198 skdev->timer_countdown--;
1199 return;
1200 }
1201 skd_restart_device(skdev);
1202 break;
1203
1204 case SKD_DRVR_STATE_RESTARTING:
1205 if (skdev->timer_countdown > 0) {
1206 skdev->timer_countdown--;
1207 return;
1208 }
1209 /* For now, we fault the drive. Could attempt resets to
1210 * revcover at some point. */
1211 skdev->state = SKD_DRVR_STATE_FAULT;
1212 pr_err("(%s): DriveFault Reconnect Timeout (%x)\n",
1213 skd_name(skdev), skdev->drive_state);
1214
1215 /*
1216 * Recovering does two things:
1217 * 1. completes IO with error
1218 * 2. reclaims dma resources
1219 * When is it safe to recover requests?
1220 * - if the drive state is faulted
1221 * - if the state is still soft reset after out timeout
1222 * - if the drive registers are dead (state = FF)
1223 * If it is "unsafe", we still need to recover, so we will
1224 * disable pci bus mastering and disable our interrupts.
1225 */
1226
1227 if ((skdev->drive_state == FIT_SR_DRIVE_SOFT_RESET) ||
1228 (skdev->drive_state == FIT_SR_DRIVE_FAULT) ||
1229 (skdev->drive_state == FIT_SR_DRIVE_STATE_MASK))
1230 /* It never came out of soft reset. Try to
1231 * recover the requests and then let them
1232 * fail. This is to mitigate hung processes. */
1233 skd_recover_requests(skdev, 0);
1234 else {
1235 pr_err("(%s): Disable BusMaster (%x)\n",
1236 skd_name(skdev), skdev->drive_state);
1237 pci_disable_device(skdev->pdev);
1238 skd_disable_interrupts(skdev);
1239 skd_recover_requests(skdev, 0);
1240 }
1241
1242 /*start the queue so we can respond with error to requests */
1243 /* wakeup anyone waiting for startup complete */
1244 skd_start_queue(skdev);
1245 skdev->gendisk_on = -1;
1246 wake_up_interruptible(&skdev->waitq);
1247 break;
1248
1249 case SKD_DRVR_STATE_RESUMING:
1250 case SKD_DRVR_STATE_STOPPING:
1251 case SKD_DRVR_STATE_SYNCING:
1252 case SKD_DRVR_STATE_FAULT:
1253 case SKD_DRVR_STATE_DISAPPEARED:
1254 default:
1255 break;
1256 }
1257}
1258
1259static int skd_start_timer(struct skd_device *skdev)
1260{
1261 int rc;
1262
1263 init_timer(&skdev->timer);
1264 setup_timer(&skdev->timer, skd_timer_tick, (ulong)skdev);
1265
1266 rc = mod_timer(&skdev->timer, (jiffies + HZ));
1267 if (rc)
1268 pr_err("%s: failed to start timer %d\n",
1269 __func__, rc);
1270 return rc;
1271}
1272
1273static void skd_kill_timer(struct skd_device *skdev)
1274{
1275 del_timer_sync(&skdev->timer);
1276}
1277
1278/*
1279 *****************************************************************************
1280 * IOCTL
1281 *****************************************************************************
1282 */
1283static int skd_ioctl_sg_io(struct skd_device *skdev,
1284 fmode_t mode, void __user *argp);
1285static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1286 struct skd_sg_io *sksgio);
1287static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1288 struct skd_sg_io *sksgio);
1289static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1290 struct skd_sg_io *sksgio);
1291static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1292 struct skd_sg_io *sksgio, int dxfer_dir);
1293static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1294 struct skd_sg_io *sksgio);
1295static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio);
1296static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1297 struct skd_sg_io *sksgio);
1298static int skd_sg_io_put_status(struct skd_device *skdev,
1299 struct skd_sg_io *sksgio);
1300
1301static void skd_complete_special(struct skd_device *skdev,
1302 volatile struct fit_completion_entry_v1
1303 *skcomp,
1304 volatile struct fit_comp_error_info *skerr,
1305 struct skd_special_context *skspcl);
1306
1307static int skd_bdev_ioctl(struct block_device *bdev, fmode_t mode,
1308 uint cmd_in, ulong arg)
1309{
1310 int rc = 0;
1311 struct gendisk *disk = bdev->bd_disk;
1312 struct skd_device *skdev = disk->private_data;
1313 void __user *p = (void *)arg;
1314
2e44b427 1315 pr_debug("%s:%s:%d %s: CMD[%s] ioctl mode 0x%x, cmd 0x%x arg %0lx\n",
1316 skdev->name, __func__, __LINE__,
1317 disk->disk_name, current->comm, mode, cmd_in, arg);
e67f86b3
AB		 1318
1319 if (!capable(CAP_SYS_ADMIN))
1320 return -EPERM;
1321
1322 switch (cmd_in) {
1323 case SG_SET_TIMEOUT:
1324 case SG_GET_TIMEOUT:
1325 case SG_GET_VERSION_NUM:
1326 rc = scsi_cmd_ioctl(disk->queue, disk, mode, cmd_in, p);
1327 break;
1328 case SG_IO:
1329 rc = skd_ioctl_sg_io(skdev, mode, p);
1330 break;
1331
1332 default:
1333 rc = -ENOTTY;
1334 break;
1335 }
1336
2e44b427 1337 pr_debug("%s:%s:%d %s: completion rc %d\n",
1338 skdev->name, __func__, __LINE__, disk->disk_name, rc);
e67f86b3
AB		 1339 return rc;
1340}
1341
1342static int skd_ioctl_sg_io(struct skd_device *skdev, fmode_t mode,
1343 void __user *argp)
1344{
1345 int rc;
1346 struct skd_sg_io sksgio;
1347
1348 memset(&sksgio, 0, sizeof(sksgio));
1349 sksgio.mode = mode;
1350 sksgio.argp = argp;
1351 sksgio.iov = &sksgio.no_iov_iov;
1352
1353 switch (skdev->state) {
1354 case SKD_DRVR_STATE_ONLINE:
1355 case SKD_DRVR_STATE_BUSY_IMMINENT:
1356 break;
1357
1358 default:
2e44b427 1359 pr_debug("%s:%s:%d drive not online\n",
1360 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1361 rc = -ENXIO;
1362 goto out;
1363 }
1364
f721bb0d
AB		 1365 rc = skd_sg_io_get_and_check_args(skdev, &sksgio);
1366 if (rc)
1367 goto out;
1368
1369 rc = skd_sg_io_obtain_skspcl(skdev, &sksgio);
1370 if (rc)
1371 goto out;
1372
1373 rc = skd_sg_io_prep_buffering(skdev, &sksgio);
1374 if (rc)
1375 goto out;
1376
1377 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_TO_DEV);
1378 if (rc)
e67f86b3
AB		 1379 goto out;
1380
f721bb0d
AB		 1381 rc = skd_sg_io_send_fitmsg(skdev, &sksgio);
1382 if (rc)
e67f86b3
AB		 1383 goto out;
1384
f721bb0d
AB		 1385 rc = skd_sg_io_await(skdev, &sksgio);
1386 if (rc)
1387 goto out;
1388
1389 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_FROM_DEV);
1390 if (rc)
1391 goto out;
1392
1393 rc = skd_sg_io_put_status(skdev, &sksgio);
1394 if (rc)
e67f86b3
AB		 1395 goto out;
1396
1397 rc = 0;
1398
1399out:
1400 skd_sg_io_release_skspcl(skdev, &sksgio);
1401
1402 if (sksgio.iov != NULL && sksgio.iov != &sksgio.no_iov_iov)
1403 kfree(sksgio.iov);
1404 return rc;
1405}
1406
1407static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1408 struct skd_sg_io *sksgio)
1409{
1410 struct sg_io_hdr *sgp = &sksgio->sg;
1411 int i, acc;
1412
1413 if (!access_ok(VERIFY_WRITE, sksgio->argp, sizeof(sg_io_hdr_t))) {
2e44b427 1414 pr_debug("%s:%s:%d access sg failed %p\n",
1415 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1416 return -EFAULT;
1417 }
1418
1419 if (__copy_from_user(sgp, sksgio->argp, sizeof(sg_io_hdr_t))) {
2e44b427 1420 pr_debug("%s:%s:%d copy_from_user sg failed %p\n",
1421 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1422 return -EFAULT;
1423 }
1424
1425 if (sgp->interface_id != SG_INTERFACE_ID_ORIG) {
2e44b427 1426 pr_debug("%s:%s:%d interface_id invalid 0x%x\n",
1427 skdev->name, __func__, __LINE__, sgp->interface_id);
e67f86b3
AB		 1428 return -EINVAL;
1429 }
1430
1431 if (sgp->cmd_len > sizeof(sksgio->cdb)) {
2e44b427 1432 pr_debug("%s:%s:%d cmd_len invalid %d\n",
1433 skdev->name, __func__, __LINE__, sgp->cmd_len);
e67f86b3
AB		 1434 return -EINVAL;
1435 }
1436
1437 if (sgp->iovec_count > 256) {
2e44b427 1438 pr_debug("%s:%s:%d iovec_count invalid %d\n",
1439 skdev->name, __func__, __LINE__, sgp->iovec_count);
e67f86b3
AB		 1440 return -EINVAL;
1441 }
1442
1443 if (sgp->dxfer_len > (PAGE_SIZE * SKD_N_SG_PER_SPECIAL)) {
2e44b427 1444 pr_debug("%s:%s:%d dxfer_len invalid %d\n",
1445 skdev->name, __func__, __LINE__, sgp->dxfer_len);
e67f86b3
AB		 1446 return -EINVAL;
1447 }
1448
1449 switch (sgp->dxfer_direction) {
1450 case SG_DXFER_NONE:
1451 acc = -1;
1452 break;
1453
1454 case SG_DXFER_TO_DEV:
1455 acc = VERIFY_READ;
1456 break;
1457
1458 case SG_DXFER_FROM_DEV:
1459 case SG_DXFER_TO_FROM_DEV:
1460 acc = VERIFY_WRITE;
1461 break;
1462
1463 default:
2e44b427 1464 pr_debug("%s:%s:%d dxfer_dir invalid %d\n",
1465 skdev->name, __func__, __LINE__, sgp->dxfer_direction);
e67f86b3
AB		 1466 return -EINVAL;
1467 }
1468
1469 if (copy_from_user(sksgio->cdb, sgp->cmdp, sgp->cmd_len)) {
2e44b427 1470 pr_debug("%s:%s:%d copy_from_user cmdp failed %p\n",
1471 skdev->name, __func__, __LINE__, sgp->cmdp);
e67f86b3
AB		 1472 return -EFAULT;
1473 }
1474
1475 if (sgp->mx_sb_len != 0) {
1476 if (!access_ok(VERIFY_WRITE, sgp->sbp, sgp->mx_sb_len)) {
2e44b427 1477 pr_debug("%s:%s:%d access sbp failed %p\n",
1478 skdev->name, __func__, __LINE__, sgp->sbp);
e67f86b3
AB		 1479 return -EFAULT;
1480 }
1481 }
1482
1483 if (sgp->iovec_count == 0) {
1484 sksgio->iov[0].iov_base = sgp->dxferp;
1485 sksgio->iov[0].iov_len = sgp->dxfer_len;
1486 sksgio->iovcnt = 1;
1487 sksgio->dxfer_len = sgp->dxfer_len;
1488 } else {
1489 struct sg_iovec *iov;
1490 uint nbytes = sizeof(*iov) * sgp->iovec_count;
1491 size_t iov_data_len;
1492
1493 iov = kmalloc(nbytes, GFP_KERNEL);
1494 if (iov == NULL) {
2e44b427 1495 pr_debug("%s:%s:%d alloc iovec failed %d\n",
1496 skdev->name, __func__, __LINE__,
1497 sgp->iovec_count);
e67f86b3
AB		 1498 return -ENOMEM;
1499 }
1500 sksgio->iov = iov;
1501 sksgio->iovcnt = sgp->iovec_count;
1502
1503 if (copy_from_user(iov, sgp->dxferp, nbytes)) {
2e44b427 1504 pr_debug("%s:%s:%d copy_from_user iovec failed %p\n",
1505 skdev->name, __func__, __LINE__, sgp->dxferp);
e67f86b3
AB		 1506 return -EFAULT;
1507 }
1508
1509 /*
1510 * Sum up the vecs, making sure they don't overflow
1511 */
1512 iov_data_len = 0;
1513 for (i = 0; i < sgp->iovec_count; i++) {
1514 if (iov_data_len + iov[i].iov_len < iov_data_len)
1515 return -EINVAL;
1516 iov_data_len += iov[i].iov_len;
1517 }
1518
1519 /* SG_IO howto says that the shorter of the two wins */
1520 if (sgp->dxfer_len < iov_data_len) {
1521 sksgio->iovcnt = iov_shorten((struct iovec *)iov,
1522 sgp->iovec_count,
1523 sgp->dxfer_len);
1524 sksgio->dxfer_len = sgp->dxfer_len;
1525 } else
1526 sksgio->dxfer_len = iov_data_len;
1527 }
1528
1529 if (sgp->dxfer_direction != SG_DXFER_NONE) {
1530 struct sg_iovec *iov = sksgio->iov;
1531 for (i = 0; i < sksgio->iovcnt; i++, iov++) {
1532 if (!access_ok(acc, iov->iov_base, iov->iov_len)) {
2e44b427 1533 pr_debug("%s:%s:%d access data failed %p/%d\n",
1534 skdev->name, __func__, __LINE__,
1535 iov->iov_base, (int)iov->iov_len);
e67f86b3
AB		 1536 return -EFAULT;
1537 }
1538 }
1539 }
1540
1541 return 0;
1542}
1543
1544static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1545 struct skd_sg_io *sksgio)
1546{
1547 struct skd_special_context *skspcl = NULL;
1548 int rc;
1549
1550 for (;; ) {
1551 ulong flags;
1552
1553 spin_lock_irqsave(&skdev->lock, flags);
1554 skspcl = skdev->skspcl_free_list;
1555 if (skspcl != NULL) {
1556 skdev->skspcl_free_list =
1557 (struct skd_special_context *)skspcl->req.next;
1558 skspcl->req.id += SKD_ID_INCR;
1559 skspcl->req.state = SKD_REQ_STATE_SETUP;
1560 skspcl->orphaned = 0;
1561 skspcl->req.n_sg = 0;
1562 }
1563 spin_unlock_irqrestore(&skdev->lock, flags);
1564
1565 if (skspcl != NULL) {
1566 rc = 0;
1567 break;
1568 }
1569
2e44b427 1570 pr_debug("%s:%s:%d blocking\n",
1571 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1572
1573 rc = wait_event_interruptible_timeout(
1574 skdev->waitq,
1575 (skdev->skspcl_free_list != NULL),
1576 msecs_to_jiffies(sksgio->sg.timeout));
1577
2e44b427 1578 pr_debug("%s:%s:%d unblocking, rc=%d\n",
1579 skdev->name, __func__, __LINE__, rc);
e67f86b3
AB		 1580
1581 if (rc <= 0) {
1582 if (rc == 0)
1583 rc = -ETIMEDOUT;
1584 else
1585 rc = -EINTR;
1586 break;
1587 }
1588 /*
1589 * If we get here rc > 0 meaning the timeout to
1590 * wait_event_interruptible_timeout() had time left, hence the
1591 * sought event -- non-empty free list -- happened.
1592 * Retry the allocation.
1593 */
1594 }
1595 sksgio->skspcl = skspcl;
1596
1597 return rc;
1598}
1599
1600static int skd_skreq_prep_buffering(struct skd_device *skdev,
1601 struct skd_request_context *skreq,
1602 u32 dxfer_len)
1603{
1604 u32 resid = dxfer_len;
1605
1606 /*
1607 * The DMA engine must have aligned addresses and byte counts.
1608 */
1609 resid += (-resid) & 3;
1610 skreq->sg_byte_count = resid;
1611
1612 skreq->n_sg = 0;
1613
1614 while (resid > 0) {
1615 u32 nbytes = PAGE_SIZE;
1616 u32 ix = skreq->n_sg;
1617 struct scatterlist *sg = &skreq->sg[ix];
1618 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1619 struct page *page;
1620
1621 if (nbytes > resid)
1622 nbytes = resid;
1623
1624 page = alloc_page(GFP_KERNEL);
1625 if (page == NULL)
1626 return -ENOMEM;
1627
1628 sg_set_page(sg, page, nbytes, 0);
1629
1630 /* TODO: This should be going through a pci_???()
1631 * routine to do proper mapping. */
1632 sksg->control = FIT_SGD_CONTROL_NOT_LAST;
1633 sksg->byte_count = nbytes;
1634
1635 sksg->host_side_addr = sg_phys(sg);
1636
1637 sksg->dev_side_addr = 0;
1638 sksg->next_desc_ptr = skreq->sksg_dma_address +
1639 (ix + 1) * sizeof(*sksg);
1640
1641 skreq->n_sg++;
1642 resid -= nbytes;
1643 }
1644
1645 if (skreq->n_sg > 0) {
1646 u32 ix = skreq->n_sg - 1;
1647 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1648
1649 sksg->control = FIT_SGD_CONTROL_LAST;
1650 sksg->next_desc_ptr = 0;
1651 }
1652
1653 if (unlikely(skdev->dbg_level > 1)) {
1654 u32 i;
1655
2e44b427 1656 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
1657 skdev->name, __func__, __LINE__,
1658 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 1659 for (i = 0; i < skreq->n_sg; i++) {
1660 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1661
2e44b427 1662 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
1663 "addr=0x%llx next=0x%llx\n",
1664 skdev->name, __func__, __LINE__,
1665 i, sgd->byte_count, sgd->control,
1666 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 1667 }
1668 }
1669
1670 return 0;
1671}
1672
1673static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1674 struct skd_sg_io *sksgio)
1675{
1676 struct skd_special_context *skspcl = sksgio->skspcl;
1677 struct skd_request_context *skreq = &skspcl->req;
1678 u32 dxfer_len = sksgio->dxfer_len;
1679 int rc;
1680
1681 rc = skd_skreq_prep_buffering(skdev, skreq, dxfer_len);
1682 /*
1683 * Eventually, errors or not, skd_release_special() is called
1684 * to recover allocations including partial allocations.
1685 */
1686 return rc;
1687}
1688
1689static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1690 struct skd_sg_io *sksgio, int dxfer_dir)
1691{
1692 struct skd_special_context *skspcl = sksgio->skspcl;
1693 u32 iov_ix = 0;
1694 struct sg_iovec curiov;
1695 u32 sksg_ix = 0;
1696 u8 *bufp = NULL;
1697 u32 buf_len = 0;
1698 u32 resid = sksgio->dxfer_len;
1699 int rc;
1700
1701 curiov.iov_len = 0;
1702 curiov.iov_base = NULL;
1703
1704 if (dxfer_dir != sksgio->sg.dxfer_direction) {
1705 if (dxfer_dir != SG_DXFER_TO_DEV ||
1706 sksgio->sg.dxfer_direction != SG_DXFER_TO_FROM_DEV)
1707 return 0;
1708 }
1709
1710 while (resid > 0) {
1711 u32 nbytes = PAGE_SIZE;
1712
1713 if (curiov.iov_len == 0) {
1714 curiov = sksgio->iov[iov_ix++];
1715 continue;
1716 }
1717
1718 if (buf_len == 0) {
1719 struct page *page;
1720 page = sg_page(&skspcl->req.sg[sksg_ix++]);
1721 bufp = page_address(page);
1722 buf_len = PAGE_SIZE;
1723 }
1724
1725 nbytes = min_t(u32, nbytes, resid);
1726 nbytes = min_t(u32, nbytes, curiov.iov_len);
1727 nbytes = min_t(u32, nbytes, buf_len);
1728
1729 if (dxfer_dir == SG_DXFER_TO_DEV)
1730 rc = __copy_from_user(bufp, curiov.iov_base, nbytes);
1731 else
1732 rc = __copy_to_user(curiov.iov_base, bufp, nbytes);
1733
1734 if (rc)
1735 return -EFAULT;
1736
1737 resid -= nbytes;
1738 curiov.iov_len -= nbytes;
1739 curiov.iov_base += nbytes;
1740 buf_len -= nbytes;
1741 }
1742
1743 return 0;
1744}
1745
1746static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1747 struct skd_sg_io *sksgio)
1748{
1749 struct skd_special_context *skspcl = sksgio->skspcl;
1750 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
1751 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
1752
1753 memset(skspcl->msg_buf, 0, SKD_N_SPECIAL_FITMSG_BYTES);
1754
1755 /* Initialize the FIT msg header */
1756 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1757 fmh->num_protocol_cmds_coalesced = 1;
1758
1759 /* Initialize the SCSI request */
1760 if (sksgio->sg.dxfer_direction != SG_DXFER_NONE)
1761 scsi_req->hdr.sg_list_dma_address =
1762 cpu_to_be64(skspcl->req.sksg_dma_address);
1763 scsi_req->hdr.tag = skspcl->req.id;
1764 scsi_req->hdr.sg_list_len_bytes =
1765 cpu_to_be32(skspcl->req.sg_byte_count);
1766 memcpy(scsi_req->cdb, sksgio->cdb, sizeof(scsi_req->cdb));
1767
1768 skspcl->req.state = SKD_REQ_STATE_BUSY;
1769 skd_send_special_fitmsg(skdev, skspcl);
1770
1771 return 0;
1772}
1773
1774static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio)
1775{
1776 unsigned long flags;
1777 int rc;
1778
1779 rc = wait_event_interruptible_timeout(skdev->waitq,
1780 (sksgio->skspcl->req.state !=
1781 SKD_REQ_STATE_BUSY),
1782 msecs_to_jiffies(sksgio->sg.
1783 timeout));
1784
1785 spin_lock_irqsave(&skdev->lock, flags);
1786
1787 if (sksgio->skspcl->req.state == SKD_REQ_STATE_ABORTED) {
2e44b427 1788 pr_debug("%s:%s:%d skspcl %p aborted\n",
1789 skdev->name, __func__, __LINE__, sksgio->skspcl);
e67f86b3
AB		 1790
1791 /* Build check cond, sense and let command finish. */
1792 /* For a timeout, we must fabricate completion and sense
1793 * data to complete the command */
1794 sksgio->skspcl->req.completion.status =
1795 SAM_STAT_CHECK_CONDITION;
1796
1797 memset(&sksgio->skspcl->req.err_info, 0,
1798 sizeof(sksgio->skspcl->req.err_info));
1799 sksgio->skspcl->req.err_info.type = 0x70;
1800 sksgio->skspcl->req.err_info.key = ABORTED_COMMAND;
1801 sksgio->skspcl->req.err_info.code = 0x44;
1802 sksgio->skspcl->req.err_info.qual = 0;
1803 rc = 0;
1804 } else if (sksgio->skspcl->req.state != SKD_REQ_STATE_BUSY)
1805 /* No longer on the adapter. We finish. */
1806 rc = 0;
1807 else {
1808 /* Something's gone wrong. Still busy. Timeout or
1809 * user interrupted (control-C). Mark as an orphan
1810 * so it will be disposed when completed. */
1811 sksgio->skspcl->orphaned = 1;
1812 sksgio->skspcl = NULL;
1813 if (rc == 0) {
2e44b427 1814 pr_debug("%s:%s:%d timed out %p (%u ms)\n",
1815 skdev->name, __func__, __LINE__,
1816 sksgio, sksgio->sg.timeout);
e67f86b3
AB		 1817 rc = -ETIMEDOUT;
1818 } else {
2e44b427 1819 pr_debug("%s:%s:%d cntlc %p\n",
1820 skdev->name, __func__, __LINE__, sksgio);
e67f86b3
AB		 1821 rc = -EINTR;
1822 }
1823 }
1824
1825 spin_unlock_irqrestore(&skdev->lock, flags);
1826
1827 return rc;
1828}
1829
1830static int skd_sg_io_put_status(struct skd_device *skdev,
1831 struct skd_sg_io *sksgio)
1832{
1833 struct sg_io_hdr *sgp = &sksgio->sg;
1834 struct skd_special_context *skspcl = sksgio->skspcl;
1835 int resid = 0;
1836
1837 u32 nb = be32_to_cpu(skspcl->req.completion.num_returned_bytes);
1838
1839 sgp->status = skspcl->req.completion.status;
1840 resid = sksgio->dxfer_len - nb;
1841
1842 sgp->masked_status = sgp->status & STATUS_MASK;
1843 sgp->msg_status = 0;
1844 sgp->host_status = 0;
1845 sgp->driver_status = 0;
1846 sgp->resid = resid;
1847 if (sgp->masked_status || sgp->host_status || sgp->driver_status)
1848 sgp->info |= SG_INFO_CHECK;
1849
2e44b427 1850 pr_debug("%s:%s:%d status %x masked %x resid 0x%x\n",
1851 skdev->name, __func__, __LINE__,
1852 sgp->status, sgp->masked_status, sgp->resid);
e67f86b3
AB		 1853
1854 if (sgp->masked_status == SAM_STAT_CHECK_CONDITION) {
1855 if (sgp->mx_sb_len > 0) {
1856 struct fit_comp_error_info *ei = &skspcl->req.err_info;
1857 u32 nbytes = sizeof(*ei);
1858
1859 nbytes = min_t(u32, nbytes, sgp->mx_sb_len);
1860
1861 sgp->sb_len_wr = nbytes;
1862
1863 if (__copy_to_user(sgp->sbp, ei, nbytes)) {
2e44b427 1864 pr_debug("%s:%s:%d copy_to_user sense failed %p\n",
1865 skdev->name, __func__, __LINE__,
1866 sgp->sbp);
e67f86b3
AB		 1867 return -EFAULT;
1868 }
1869 }
1870 }
1871
1872 if (__copy_to_user(sksgio->argp, sgp, sizeof(sg_io_hdr_t))) {
2e44b427 1873 pr_debug("%s:%s:%d copy_to_user sg failed %p\n",
1874 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1875 return -EFAULT;
1876 }
1877
1878 return 0;
1879}
1880
1881static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1882 struct skd_sg_io *sksgio)
1883{
1884 struct skd_special_context *skspcl = sksgio->skspcl;
1885
1886 if (skspcl != NULL) {
1887 ulong flags;
1888
1889 sksgio->skspcl = NULL;
1890
1891 spin_lock_irqsave(&skdev->lock, flags);
1892 skd_release_special(skdev, skspcl);
1893 spin_unlock_irqrestore(&skdev->lock, flags);
1894 }
1895
1896 return 0;
1897}
1898
1899/*
1900 *****************************************************************************
1901 * INTERNAL REQUESTS -- generated by driver itself
1902 *****************************************************************************
1903 */
1904
1905static int skd_format_internal_skspcl(struct skd_device *skdev)
1906{
1907 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1908 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1909 struct fit_msg_hdr *fmh;
1910 uint64_t dma_address;
1911 struct skd_scsi_request *scsi;
1912
1913 fmh = (struct fit_msg_hdr *)&skspcl->msg_buf[0];
1914 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1915 fmh->num_protocol_cmds_coalesced = 1;
1916
1917 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1918 memset(scsi, 0, sizeof(*scsi));
1919 dma_address = skspcl->req.sksg_dma_address;
1920 scsi->hdr.sg_list_dma_address = cpu_to_be64(dma_address);
1921 sgd->control = FIT_SGD_CONTROL_LAST;
1922 sgd->byte_count = 0;
1923 sgd->host_side_addr = skspcl->db_dma_address;
1924 sgd->dev_side_addr = 0;
1925 sgd->next_desc_ptr = 0LL;
1926
1927 return 1;
1928}
1929
1930#define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
1931
1932static void skd_send_internal_skspcl(struct skd_device *skdev,
1933 struct skd_special_context *skspcl,
1934 u8 opcode)
1935{
1936 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1937 struct skd_scsi_request *scsi;
1938 unsigned char *buf = skspcl->data_buf;
1939 int i;
1940
1941 if (skspcl->req.state != SKD_REQ_STATE_IDLE)
1942 /*
1943 * A refresh is already in progress.
1944 * Just wait for it to finish.
1945 */
1946 return;
1947
1948 SKD_ASSERT((skspcl->req.id & SKD_ID_INCR) == 0);
1949 skspcl->req.state = SKD_REQ_STATE_BUSY;
1950 skspcl->req.id += SKD_ID_INCR;
1951
1952 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1953 scsi->hdr.tag = skspcl->req.id;
1954
1955 memset(scsi->cdb, 0, sizeof(scsi->cdb));
1956
1957 switch (opcode) {
1958 case TEST_UNIT_READY:
1959 scsi->cdb[0] = TEST_UNIT_READY;
1960 sgd->byte_count = 0;
1961 scsi->hdr.sg_list_len_bytes = 0;
1962 break;
1963
1964 case READ_CAPACITY:
1965 scsi->cdb[0] = READ_CAPACITY;
1966 sgd->byte_count = SKD_N_READ_CAP_BYTES;
1967 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1968 break;
1969
1970 case INQUIRY:
1971 scsi->cdb[0] = INQUIRY;
1972 scsi->cdb[1] = 0x01; /* evpd */
1973 scsi->cdb[2] = 0x80; /* serial number page */
1974 scsi->cdb[4] = 0x10;
1975 sgd->byte_count = 16;
1976 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1977 break;
1978
1979 case SYNCHRONIZE_CACHE:
1980 scsi->cdb[0] = SYNCHRONIZE_CACHE;
1981 sgd->byte_count = 0;
1982 scsi->hdr.sg_list_len_bytes = 0;
1983 break;
1984
1985 case WRITE_BUFFER:
1986 scsi->cdb[0] = WRITE_BUFFER;
1987 scsi->cdb[1] = 0x02;
1988 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1989 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1990 sgd->byte_count = WR_BUF_SIZE;
1991 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1992 /* fill incrementing byte pattern */
1993 for (i = 0; i < sgd->byte_count; i++)
1994 buf[i] = i & 0xFF;
1995 break;
1996
1997 case READ_BUFFER:
1998 scsi->cdb[0] = READ_BUFFER;
1999 scsi->cdb[1] = 0x02;
2000 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
2001 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
2002 sgd->byte_count = WR_BUF_SIZE;
2003 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
2004 memset(skspcl->data_buf, 0, sgd->byte_count);
2005 break;
2006
2007 default:
2008 SKD_ASSERT("Don't know what to send");
2009 return;
2010
2011 }
2012 skd_send_special_fitmsg(skdev, skspcl);
2013}
2014
2015static void skd_refresh_device_data(struct skd_device *skdev)
2016{
2017 struct skd_special_context *skspcl = &skdev->internal_skspcl;
2018
2019 skd_send_internal_skspcl(skdev, skspcl, TEST_UNIT_READY);
2020}
2021
2022static int skd_chk_read_buf(struct skd_device *skdev,
2023 struct skd_special_context *skspcl)
2024{
2025 unsigned char *buf = skspcl->data_buf;
2026 int i;
2027
2028 /* check for incrementing byte pattern */
2029 for (i = 0; i < WR_BUF_SIZE; i++)
2030 if (buf[i] != (i & 0xFF))
2031 return 1;
2032
2033 return 0;
2034}
2035
2036static void skd_log_check_status(struct skd_device *skdev, u8 status, u8 key,
2037 u8 code, u8 qual, u8 fruc)
2038{
2039 /* If the check condition is of special interest, log a message */
2040 if ((status == SAM_STAT_CHECK_CONDITION) && (key == 0x02)
2041 && (code == 0x04) && (qual == 0x06)) {
2042 pr_err("(%s): *** LOST_WRITE_DATA ERROR *** key/asc/"
2043 "ascq/fruc %02x/%02x/%02x/%02x\n",
2044 skd_name(skdev), key, code, qual, fruc);
2045 }
2046}
2047
2048static void skd_complete_internal(struct skd_device *skdev,
2049 volatile struct fit_completion_entry_v1
2050 *skcomp,
2051 volatile struct fit_comp_error_info *skerr,
2052 struct skd_special_context *skspcl)
2053{
2054 u8 *buf = skspcl->data_buf;
2055 u8 status;
2056 int i;
2057 struct skd_scsi_request *scsi =
2058 (struct skd_scsi_request *)&skspcl->msg_buf[64];
2059
2060 SKD_ASSERT(skspcl == &skdev->internal_skspcl);
2061
2e44b427 2062 pr_debug("%s:%s:%d complete internal %x\n",
2063 skdev->name, __func__, __LINE__, scsi->cdb[0]);
e67f86b3
AB		 2064
2065 skspcl->req.completion = *skcomp;
2066 skspcl->req.state = SKD_REQ_STATE_IDLE;
2067 skspcl->req.id += SKD_ID_INCR;
2068
2069 status = skspcl->req.completion.status;
2070
2071 skd_log_check_status(skdev, status, skerr->key, skerr->code,
2072 skerr->qual, skerr->fruc);
2073
2074 switch (scsi->cdb[0]) {
2075 case TEST_UNIT_READY:
2076 if (status == SAM_STAT_GOOD)
2077 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2078 else if ((status == SAM_STAT_CHECK_CONDITION) &&
2079 (skerr->key == MEDIUM_ERROR))
2080 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2081 else {
2082 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2083 pr_debug("%s:%s:%d TUR failed, don't send anymore state 0x%x\n",
2084 skdev->name, __func__, __LINE__,
2085 skdev->state);
e67f86b3
AB		 2086 return;
2087 }
2e44b427 2088 pr_debug("%s:%s:%d **** TUR failed, retry skerr\n",
2089 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2090 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2091 }
2092 break;
2093
2094 case WRITE_BUFFER:
2095 if (status == SAM_STAT_GOOD)
2096 skd_send_internal_skspcl(skdev, skspcl, READ_BUFFER);
2097 else {
2098 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2099 pr_debug("%s:%s:%d write buffer failed, don't send anymore state 0x%x\n",
2100 skdev->name, __func__, __LINE__,
2101 skdev->state);
e67f86b3
AB		 2102 return;
2103 }
2e44b427 2104 pr_debug("%s:%s:%d **** write buffer failed, retry skerr\n",
2105 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2106 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2107 }
2108 break;
2109
2110 case READ_BUFFER:
2111 if (status == SAM_STAT_GOOD) {
2112 if (skd_chk_read_buf(skdev, skspcl) == 0)
2113 skd_send_internal_skspcl(skdev, skspcl,
2114 READ_CAPACITY);
2115 else {
2116 pr_err(
2117 "(%s):*** W/R Buffer mismatch %d ***\n",
2118 skd_name(skdev), skdev->connect_retries);
2119 if (skdev->connect_retries <
2120 SKD_MAX_CONNECT_RETRIES) {
2121 skdev->connect_retries++;
2122 skd_soft_reset(skdev);
2123 } else {
2124 pr_err(
2125 "(%s): W/R Buffer Connect Error\n",
2126 skd_name(skdev));
2127 return;
2128 }
2129 }
2130
2131 } else {
2132 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2133 pr_debug("%s:%s:%d "
2134 "read buffer failed, don't send anymore state 0x%x\n",
2135 skdev->name, __func__, __LINE__,
2136 skdev->state);
e67f86b3
AB		 2137 return;
2138 }
2e44b427 2139 pr_debug("%s:%s:%d "
2140 "**** read buffer failed, retry skerr\n",
2141 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2142 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2143 }
2144 break;
2145
2146 case READ_CAPACITY:
2147 skdev->read_cap_is_valid = 0;
2148 if (status == SAM_STAT_GOOD) {
2149 skdev->read_cap_last_lba =
2150 (buf[0] << 24) | (buf[1] << 16) |
2151 (buf[2] << 8) | buf[3];
2152 skdev->read_cap_blocksize =
2153 (buf[4] << 24) | (buf[5] << 16) |
2154 (buf[6] << 8) | buf[7];
2155
2e44b427 2156 pr_debug("%s:%s:%d last lba %d, bs %d\n",
2157 skdev->name, __func__, __LINE__,
2158 skdev->read_cap_last_lba,
2159 skdev->read_cap_blocksize);
e67f86b3
AB		 2160
2161 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2162
2163 skdev->read_cap_is_valid = 1;
2164
2165 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2166 } else if ((status == SAM_STAT_CHECK_CONDITION) &&
2167 (skerr->key == MEDIUM_ERROR)) {
2168 skdev->read_cap_last_lba = ~0;
2169 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2e44b427 2170 pr_debug("%s:%s:%d "
2171 "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n",
2172 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2173 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2174 } else {
2e44b427 2175 pr_debug("%s:%s:%d **** READCAP failed, retry TUR\n",
2176 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2177 skd_send_internal_skspcl(skdev, skspcl,
2178 TEST_UNIT_READY);
2179 }
2180 break;
2181
2182 case INQUIRY:
2183 skdev->inquiry_is_valid = 0;
2184 if (status == SAM_STAT_GOOD) {
2185 skdev->inquiry_is_valid = 1;
2186
2187 for (i = 0; i < 12; i++)
2188 skdev->inq_serial_num[i] = buf[i + 4];
2189 skdev->inq_serial_num[12] = 0;
2190 }
2191
2192 if (skd_unquiesce_dev(skdev) < 0)
2e44b427 2193 pr_debug("%s:%s:%d **** failed, to ONLINE device\n",
2194 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2195 /* connection is complete */
2196 skdev->connect_retries = 0;
2197 break;
2198
2199 case SYNCHRONIZE_CACHE:
2200 if (status == SAM_STAT_GOOD)
2201 skdev->sync_done = 1;
2202 else
2203 skdev->sync_done = -1;
2204 wake_up_interruptible(&skdev->waitq);
2205 break;
2206
2207 default:
2208 SKD_ASSERT("we didn't send this");
2209 }
2210}
2211
2212/*
2213 *****************************************************************************
2214 * FIT MESSAGES
2215 *****************************************************************************
2216 */
2217
2218static void skd_send_fitmsg(struct skd_device *skdev,
2219 struct skd_fitmsg_context *skmsg)
2220{
2221 u64 qcmd;
2222 struct fit_msg_hdr *fmh;
2223
2e44b427 2224 pr_debug("%s:%s:%d dma address 0x%llx, busy=%d\n",
2225 skdev->name, __func__, __LINE__,
2226 skmsg->mb_dma_address, skdev->in_flight);
2227 pr_debug("%s:%s:%d msg_buf 0x%p, offset %x\n",
2228 skdev->name, __func__, __LINE__,
2229 skmsg->msg_buf, skmsg->offset);
e67f86b3
AB		 2230
2231 qcmd = skmsg->mb_dma_address;
2232 qcmd |= FIT_QCMD_QID_NORMAL;
2233
2234 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
2235 skmsg->outstanding = fmh->num_protocol_cmds_coalesced;
2236
2237 if (unlikely(skdev->dbg_level > 1)) {
2238 u8 *bp = (u8 *)skmsg->msg_buf;
2239 int i;
2240 for (i = 0; i < skmsg->length; i += 8) {
2e44b427 2241 pr_debug("%s:%s:%d msg[%2d] %02x %02x %02x %02x "
2242 "%02x %02x %02x %02x\n",
2243 skdev->name, __func__, __LINE__,
2244 i, bp[i + 0], bp[i + 1], bp[i + 2],
2245 bp[i + 3], bp[i + 4], bp[i + 5],
2246 bp[i + 6], bp[i + 7]);
e67f86b3
AB		 2247 if (i == 0)
2248 i = 64 - 8;
2249 }
2250 }
2251
2252 if (skmsg->length > 256)
2253 qcmd |= FIT_QCMD_MSGSIZE_512;
2254 else if (skmsg->length > 128)
2255 qcmd |= FIT_QCMD_MSGSIZE_256;
2256 else if (skmsg->length > 64)
2257 qcmd |= FIT_QCMD_MSGSIZE_128;
2258 else
2259 /*
2260 * This makes no sense because the FIT msg header is
2261 * 64 bytes. If the msg is only 64 bytes long it has
2262 * no payload.
2263 */
2264 qcmd |= FIT_QCMD_MSGSIZE_64;
2265
2266 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2267
2268}
2269
2270static void skd_send_special_fitmsg(struct skd_device *skdev,
2271 struct skd_special_context *skspcl)
2272{
2273 u64 qcmd;
2274
2275 if (unlikely(skdev->dbg_level > 1)) {
2276 u8 *bp = (u8 *)skspcl->msg_buf;
2277 int i;
2278
2279 for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
2e44b427 2280 pr_debug("%s:%s:%d spcl[%2d] %02x %02x %02x %02x "
2281 "%02x %02x %02x %02x\n",
2282 skdev->name, __func__, __LINE__, i,
2283 bp[i + 0], bp[i + 1], bp[i + 2], bp[i + 3],
2284 bp[i + 4], bp[i + 5], bp[i + 6], bp[i + 7]);
e67f86b3
AB		 2285 if (i == 0)
2286 i = 64 - 8;
2287 }
2288
2e44b427 2289 pr_debug("%s:%s:%d skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
2290 skdev->name, __func__, __LINE__,
2291 skspcl, skspcl->req.id, skspcl->req.sksg_list,
2292 skspcl->req.sksg_dma_address);
e67f86b3
AB		 2293 for (i = 0; i < skspcl->req.n_sg; i++) {
2294 struct fit_sg_descriptor *sgd =
2295 &skspcl->req.sksg_list[i];
2296
2e44b427 2297 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
2298 "addr=0x%llx next=0x%llx\n",
2299 skdev->name, __func__, __LINE__,
2300 i, sgd->byte_count, sgd->control,
2301 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 2302 }
2303 }
2304
2305 /*
2306 * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
2307 * and one 64-byte SSDI command.
2308 */
2309 qcmd = skspcl->mb_dma_address;
2310 qcmd |= FIT_QCMD_QID_NORMAL + FIT_QCMD_MSGSIZE_128;
2311
2312 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2313}
2314
2315/*
2316 *****************************************************************************
2317 * COMPLETION QUEUE
2318 *****************************************************************************
2319 */
2320
2321static void skd_complete_other(struct skd_device *skdev,
2322 volatile struct fit_completion_entry_v1 *skcomp,
2323 volatile struct fit_comp_error_info *skerr);
2324
2325
2326static void skd_requeue_request(struct skd_device *skdev,
2327 struct skd_request_context *skreq);
2328
2329struct sns_info {
2330 u8 type;
2331 u8 stat;
2332 u8 key;
2333 u8 asc;
2334 u8 ascq;
2335 u8 mask;
2336 enum skd_check_status_action action;
2337};
2338
2339static struct sns_info skd_chkstat_table[] = {
2340 /* Good */
2341 { 0x70, 0x02, RECOVERED_ERROR, 0, 0, 0x1c,
2342 SKD_CHECK_STATUS_REPORT_GOOD },
2343
2344 /* Smart alerts */
2345 { 0x70, 0x02, NO_SENSE, 0x0B, 0x00, 0x1E, /* warnings */
2346 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2347 { 0x70, 0x02, NO_SENSE, 0x5D, 0x00, 0x1E, /* thresholds */
2348 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2349 { 0x70, 0x02, RECOVERED_ERROR, 0x0B, 0x01, 0x1F, /* temperature over trigger */
2350 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2351
2352 /* Retry (with limits) */
2353 { 0x70, 0x02, 0x0B, 0, 0, 0x1C, /* This one is for DMA ERROR */
2354 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2355 { 0x70, 0x02, 0x06, 0x0B, 0x00, 0x1E, /* warnings */
2356 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2357 { 0x70, 0x02, 0x06, 0x5D, 0x00, 0x1E, /* thresholds */
2358 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2359 { 0x70, 0x02, 0x06, 0x80, 0x30, 0x1F, /* backup power */
2360 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2361
2362 /* Busy (or about to be) */
2363 { 0x70, 0x02, 0x06, 0x3f, 0x01, 0x1F, /* fw changed */
2364 SKD_CHECK_STATUS_BUSY_IMMINENT },
2365};
2366
2367/*
2368 * Look up status and sense data to decide how to handle the error
2369 * from the device.
2370 * mask says which fields must match e.g., mask=0x18 means check
2371 * type and stat, ignore key, asc, ascq.
2372 */
2373
2374static enum skd_check_status_action skd_check_status(struct skd_device *skdev,
2375 u8 cmp_status,
2376 volatile struct fit_comp_error_info *skerr)
2377{
2378 int i, n;
2379
2380 pr_err("(%s): key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
2381 skd_name(skdev), skerr->key, skerr->code, skerr->qual,
2382 skerr->fruc);
2383
2e44b427 2384 pr_debug("%s:%s:%d stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
2385 skdev->name, __func__, __LINE__, skerr->type, cmp_status,
2386 skerr->key, skerr->code, skerr->qual, skerr->fruc);
e67f86b3
AB		 2387
2388 /* Does the info match an entry in the good category? */
2389 n = sizeof(skd_chkstat_table) / sizeof(skd_chkstat_table[0]);
2390 for (i = 0; i < n; i++) {
2391 struct sns_info *sns = &skd_chkstat_table[i];
2392
2393 if (sns->mask & 0x10)
2394 if (skerr->type != sns->type)
2395 continue;
2396
2397 if (sns->mask & 0x08)
2398 if (cmp_status != sns->stat)
2399 continue;
2400
2401 if (sns->mask & 0x04)
2402 if (skerr->key != sns->key)
2403 continue;
2404
2405 if (sns->mask & 0x02)
2406 if (skerr->code != sns->asc)
2407 continue;
2408
2409 if (sns->mask & 0x01)
2410 if (skerr->qual != sns->ascq)
2411 continue;
2412
2413 if (sns->action == SKD_CHECK_STATUS_REPORT_SMART_ALERT) {
2414 pr_err("(%s): SMART Alert: sense key/asc/ascq "
2415 "%02x/%02x/%02x\n",
2416 skd_name(skdev), skerr->key,
2417 skerr->code, skerr->qual);
2418 }
2419 return sns->action;
2420 }
2421
2422 /* No other match, so nonzero status means error,
2423 * zero status means good
2424 */
2425 if (cmp_status) {
2e44b427 2426 pr_debug("%s:%s:%d status check: error\n",
2427 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2428 return SKD_CHECK_STATUS_REPORT_ERROR;
2429 }
2430
2e44b427 2431 pr_debug("%s:%s:%d status check good default\n",
2432 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2433 return SKD_CHECK_STATUS_REPORT_GOOD;
2434}
2435
2436static void skd_resolve_req_exception(struct skd_device *skdev,
2437 struct skd_request_context *skreq)
2438{
2439 u8 cmp_status = skreq->completion.status;
2440
2441 switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {
2442 case SKD_CHECK_STATUS_REPORT_GOOD:
2443 case SKD_CHECK_STATUS_REPORT_SMART_ALERT:
2444 skd_end_request(skdev, skreq, 0);
2445 break;
2446
2447 case SKD_CHECK_STATUS_BUSY_IMMINENT:
2448 skd_log_skreq(skdev, skreq, "retry(busy)");
2449 skd_requeue_request(skdev, skreq);
2450 pr_info("(%s) drive BUSY imminent\n", skd_name(skdev));
2451 skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT;
2452 skdev->timer_countdown = SKD_TIMER_MINUTES(20);
2453 skd_quiesce_dev(skdev);
2454 break;
2455
2456 case SKD_CHECK_STATUS_REQUEUE_REQUEST:
fcd37eb3
JA		 2457 if ((unsigned long) ++skreq->req->special < SKD_MAX_RETRIES) {
2458 skd_log_skreq(skdev, skreq, "retry");
2459 skd_requeue_request(skdev, skreq);
2460 break;
e67f86b3
AB		 2461 }
2462 /* fall through to report error */
2463
2464 case SKD_CHECK_STATUS_REPORT_ERROR:
2465 default:
2466 skd_end_request(skdev, skreq, -EIO);
2467 break;
2468 }
2469}
2470
2471static void skd_requeue_request(struct skd_device *skdev,
2472 struct skd_request_context *skreq)
2473{
fcd37eb3 2474 blk_requeue_request(skdev->queue, skreq->req);
e67f86b3
AB		 2475}
2476
2477
2478
2479/* assume spinlock is already held */
2480static void skd_release_skreq(struct skd_device *skdev,
2481 struct skd_request_context *skreq)
2482{
2483 u32 msg_slot;
2484 struct skd_fitmsg_context *skmsg;
2485
2486 u32 timo_slot;
2487
2488 /*
2489 * Reclaim the FIT msg buffer if this is
2490 * the first of the requests it carried to
2491 * be completed. The FIT msg buffer used to
2492 * send this request cannot be reused until
2493 * we are sure the s1120 card has copied
2494 * it to its memory. The FIT msg might have
2495 * contained several requests. As soon as
2496 * any of them are completed we know that
2497 * the entire FIT msg was transferred.
2498 * Only the first completed request will
2499 * match the FIT msg buffer id. The FIT
2500 * msg buffer id is immediately updated.
2501 * When subsequent requests complete the FIT
2502 * msg buffer id won't match, so we know
2503 * quite cheaply that it is already done.
2504 */
2505 msg_slot = skreq->fitmsg_id & SKD_ID_SLOT_MASK;
2506 SKD_ASSERT(msg_slot < skdev->num_fitmsg_context);
2507
2508 skmsg = &skdev->skmsg_table[msg_slot];
2509 if (skmsg->id == skreq->fitmsg_id) {
2510 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_BUSY);
2511 SKD_ASSERT(skmsg->outstanding > 0);
2512 skmsg->outstanding--;
2513 if (skmsg->outstanding == 0) {
2514 skmsg->state = SKD_MSG_STATE_IDLE;
2515 skmsg->id += SKD_ID_INCR;
2516 skmsg->next = skdev->skmsg_free_list;
2517 skdev->skmsg_free_list = skmsg;
2518 }
2519 }
2520
2521 /*
2522 * Decrease the number of active requests.
2523 * Also decrements the count in the timeout slot.
2524 */
2525 SKD_ASSERT(skdev->in_flight > 0);
2526 skdev->in_flight -= 1;
2527
2528 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
2529 SKD_ASSERT(skdev->timeout_slot[timo_slot] > 0);
2530 skdev->timeout_slot[timo_slot] -= 1;
2531
2532 /*
2533 * Reset backpointer
2534 */
fcd37eb3 2535 skreq->req = NULL;
e67f86b3
AB		 2536
2537 /*
2538 * Reclaim the skd_request_context
2539 */
2540 skreq->state = SKD_REQ_STATE_IDLE;
2541 skreq->id += SKD_ID_INCR;
2542 skreq->next = skdev->skreq_free_list;
2543 skdev->skreq_free_list = skreq;
2544}
2545
2546#define DRIVER_INQ_EVPD_PAGE_CODE 0xDA
2547
2548static void skd_do_inq_page_00(struct skd_device *skdev,
2549 volatile struct fit_completion_entry_v1 *skcomp,
2550 volatile struct fit_comp_error_info *skerr,
2551 uint8_t *cdb, uint8_t *buf)
2552{
2553 uint16_t insert_pt, max_bytes, drive_pages, drive_bytes, new_size;
2554
2555 /* Caller requested "supported pages". The driver needs to insert
2556 * its page.
2557 */
2e44b427 2558 pr_debug("%s:%s:%d skd_do_driver_inquiry: modify supported pages.\n",
2559 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2560
2561 /* If the device rejected the request because the CDB was
2562 * improperly formed, then just leave.
2563 */
2564 if (skcomp->status == SAM_STAT_CHECK_CONDITION &&
2565 skerr->key == ILLEGAL_REQUEST && skerr->code == 0x24)
2566 return;
2567
2568 /* Get the amount of space the caller allocated */
2569 max_bytes = (cdb[3] << 8) | cdb[4];
2570
2571 /* Get the number of pages actually returned by the device */
2572 drive_pages = (buf[2] << 8) | buf[3];
2573 drive_bytes = drive_pages + 4;
2574 new_size = drive_pages + 1;
2575
2576 /* Supported pages must be in numerical order, so find where
2577 * the driver page needs to be inserted into the list of
2578 * pages returned by the device.
2579 */
2580 for (insert_pt = 4; insert_pt < drive_bytes; insert_pt++) {
2581 if (buf[insert_pt] == DRIVER_INQ_EVPD_PAGE_CODE)
2582 return; /* Device using this page code. abort */
2583 else if (buf[insert_pt] > DRIVER_INQ_EVPD_PAGE_CODE)
2584 break;
2585 }
2586
2587 if (insert_pt < max_bytes) {
2588 uint16_t u;
2589
2590 /* Shift everything up one byte to make room. */
2591 for (u = new_size + 3; u > insert_pt; u--)
2592 buf[u] = buf[u - 1];
2593 buf[insert_pt] = DRIVER_INQ_EVPD_PAGE_CODE;
2594
2595 /* SCSI byte order increment of num_returned_bytes by 1 */
2596 skcomp->num_returned_bytes =
2597 be32_to_cpu(skcomp->num_returned_bytes) + 1;
2598 skcomp->num_returned_bytes =
2599 be32_to_cpu(skcomp->num_returned_bytes);
2600 }
2601
2602 /* update page length field to reflect the driver's page too */
2603 buf[2] = (uint8_t)((new_size >> 8) & 0xFF);
2604 buf[3] = (uint8_t)((new_size >> 0) & 0xFF);
2605}
2606
2607static void skd_get_link_info(struct pci_dev *pdev, u8 *speed, u8 *width)
2608{
2609 int pcie_reg;
2610 u16 pci_bus_speed;
2611 u8 pci_lanes;
2612
2613 pcie_reg = pci_find_capability(pdev, PCI_CAP_ID_EXP);
2614 if (pcie_reg) {
2615 u16 linksta;
2616 pci_read_config_word(pdev, pcie_reg + PCI_EXP_LNKSTA, &linksta);
2617
2618 pci_bus_speed = linksta & 0xF;
2619 pci_lanes = (linksta & 0x3F0) >> 4;
2620 } else {
2621 *speed = STEC_LINK_UNKNOWN;
2622 *width = 0xFF;
2623 return;
2624 }
2625
2626 switch (pci_bus_speed) {
2627 case 1:
2628 *speed = STEC_LINK_2_5GTS;
2629 break;
2630 case 2:
2631 *speed = STEC_LINK_5GTS;
2632 break;
2633 case 3:
2634 *speed = STEC_LINK_8GTS;
2635 break;
2636 default:
2637 *speed = STEC_LINK_UNKNOWN;
2638 break;
2639 }
2640
2641 if (pci_lanes <= 0x20)
2642 *width = pci_lanes;
2643 else
2644 *width = 0xFF;
2645}
2646
2647static void skd_do_inq_page_da(struct skd_device *skdev,
2648 volatile struct fit_completion_entry_v1 *skcomp,
2649 volatile struct fit_comp_error_info *skerr,
2650 uint8_t *cdb, uint8_t *buf)
2651{
2652 unsigned max_bytes;
2653 struct driver_inquiry_data inq;
2654 u16 val;
2655
2e44b427 2656 pr_debug("%s:%s:%d skd_do_driver_inquiry: return driver page\n",
2657 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2658
2659 memset(&inq, 0, sizeof(inq));
2660
2661 inq.page_code = DRIVER_INQ_EVPD_PAGE_CODE;
2662
2663 if (skdev->pdev && skdev->pdev->bus) {
2664 skd_get_link_info(skdev->pdev,
2665 &inq.pcie_link_speed, &inq.pcie_link_lanes);
2666 inq.pcie_bus_number = cpu_to_be16(skdev->pdev->bus->number);
2667 inq.pcie_device_number = PCI_SLOT(skdev->pdev->devfn);
2668 inq.pcie_function_number = PCI_FUNC(skdev->pdev->devfn);
2669
2670 pci_read_config_word(skdev->pdev, PCI_VENDOR_ID, &val);
2671 inq.pcie_vendor_id = cpu_to_be16(val);
2672
2673 pci_read_config_word(skdev->pdev, PCI_DEVICE_ID, &val);
2674 inq.pcie_device_id = cpu_to_be16(val);
2675
2676 pci_read_config_word(skdev->pdev, PCI_SUBSYSTEM_VENDOR_ID,
2677 &val);
2678 inq.pcie_subsystem_vendor_id = cpu_to_be16(val);
2679
2680 pci_read_config_word(skdev->pdev, PCI_SUBSYSTEM_ID, &val);
2681 inq.pcie_subsystem_device_id = cpu_to_be16(val);
2682 } else {
2683 inq.pcie_bus_number = 0xFFFF;
2684 inq.pcie_device_number = 0xFF;
2685 inq.pcie_function_number = 0xFF;
2686 inq.pcie_link_speed = 0xFF;
2687 inq.pcie_link_lanes = 0xFF;
2688 inq.pcie_vendor_id = 0xFFFF;
2689 inq.pcie_device_id = 0xFFFF;
2690 inq.pcie_subsystem_vendor_id = 0xFFFF;
2691 inq.pcie_subsystem_device_id = 0xFFFF;
2692 }
2693
2694 /* Driver version, fixed lenth, padded with spaces on the right */
2695 inq.driver_version_length = sizeof(inq.driver_version);
2696 memset(&inq.driver_version, ' ', sizeof(inq.driver_version));
2697 memcpy(inq.driver_version, DRV_VER_COMPL,
2698 min(sizeof(inq.driver_version), strlen(DRV_VER_COMPL)));
2699
2700 inq.page_length = cpu_to_be16((sizeof(inq) - 4));
2701
2702 /* Clear the error set by the device */
2703 skcomp->status = SAM_STAT_GOOD;
2704 memset((void *)skerr, 0, sizeof(*skerr));
2705
2706 /* copy response into output buffer */
2707 max_bytes = (cdb[3] << 8) | cdb[4];
2708 memcpy(buf, &inq, min_t(unsigned, max_bytes, sizeof(inq)));
2709
2710 skcomp->num_returned_bytes =
2711 be32_to_cpu(min_t(uint16_t, max_bytes, sizeof(inq)));
2712}
2713
2714static void skd_do_driver_inq(struct skd_device *skdev,
2715 volatile struct fit_completion_entry_v1 *skcomp,
2716 volatile struct fit_comp_error_info *skerr,
2717 uint8_t *cdb, uint8_t *buf)
2718{
2719 if (!buf)
2720 return;
2721 else if (cdb[0] != INQUIRY)
2722 return; /* Not an INQUIRY */
2723 else if ((cdb[1] & 1) == 0)
2724 return; /* EVPD not set */
2725 else if (cdb[2] == 0)
2726 /* Need to add driver's page to supported pages list */
2727 skd_do_inq_page_00(skdev, skcomp, skerr, cdb, buf);
2728 else if (cdb[2] == DRIVER_INQ_EVPD_PAGE_CODE)
2729 /* Caller requested driver's page */
2730 skd_do_inq_page_da(skdev, skcomp, skerr, cdb, buf);
2731}
2732
2733static unsigned char *skd_sg_1st_page_ptr(struct scatterlist *sg)
2734{
2735 if (!sg)
2736 return NULL;
2737 if (!sg_page(sg))
2738 return NULL;
2739 return sg_virt(sg);
2740}
2741
2742static void skd_process_scsi_inq(struct skd_device *skdev,
2743 volatile struct fit_completion_entry_v1
2744 *skcomp,
2745 volatile struct fit_comp_error_info *skerr,
2746 struct skd_special_context *skspcl)
2747{
2748 uint8_t *buf;
2749 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
2750 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
2751
2752 dma_sync_sg_for_cpu(skdev->class_dev, skspcl->req.sg, skspcl->req.n_sg,
2753 skspcl->req.sg_data_dir);
2754 buf = skd_sg_1st_page_ptr(skspcl->req.sg);
2755
2756 if (buf)
2757 skd_do_driver_inq(skdev, skcomp, skerr, scsi_req->cdb, buf);
2758}
2759
2760
2761static int skd_isr_completion_posted(struct skd_device *skdev,
2762 int limit, int *enqueued)
2763{
2764 volatile struct fit_completion_entry_v1 *skcmp = NULL;
2765 volatile struct fit_comp_error_info *skerr;
2766 u16 req_id;
2767 u32 req_slot;
2768 struct skd_request_context *skreq;
2769 u16 cmp_cntxt = 0;
2770 u8 cmp_status = 0;
2771 u8 cmp_cycle = 0;
2772 u32 cmp_bytes = 0;
2773 int rc = 0;
2774 int processed = 0;
e67f86b3
AB		 2775
2776 for (;; ) {
2777 SKD_ASSERT(skdev->skcomp_ix < SKD_N_COMPLETION_ENTRY);
2778
2779 skcmp = &skdev->skcomp_table[skdev->skcomp_ix];
2780 cmp_cycle = skcmp->cycle;
2781 cmp_cntxt = skcmp->tag;
2782 cmp_status = skcmp->status;
2783 cmp_bytes = be32_to_cpu(skcmp->num_returned_bytes);
2784
2785 skerr = &skdev->skerr_table[skdev->skcomp_ix];
2786
2e44b427 2787 pr_debug("%s:%s:%d "
2788 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d "
2789 "busy=%d rbytes=0x%x proto=%d\n",
2790 skdev->name, __func__, __LINE__, skdev->skcomp_cycle,
2791 skdev->skcomp_ix, cmp_cycle, cmp_cntxt, cmp_status,
2792 skdev->in_flight, cmp_bytes, skdev->proto_ver);
e67f86b3
AB		 2793
2794 if (cmp_cycle != skdev->skcomp_cycle) {
2e44b427 2795 pr_debug("%s:%s:%d end of completions\n",
2796 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2797 break;
2798 }
2799 /*
2800 * Update the completion queue head index and possibly
2801 * the completion cycle count. 8-bit wrap-around.
2802 */
2803 skdev->skcomp_ix++;
2804 if (skdev->skcomp_ix >= SKD_N_COMPLETION_ENTRY) {
2805 skdev->skcomp_ix = 0;
2806 skdev->skcomp_cycle++;
2807 }
2808
2809 /*
2810 * The command context is a unique 32-bit ID. The low order
2811 * bits help locate the request. The request is usually a
2812 * r/w request (see skd_start() above) or a special request.
2813 */
2814 req_id = cmp_cntxt;
2815 req_slot = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
2816
2817 /* Is this other than a r/w request? */
2818 if (req_slot >= skdev->num_req_context) {
2819 /*
2820 * This is not a completion for a r/w request.
2821 */
2822 skd_complete_other(skdev, skcmp, skerr);
2823 continue;
2824 }
2825
2826 skreq = &skdev->skreq_table[req_slot];
2827
2828 /*
2829 * Make sure the request ID for the slot matches.
2830 */
2831 if (skreq->id != req_id) {
2e44b427 2832 pr_debug("%s:%s:%d mismatch comp_id=0x%x req_id=0x%x\n",
2833 skdev->name, __func__, __LINE__,
2834 req_id, skreq->id);
e67f86b3
AB		 2835 {
2836 u16 new_id = cmp_cntxt;
2837 pr_err("(%s): Completion mismatch "
2838 "comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
2839 skd_name(skdev), req_id,
2840 skreq->id, new_id);
2841
2842 continue;
2843 }
2844 }
2845
2846 SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);
2847
2848 if (skreq->state == SKD_REQ_STATE_ABORTED) {
2e44b427 2849 pr_debug("%s:%s:%d reclaim req %p id=%04x\n",
2850 skdev->name, __func__, __LINE__,
2851 skreq, skreq->id);
e67f86b3
AB		 2852 /* a previously timed out command can
2853 * now be cleaned up */
2854 skd_release_skreq(skdev, skreq);
2855 continue;
2856 }
2857
2858 skreq->completion = *skcmp;
2859 if (unlikely(cmp_status == SAM_STAT_CHECK_CONDITION)) {
2860 skreq->err_info = *skerr;
2861 skd_log_check_status(skdev, cmp_status, skerr->key,
2862 skerr->code, skerr->qual,
2863 skerr->fruc);
2864 }
2865 /* Release DMA resources for the request. */
2866 if (skreq->n_sg > 0)
2867 skd_postop_sg_list(skdev, skreq);
2868
fcd37eb3 2869 if (!skreq->req) {
2e44b427 2870 pr_debug("%s:%s:%d NULL backptr skdreq %p, "
2871 "req=0x%x req_id=0x%x\n",
2872 skdev->name, __func__, __LINE__,
2873 skreq, skreq->id, req_id);
e67f86b3
AB		 2874 } else {
2875 /*
2876 * Capture the outcome and post it back to the
2877 * native request.
2878 */
fcd37eb3
JA		 2879 if (likely(cmp_status == SAM_STAT_GOOD))
2880 skd_end_request(skdev, skreq, 0);
2881 else
e67f86b3 2882 skd_resolve_req_exception(skdev, skreq);
e67f86b3
AB		 2883 }
2884
2885 /*
2886 * Release the skreq, its FIT msg (if one), timeout slot,
2887 * and queue depth.
2888 */
2889 skd_release_skreq(skdev, skreq);
2890
2891 /* skd_isr_comp_limit equal zero means no limit */
2892 if (limit) {
2893 if (++processed >= limit) {
2894 rc = 1;
2895 break;
2896 }
2897 }
2898 }
2899
2900 if ((skdev->state == SKD_DRVR_STATE_PAUSING)
2901 && (skdev->in_flight) == 0) {
2902 skdev->state = SKD_DRVR_STATE_PAUSED;
2903 wake_up_interruptible(&skdev->waitq);
2904 }
2905
2906 return rc;
2907}
2908
2909static void skd_complete_other(struct skd_device *skdev,
2910 volatile struct fit_completion_entry_v1 *skcomp,
2911 volatile struct fit_comp_error_info *skerr)
2912{
2913 u32 req_id = 0;
2914 u32 req_table;
2915 u32 req_slot;
2916 struct skd_special_context *skspcl;
2917
2918 req_id = skcomp->tag;
2919 req_table = req_id & SKD_ID_TABLE_MASK;
2920 req_slot = req_id & SKD_ID_SLOT_MASK;
2921
2e44b427 2922 pr_debug("%s:%s:%d table=0x%x id=0x%x slot=%d\n",
2923 skdev->name, __func__, __LINE__,
2924 req_table, req_id, req_slot);
e67f86b3
AB		 2925
2926 /*
2927 * Based on the request id, determine how to dispatch this completion.
2928 * This swich/case is finding the good cases and forwarding the
2929 * completion entry. Errors are reported below the switch.
2930 */
2931 switch (req_table) {
2932 case SKD_ID_RW_REQUEST:
2933 /*
2934 * The caller, skd_completion_posted_isr() above,
2935 * handles r/w requests. The only way we get here
2936 * is if the req_slot is out of bounds.
2937 */
2938 break;
2939
2940 case SKD_ID_SPECIAL_REQUEST:
2941 /*
2942 * Make sure the req_slot is in bounds and that the id
2943 * matches.
2944 */
2945 if (req_slot < skdev->n_special) {
2946 skspcl = &skdev->skspcl_table[req_slot];
2947 if (skspcl->req.id == req_id &&
2948 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2949 skd_complete_special(skdev,
2950 skcomp, skerr, skspcl);
2951 return;
2952 }
2953 }
2954 break;
2955
2956 case SKD_ID_INTERNAL:
2957 if (req_slot == 0) {
2958 skspcl = &skdev->internal_skspcl;
2959 if (skspcl->req.id == req_id &&
2960 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2961 skd_complete_internal(skdev,
2962 skcomp, skerr, skspcl);
2963 return;
2964 }
2965 }
2966 break;
2967
2968 case SKD_ID_FIT_MSG:
2969 /*
2970 * These id's should never appear in a completion record.
2971 */
2972 break;
2973
2974 default:
2975 /*
2976 * These id's should never appear anywhere;
2977 */
2978 break;
2979 }
2980
2981 /*
2982 * If we get here it is a bad or stale id.
2983 */
2984}
2985
2986static void skd_complete_special(struct skd_device *skdev,
2987 volatile struct fit_completion_entry_v1
2988 *skcomp,
2989 volatile struct fit_comp_error_info *skerr,
2990 struct skd_special_context *skspcl)
2991{
2e44b427 2992 pr_debug("%s:%s:%d completing special request %p\n",
2993 skdev->name, __func__, __LINE__, skspcl);
e67f86b3
AB		 2994 if (skspcl->orphaned) {
2995 /* Discard orphaned request */
2996 /* ?: Can this release directly or does it need
2997 * to use a worker? */
2e44b427 2998 pr_debug("%s:%s:%d release orphaned %p\n",
2999 skdev->name, __func__, __LINE__, skspcl);
e67f86b3
AB		 3000 skd_release_special(skdev, skspcl);
3001 return;
3002 }
3003
3004 skd_process_scsi_inq(skdev, skcomp, skerr, skspcl);
3005
3006 skspcl->req.state = SKD_REQ_STATE_COMPLETED;
3007 skspcl->req.completion = *skcomp;
3008 skspcl->req.err_info = *skerr;
3009
3010 skd_log_check_status(skdev, skspcl->req.completion.status, skerr->key,
3011 skerr->code, skerr->qual, skerr->fruc);
3012
3013 wake_up_interruptible(&skdev->waitq);
3014}
3015
3016/* assume spinlock is already held */
3017static void skd_release_special(struct skd_device *skdev,
3018 struct skd_special_context *skspcl)
3019{
3020 int i, was_depleted;
3021
3022 for (i = 0; i < skspcl->req.n_sg; i++) {
3023
3024 struct page *page = sg_page(&skspcl->req.sg[i]);
3025 __free_page(page);
3026 }
3027
3028 was_depleted = (skdev->skspcl_free_list == NULL);
3029
3030 skspcl->req.state = SKD_REQ_STATE_IDLE;
3031 skspcl->req.id += SKD_ID_INCR;
3032 skspcl->req.next =
3033 (struct skd_request_context *)skdev->skspcl_free_list;
3034 skdev->skspcl_free_list = (struct skd_special_context *)skspcl;
3035
3036 if (was_depleted) {
2e44b427 3037 pr_debug("%s:%s:%d skspcl was depleted\n",
3038 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3039 /* Free list was depleted. Their might be waiters. */
3040 wake_up_interruptible(&skdev->waitq);
3041 }
3042}
3043
3044static void skd_reset_skcomp(struct skd_device *skdev)
3045{
3046 u32 nbytes;
3047 struct fit_completion_entry_v1 *skcomp;
3048
3049 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
3050 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
3051
3052 memset(skdev->skcomp_table, 0, nbytes);
3053
3054 skdev->skcomp_ix = 0;
3055 skdev->skcomp_cycle = 1;
3056}
3057
3058/*
3059 *****************************************************************************
3060 * INTERRUPTS
3061 *****************************************************************************
3062 */
3063static void skd_completion_worker(struct work_struct *work)
3064{
3065 struct skd_device *skdev =
3066 container_of(work, struct skd_device, completion_worker);
3067 unsigned long flags;
3068 int flush_enqueued = 0;
3069
3070 spin_lock_irqsave(&skdev->lock, flags);
3071
3072 /*
3073 * pass in limit=0, which means no limit..
3074 * process everything in compq
3075 */
3076 skd_isr_completion_posted(skdev, 0, &flush_enqueued);
3077 skd_request_fn(skdev->queue);
3078
3079 spin_unlock_irqrestore(&skdev->lock, flags);
3080}
3081
3082static void skd_isr_msg_from_dev(struct skd_device *skdev);
3083
3084irqreturn_t
3085static skd_isr(int irq, void *ptr)
3086{
3087 struct skd_device *skdev;
3088 u32 intstat;
3089 u32 ack;
3090 int rc = 0;
3091 int deferred = 0;
3092 int flush_enqueued = 0;
3093
3094 skdev = (struct skd_device *)ptr;
3095 spin_lock(&skdev->lock);
3096
3097 for (;; ) {
3098 intstat = SKD_READL(skdev, FIT_INT_STATUS_HOST);
3099
3100 ack = FIT_INT_DEF_MASK;
3101 ack &= intstat;
3102
2e44b427 3103 pr_debug("%s:%s:%d intstat=0x%x ack=0x%x\n",
3104 skdev->name, __func__, __LINE__, intstat, ack);
e67f86b3
AB		 3105
3106 /* As long as there is an int pending on device, keep
3107 * running loop. When none, get out, but if we've never
3108 * done any processing, call completion handler?
3109 */
3110 if (ack == 0) {
3111 /* No interrupts on device, but run the completion
3112 * processor anyway?
3113 */
3114 if (rc == 0)
3115 if (likely (skdev->state
3116 == SKD_DRVR_STATE_ONLINE))
3117 deferred = 1;
3118 break;
3119 }
3120
3121 rc = IRQ_HANDLED;
3122
3123 SKD_WRITEL(skdev, ack, FIT_INT_STATUS_HOST);
3124
3125 if (likely((skdev->state != SKD_DRVR_STATE_LOAD) &&
3126 (skdev->state != SKD_DRVR_STATE_STOPPING))) {
3127 if (intstat & FIT_ISH_COMPLETION_POSTED) {
3128 /*
3129 * If we have already deferred completion
3130 * processing, don't bother running it again
3131 */
3132 if (deferred == 0)
3133 deferred =
3134 skd_isr_completion_posted(skdev,
3135 skd_isr_comp_limit, &flush_enqueued);
3136 }
3137
3138 if (intstat & FIT_ISH_FW_STATE_CHANGE) {
3139 skd_isr_fwstate(skdev);
3140 if (skdev->state == SKD_DRVR_STATE_FAULT ||
3141 skdev->state ==
3142 SKD_DRVR_STATE_DISAPPEARED) {
3143 spin_unlock(&skdev->lock);
3144 return rc;
3145 }
3146 }
3147
3148 if (intstat & FIT_ISH_MSG_FROM_DEV)
3149 skd_isr_msg_from_dev(skdev);
3150 }
3151 }
3152
3153 if (unlikely(flush_enqueued))
3154 skd_request_fn(skdev->queue);
3155
3156 if (deferred)
3157 schedule_work(&skdev->completion_worker);
3158 else if (!flush_enqueued)
3159 skd_request_fn(skdev->queue);
3160
3161 spin_unlock(&skdev->lock);
3162
3163 return rc;
3164}
3165
3166
3167static void skd_drive_fault(struct skd_device *skdev)
3168{
3169 skdev->state = SKD_DRVR_STATE_FAULT;
3170 pr_err("(%s): Drive FAULT\n", skd_name(skdev));
3171}
3172
3173static void skd_drive_disappeared(struct skd_device *skdev)
3174{
3175 skdev->state = SKD_DRVR_STATE_DISAPPEARED;
3176 pr_err("(%s): Drive DISAPPEARED\n", skd_name(skdev));
3177}
3178
3179static void skd_isr_fwstate(struct skd_device *skdev)
3180{
3181 u32 sense;
3182 u32 state;
3183 u32 mtd;
3184 int prev_driver_state = skdev->state;
3185
3186 sense = SKD_READL(skdev, FIT_STATUS);
3187 state = sense & FIT_SR_DRIVE_STATE_MASK;
3188
3189 pr_err("(%s): s1120 state %s(%d)=>%s(%d)\n",
3190 skd_name(skdev),
3191 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
3192 skd_drive_state_to_str(state), state);
3193
3194 skdev->drive_state = state;
3195
3196 switch (skdev->drive_state) {
3197 case FIT_SR_DRIVE_INIT:
3198 if (skdev->state == SKD_DRVR_STATE_PROTOCOL_MISMATCH) {
3199 skd_disable_interrupts(skdev);
3200 break;
3201 }
3202 if (skdev->state == SKD_DRVR_STATE_RESTARTING)
3203 skd_recover_requests(skdev, 0);
3204 if (skdev->state == SKD_DRVR_STATE_WAIT_BOOT) {
3205 skdev->timer_countdown = SKD_STARTING_TIMO;
3206 skdev->state = SKD_DRVR_STATE_STARTING;
3207 skd_soft_reset(skdev);
3208 break;
3209 }
3210 mtd = FIT_MXD_CONS(FIT_MTD_FITFW_INIT, 0, 0);
3211 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3212 skdev->last_mtd = mtd;
3213 break;
3214
3215 case FIT_SR_DRIVE_ONLINE:
3216 skdev->cur_max_queue_depth = skd_max_queue_depth;
3217 if (skdev->cur_max_queue_depth > skdev->dev_max_queue_depth)
3218 skdev->cur_max_queue_depth = skdev->dev_max_queue_depth;
3219
3220 skdev->queue_low_water_mark =
3221 skdev->cur_max_queue_depth * 2 / 3 + 1;
3222 if (skdev->queue_low_water_mark < 1)
3223 skdev->queue_low_water_mark = 1;
3224 pr_info(
3225 "(%s): Queue depth limit=%d dev=%d lowat=%d\n",
3226 skd_name(skdev),
3227 skdev->cur_max_queue_depth,
3228 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
3229
3230 skd_refresh_device_data(skdev);
3231 break;
3232
3233 case FIT_SR_DRIVE_BUSY:
3234 skdev->state = SKD_DRVR_STATE_BUSY;
3235 skdev->timer_countdown = SKD_BUSY_TIMO;
3236 skd_quiesce_dev(skdev);
3237 break;
3238 case FIT_SR_DRIVE_BUSY_SANITIZE:
3239 /* set timer for 3 seconds, we'll abort any unfinished
3240 * commands after that expires
3241 */
3242 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3243 skdev->timer_countdown = SKD_TIMER_SECONDS(3);
3244 skd_start_queue(skdev);
3245 break;
3246 case FIT_SR_DRIVE_BUSY_ERASE:
3247 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3248 skdev->timer_countdown = SKD_BUSY_TIMO;
3249 break;
3250 case FIT_SR_DRIVE_OFFLINE:
3251 skdev->state = SKD_DRVR_STATE_IDLE;
3252 break;
3253 case FIT_SR_DRIVE_SOFT_RESET:
3254 switch (skdev->state) {
3255 case SKD_DRVR_STATE_STARTING:
3256 case SKD_DRVR_STATE_RESTARTING:
3257 /* Expected by a caller of skd_soft_reset() */
3258 break;
3259 default:
3260 skdev->state = SKD_DRVR_STATE_RESTARTING;
3261 break;
3262 }
3263 break;
3264 case FIT_SR_DRIVE_FW_BOOTING:
2e44b427 3265 pr_debug("%s:%s:%d ISR FIT_SR_DRIVE_FW_BOOTING %s\n",
3266 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3267 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3268 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3269 break;
3270
3271 case FIT_SR_DRIVE_DEGRADED:
3272 case FIT_SR_PCIE_LINK_DOWN:
3273 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
3274 break;
3275
3276 case FIT_SR_DRIVE_FAULT:
3277 skd_drive_fault(skdev);
3278 skd_recover_requests(skdev, 0);
3279 skd_start_queue(skdev);
3280 break;
3281
3282 /* PCIe bus returned all Fs? */
3283 case 0xFF:
3284 pr_info("(%s): state=0x%x sense=0x%x\n",
3285 skd_name(skdev), state, sense);
3286 skd_drive_disappeared(skdev);
3287 skd_recover_requests(skdev, 0);
3288 skd_start_queue(skdev);
3289 break;
3290 default:
3291 /*
3292 * Uknown FW State. Wait for a state we recognize.
3293 */
3294 break;
3295 }
3296 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3297 skd_name(skdev),
3298 skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
3299 skd_skdev_state_to_str(skdev->state), skdev->state);
3300}
3301
3302static void skd_recover_requests(struct skd_device *skdev, int requeue)
3303{
3304 int i;
3305
3306 for (i = 0; i < skdev->num_req_context; i++) {
3307 struct skd_request_context *skreq = &skdev->skreq_table[i];
3308
3309 if (skreq->state == SKD_REQ_STATE_BUSY) {
3310 skd_log_skreq(skdev, skreq, "recover");
3311
3312 SKD_ASSERT((skreq->id & SKD_ID_INCR) != 0);
fcd37eb3 3313 SKD_ASSERT(skreq->req != NULL);
e67f86b3
AB		 3314
3315 /* Release DMA resources for the request. */
3316 if (skreq->n_sg > 0)
3317 skd_postop_sg_list(skdev, skreq);
3318
fcd37eb3
JA		 3319 if (requeue &&
3320 (unsigned long) ++skreq->req->special <
3321 SKD_MAX_RETRIES)
3322 skd_requeue_request(skdev, skreq);
3323 else
e67f86b3
AB		 3324 skd_end_request(skdev, skreq, -EIO);
3325
fcd37eb3 3326 skreq->req = NULL;
e67f86b3
AB		 3327
3328 skreq->state = SKD_REQ_STATE_IDLE;
3329 skreq->id += SKD_ID_INCR;
3330
3331
3332 }
3333 if (i > 0)
3334 skreq[-1].next = skreq;
3335 skreq->next = NULL;
3336 }
3337 skdev->skreq_free_list = skdev->skreq_table;
3338
3339 for (i = 0; i < skdev->num_fitmsg_context; i++) {
3340 struct skd_fitmsg_context *skmsg = &skdev->skmsg_table[i];
3341
3342 if (skmsg->state == SKD_MSG_STATE_BUSY) {
3343 skd_log_skmsg(skdev, skmsg, "salvaged");
3344 SKD_ASSERT((skmsg->id & SKD_ID_INCR) != 0);
3345 skmsg->state = SKD_MSG_STATE_IDLE;
3346 skmsg->id += SKD_ID_INCR;
3347 }
3348 if (i > 0)
3349 skmsg[-1].next = skmsg;
3350 skmsg->next = NULL;
3351 }
3352 skdev->skmsg_free_list = skdev->skmsg_table;
3353
3354 for (i = 0; i < skdev->n_special; i++) {
3355 struct skd_special_context *skspcl = &skdev->skspcl_table[i];
3356
3357 /* If orphaned, reclaim it because it has already been reported
3358 * to the process as an error (it was just waiting for
3359 * a completion that didn't come, and now it will never come)
3360 * If busy, change to a state that will cause it to error
3361 * out in the wait routine and let it do the normal
3362 * reporting and reclaiming
3363 */
3364 if (skspcl->req.state == SKD_REQ_STATE_BUSY) {
3365 if (skspcl->orphaned) {
2e44b427 3366 pr_debug("%s:%s:%d orphaned %p\n",
3367 skdev->name, __func__, __LINE__,
3368 skspcl);
e67f86b3
AB		 3369 skd_release_special(skdev, skspcl);
3370 } else {
2e44b427 3371 pr_debug("%s:%s:%d not orphaned %p\n",
3372 skdev->name, __func__, __LINE__,
3373 skspcl);
e67f86b3
AB		 3374 skspcl->req.state = SKD_REQ_STATE_ABORTED;
3375 }
3376 }
3377 }
3378 skdev->skspcl_free_list = skdev->skspcl_table;
3379
3380 for (i = 0; i < SKD_N_TIMEOUT_SLOT; i++)
3381 skdev->timeout_slot[i] = 0;
3382
3383 skdev->in_flight = 0;
3384}
3385
3386static void skd_isr_msg_from_dev(struct skd_device *skdev)
3387{
3388 u32 mfd;
3389 u32 mtd;
3390 u32 data;
3391
3392 mfd = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3393
2e44b427 3394 pr_debug("%s:%s:%d mfd=0x%x last_mtd=0x%x\n",
3395 skdev->name, __func__, __LINE__, mfd, skdev->last_mtd);
e67f86b3
AB		 3396
3397 /* ignore any mtd that is an ack for something we didn't send */
3398 if (FIT_MXD_TYPE(mfd) != FIT_MXD_TYPE(skdev->last_mtd))
3399 return;
3400
3401 switch (FIT_MXD_TYPE(mfd)) {
3402 case FIT_MTD_FITFW_INIT:
3403 skdev->proto_ver = FIT_PROTOCOL_MAJOR_VER(mfd);
3404
3405 if (skdev->proto_ver != FIT_PROTOCOL_VERSION_1) {
3406 pr_err("(%s): protocol mismatch\n",
3407 skdev->name);
3408 pr_err("(%s): got=%d support=%d\n",
3409 skdev->name, skdev->proto_ver,
3410 FIT_PROTOCOL_VERSION_1);
3411 pr_err("(%s): please upgrade driver\n",
3412 skdev->name);
3413 skdev->state = SKD_DRVR_STATE_PROTOCOL_MISMATCH;
3414 skd_soft_reset(skdev);
3415 break;
3416 }
3417 mtd = FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH, 0, 0);
3418 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3419 skdev->last_mtd = mtd;
3420 break;
3421
3422 case FIT_MTD_GET_CMDQ_DEPTH:
3423 skdev->dev_max_queue_depth = FIT_MXD_DATA(mfd);
3424 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH, 0,
3425 SKD_N_COMPLETION_ENTRY);
3426 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3427 skdev->last_mtd = mtd;
3428 break;
3429
3430 case FIT_MTD_SET_COMPQ_DEPTH:
3431 SKD_WRITEQ(skdev, skdev->cq_dma_address, FIT_MSG_TO_DEVICE_ARG);
3432 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR, 0, 0);
3433 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3434 skdev->last_mtd = mtd;
3435 break;
3436
3437 case FIT_MTD_SET_COMPQ_ADDR:
3438 skd_reset_skcomp(skdev);
3439 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID, 0, skdev->devno);
3440 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3441 skdev->last_mtd = mtd;
3442 break;
3443
3444 case FIT_MTD_CMD_LOG_HOST_ID:
3445 skdev->connect_time_stamp = get_seconds();
3446 data = skdev->connect_time_stamp & 0xFFFF;
3447 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO, 0, data);
3448 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3449 skdev->last_mtd = mtd;
3450 break;
3451
3452 case FIT_MTD_CMD_LOG_TIME_STAMP_LO:
3453 skdev->drive_jiffies = FIT_MXD_DATA(mfd);
3454 data = (skdev->connect_time_stamp >> 16) & 0xFFFF;
3455 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI, 0, data);
3456 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3457 skdev->last_mtd = mtd;
3458 break;
3459
3460 case FIT_MTD_CMD_LOG_TIME_STAMP_HI:
3461 skdev->drive_jiffies |= (FIT_MXD_DATA(mfd) << 16);
3462 mtd = FIT_MXD_CONS(FIT_MTD_ARM_QUEUE, 0, 0);
3463 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3464 skdev->last_mtd = mtd;
3465
3466 pr_err("(%s): Time sync driver=0x%x device=0x%x\n",
3467 skd_name(skdev),
3468 skdev->connect_time_stamp, skdev->drive_jiffies);
3469 break;
3470
3471 case FIT_MTD_ARM_QUEUE:
3472 skdev->last_mtd = 0;
3473 /*
3474 * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
3475 */
3476 break;
3477
3478 default:
3479 break;
3480 }
3481}
3482
3483static void skd_disable_interrupts(struct skd_device *skdev)
3484{
3485 u32 sense;
3486
3487 sense = SKD_READL(skdev, FIT_CONTROL);
3488 sense &= ~FIT_CR_ENABLE_INTERRUPTS;
3489 SKD_WRITEL(skdev, sense, FIT_CONTROL);
2e44b427 3490 pr_debug("%s:%s:%d sense 0x%x\n",
3491 skdev->name, __func__, __LINE__, sense);
e67f86b3
AB		 3492
3493 /* Note that the 1s is written. A 1-bit means
3494 * disable, a 0 means enable.
3495 */
3496 SKD_WRITEL(skdev, ~0, FIT_INT_MASK_HOST);
3497}
3498
3499static void skd_enable_interrupts(struct skd_device *skdev)
3500{
3501 u32 val;
3502
3503 /* unmask interrupts first */
3504 val = FIT_ISH_FW_STATE_CHANGE +
3505 FIT_ISH_COMPLETION_POSTED + FIT_ISH_MSG_FROM_DEV;
3506
3507 /* Note that the compliment of mask is written. A 1-bit means
3508 * disable, a 0 means enable. */
3509 SKD_WRITEL(skdev, ~val, FIT_INT_MASK_HOST);
2e44b427 3510 pr_debug("%s:%s:%d interrupt mask=0x%x\n",
3511 skdev->name, __func__, __LINE__, ~val);
e67f86b3
AB		 3512
3513 val = SKD_READL(skdev, FIT_CONTROL);
3514 val |= FIT_CR_ENABLE_INTERRUPTS;
2e44b427 3515 pr_debug("%s:%s:%d control=0x%x\n",
3516 skdev->name, __func__, __LINE__, val);
e67f86b3
AB		 3517 SKD_WRITEL(skdev, val, FIT_CONTROL);
3518}
3519
3520/*
3521 *****************************************************************************
3522 * START, STOP, RESTART, QUIESCE, UNQUIESCE
3523 *****************************************************************************
3524 */
3525
3526static void skd_soft_reset(struct skd_device *skdev)
3527{
3528 u32 val;
3529
3530 val = SKD_READL(skdev, FIT_CONTROL);
3531 val |= (FIT_CR_SOFT_RESET);
2e44b427 3532 pr_debug("%s:%s:%d control=0x%x\n",
3533 skdev->name, __func__, __LINE__, val);
e67f86b3
AB		 3534 SKD_WRITEL(skdev, val, FIT_CONTROL);
3535}
3536
3537static void skd_start_device(struct skd_device *skdev)
3538{
3539 unsigned long flags;
3540 u32 sense;
3541 u32 state;
3542
3543 spin_lock_irqsave(&skdev->lock, flags);
3544
3545 /* ack all ghost interrupts */
3546 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3547
3548 sense = SKD_READL(skdev, FIT_STATUS);
3549
2e44b427 3550 pr_debug("%s:%s:%d initial status=0x%x\n",
3551 skdev->name, __func__, __LINE__, sense);
e67f86b3
AB		 3552
3553 state = sense & FIT_SR_DRIVE_STATE_MASK;
3554 skdev->drive_state = state;
3555 skdev->last_mtd = 0;
3556
3557 skdev->state = SKD_DRVR_STATE_STARTING;
3558 skdev->timer_countdown = SKD_STARTING_TIMO;
3559
3560 skd_enable_interrupts(skdev);
3561
3562 switch (skdev->drive_state) {
3563 case FIT_SR_DRIVE_OFFLINE:
3564 pr_err("(%s): Drive offline...\n", skd_name(skdev));
3565 break;
3566
3567 case FIT_SR_DRIVE_FW_BOOTING:
2e44b427 3568 pr_debug("%s:%s:%d FIT_SR_DRIVE_FW_BOOTING %s\n",
3569 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3570 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3571 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3572 break;
3573
3574 case FIT_SR_DRIVE_BUSY_SANITIZE:
3575 pr_info("(%s): Start: BUSY_SANITIZE\n",
3576 skd_name(skdev));
3577 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3578 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3579 break;
3580
3581 case FIT_SR_DRIVE_BUSY_ERASE:
3582 pr_info("(%s): Start: BUSY_ERASE\n", skd_name(skdev));
3583 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3584 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3585 break;
3586
3587 case FIT_SR_DRIVE_INIT:
3588 case FIT_SR_DRIVE_ONLINE:
3589 skd_soft_reset(skdev);
3590 break;
3591
3592 case FIT_SR_DRIVE_BUSY:
3593 pr_err("(%s): Drive Busy...\n", skd_name(skdev));
3594 skdev->state = SKD_DRVR_STATE_BUSY;
3595 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3596 break;
3597
3598 case FIT_SR_DRIVE_SOFT_RESET:
3599 pr_err("(%s) drive soft reset in prog\n",
3600 skd_name(skdev));
3601 break;
3602
3603 case FIT_SR_DRIVE_FAULT:
3604 /* Fault state is bad...soft reset won't do it...
3605 * Hard reset, maybe, but does it work on device?
3606 * For now, just fault so the system doesn't hang.
3607 */
3608 skd_drive_fault(skdev);
3609 /*start the queue so we can respond with error to requests */
2e44b427 3610 pr_debug("%s:%s:%d starting %s queue\n",
3611 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3612 skd_start_queue(skdev);
3613 skdev->gendisk_on = -1;
3614 wake_up_interruptible(&skdev->waitq);
3615 break;
3616
3617 case 0xFF:
3618 /* Most likely the device isn't there or isn't responding
3619 * to the BAR1 addresses. */
3620 skd_drive_disappeared(skdev);
3621 /*start the queue so we can respond with error to requests */
2e44b427 3622 pr_debug("%s:%s:%d starting %s queue to error-out reqs\n",
3623 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3624 skd_start_queue(skdev);
3625 skdev->gendisk_on = -1;
3626 wake_up_interruptible(&skdev->waitq);
3627 break;
3628
3629 default:
3630 pr_err("(%s) Start: unknown state %x\n",
3631 skd_name(skdev), skdev->drive_state);
3632 break;
3633 }
3634
3635 state = SKD_READL(skdev, FIT_CONTROL);
2e44b427 3636 pr_debug("%s:%s:%d FIT Control Status=0x%x\n",
3637 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3638
3639 state = SKD_READL(skdev, FIT_INT_STATUS_HOST);
2e44b427 3640 pr_debug("%s:%s:%d Intr Status=0x%x\n",
3641 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3642
3643 state = SKD_READL(skdev, FIT_INT_MASK_HOST);
2e44b427 3644 pr_debug("%s:%s:%d Intr Mask=0x%x\n",
3645 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3646
3647 state = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
2e44b427 3648 pr_debug("%s:%s:%d Msg from Dev=0x%x\n",
3649 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3650
3651 state = SKD_READL(skdev, FIT_HW_VERSION);
2e44b427 3652 pr_debug("%s:%s:%d HW version=0x%x\n",
3653 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3654
3655 spin_unlock_irqrestore(&skdev->lock, flags);
3656}
3657
3658static void skd_stop_device(struct skd_device *skdev)
3659{
3660 unsigned long flags;
3661 struct skd_special_context *skspcl = &skdev->internal_skspcl;
3662 u32 dev_state;
3663 int i;
3664
3665 spin_lock_irqsave(&skdev->lock, flags);
3666
3667 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
3668 pr_err("(%s): skd_stop_device not online no sync\n",
3669 skd_name(skdev));
3670 goto stop_out;
3671 }
3672
3673 if (skspcl->req.state != SKD_REQ_STATE_IDLE) {
3674 pr_err("(%s): skd_stop_device no special\n",
3675 skd_name(skdev));
3676 goto stop_out;
3677 }
3678
3679 skdev->state = SKD_DRVR_STATE_SYNCING;
3680 skdev->sync_done = 0;
3681
3682 skd_send_internal_skspcl(skdev, skspcl, SYNCHRONIZE_CACHE);
3683
3684 spin_unlock_irqrestore(&skdev->lock, flags);
3685
3686 wait_event_interruptible_timeout(skdev->waitq,
3687 (skdev->sync_done), (10 * HZ));
3688
3689 spin_lock_irqsave(&skdev->lock, flags);
3690
3691 switch (skdev->sync_done) {
3692 case 0:
3693 pr_err("(%s): skd_stop_device no sync\n",
3694 skd_name(skdev));
3695 break;
3696 case 1:
3697 pr_err("(%s): skd_stop_device sync done\n",
3698 skd_name(skdev));
3699 break;
3700 default:
3701 pr_err("(%s): skd_stop_device sync error\n",
3702 skd_name(skdev));
3703 }
3704
3705stop_out:
3706 skdev->state = SKD_DRVR_STATE_STOPPING;
3707 spin_unlock_irqrestore(&skdev->lock, flags);
3708
3709 skd_kill_timer(skdev);
3710
3711 spin_lock_irqsave(&skdev->lock, flags);
3712 skd_disable_interrupts(skdev);
3713
3714 /* ensure all ints on device are cleared */
3715 /* soft reset the device to unload with a clean slate */
3716 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3717 SKD_WRITEL(skdev, FIT_CR_SOFT_RESET, FIT_CONTROL);
3718
3719 spin_unlock_irqrestore(&skdev->lock, flags);
3720
3721 /* poll every 100ms, 1 second timeout */
3722 for (i = 0; i < 10; i++) {
3723 dev_state =
3724 SKD_READL(skdev, FIT_STATUS) & FIT_SR_DRIVE_STATE_MASK;
3725 if (dev_state == FIT_SR_DRIVE_INIT)
3726 break;
3727 set_current_state(TASK_INTERRUPTIBLE);
3728 schedule_timeout(msecs_to_jiffies(100));
3729 }
3730
3731 if (dev_state != FIT_SR_DRIVE_INIT)
3732 pr_err("(%s): skd_stop_device state error 0x%02x\n",
3733 skd_name(skdev), dev_state);
3734}
3735
3736/* assume spinlock is held */
3737static void skd_restart_device(struct skd_device *skdev)
3738{
3739 u32 state;
3740
3741 /* ack all ghost interrupts */
3742 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3743
3744 state = SKD_READL(skdev, FIT_STATUS);
3745
2e44b427 3746 pr_debug("%s:%s:%d drive status=0x%x\n",
3747 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3748
3749 state &= FIT_SR_DRIVE_STATE_MASK;
3750 skdev->drive_state = state;
3751 skdev->last_mtd = 0;
3752
3753 skdev->state = SKD_DRVR_STATE_RESTARTING;
3754 skdev->timer_countdown = SKD_RESTARTING_TIMO;
3755
3756 skd_soft_reset(skdev);
3757}
3758
3759/* assume spinlock is held */
3760static int skd_quiesce_dev(struct skd_device *skdev)
3761{
3762 int rc = 0;
3763
3764 switch (skdev->state) {
3765 case SKD_DRVR_STATE_BUSY:
3766 case SKD_DRVR_STATE_BUSY_IMMINENT:
2e44b427 3767 pr_debug("%s:%s:%d stopping %s queue\n",
3768 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3769 skd_stop_queue(skdev);
3770 break;
3771 case SKD_DRVR_STATE_ONLINE:
3772 case SKD_DRVR_STATE_STOPPING:
3773 case SKD_DRVR_STATE_SYNCING:
3774 case SKD_DRVR_STATE_PAUSING:
3775 case SKD_DRVR_STATE_PAUSED:
3776 case SKD_DRVR_STATE_STARTING:
3777 case SKD_DRVR_STATE_RESTARTING:
3778 case SKD_DRVR_STATE_RESUMING:
3779 default:
3780 rc = -EINVAL;
2e44b427 3781 pr_debug("%s:%s:%d state [%d] not implemented\n",
3782 skdev->name, __func__, __LINE__, skdev->state);
e67f86b3
AB		 3783 }
3784 return rc;
3785}
3786
3787/* assume spinlock is held */
3788static int skd_unquiesce_dev(struct skd_device *skdev)
3789{
3790 int prev_driver_state = skdev->state;
3791
3792 skd_log_skdev(skdev, "unquiesce");
3793 if (skdev->state == SKD_DRVR_STATE_ONLINE) {
2e44b427 3794 pr_debug("%s:%s:%d **** device already ONLINE\n",
3795 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3796 return 0;
3797 }
3798 if (skdev->drive_state != FIT_SR_DRIVE_ONLINE) {
3799 /*
3800 * If there has been an state change to other than
3801 * ONLINE, we will rely on controller state change
3802 * to come back online and restart the queue.
3803 * The BUSY state means that driver is ready to
3804 * continue normal processing but waiting for controller
3805 * to become available.
3806 */
3807 skdev->state = SKD_DRVR_STATE_BUSY;
2e44b427 3808 pr_debug("%s:%s:%d drive BUSY state\n",
3809 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3810 return 0;
3811 }
3812
3813 /*
3814 * Drive has just come online, driver is either in startup,
3815 * paused performing a task, or bust waiting for hardware.
3816 */
3817 switch (skdev->state) {
3818 case SKD_DRVR_STATE_PAUSED:
3819 case SKD_DRVR_STATE_BUSY:
3820 case SKD_DRVR_STATE_BUSY_IMMINENT:
3821 case SKD_DRVR_STATE_BUSY_ERASE:
3822 case SKD_DRVR_STATE_STARTING:
3823 case SKD_DRVR_STATE_RESTARTING:
3824 case SKD_DRVR_STATE_FAULT:
3825 case SKD_DRVR_STATE_IDLE:
3826 case SKD_DRVR_STATE_LOAD:
3827 skdev->state = SKD_DRVR_STATE_ONLINE;
3828 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3829 skd_name(skdev),
3830 skd_skdev_state_to_str(prev_driver_state),
3831 prev_driver_state, skd_skdev_state_to_str(skdev->state),
3832 skdev->state);
2e44b427 3833 pr_debug("%s:%s:%d **** device ONLINE...starting block queue\n",
3834 skdev->name, __func__, __LINE__);
3835 pr_debug("%s:%s:%d starting %s queue\n",
3836 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3837 pr_info("(%s): STEC s1120 ONLINE\n", skd_name(skdev));
3838 skd_start_queue(skdev);
3839 skdev->gendisk_on = 1;
3840 wake_up_interruptible(&skdev->waitq);
3841 break;
3842
3843 case SKD_DRVR_STATE_DISAPPEARED:
3844 default:
2e44b427 3845 pr_debug("%s:%s:%d **** driver state %d, not implemented \n",
3846 skdev->name, __func__, __LINE__,
3847 skdev->state);
e67f86b3
AB		 3848 return -EBUSY;
3849 }
3850 return 0;
3851}
3852
3853/*
3854 *****************************************************************************
3855 * PCIe MSI/MSI-X INTERRUPT HANDLERS
3856 *****************************************************************************
3857 */
3858
3859static irqreturn_t skd_reserved_isr(int irq, void *skd_host_data)
3860{
3861 struct skd_device *skdev = skd_host_data;
3862 unsigned long flags;
3863
3864 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3865 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3866 skdev->name, __func__, __LINE__,
3867 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3868 pr_err("(%s): MSIX reserved irq %d = 0x%x\n", skd_name(skdev),
3869 irq, SKD_READL(skdev, FIT_INT_STATUS_HOST));
3870 SKD_WRITEL(skdev, FIT_INT_RESERVED_MASK, FIT_INT_STATUS_HOST);
3871 spin_unlock_irqrestore(&skdev->lock, flags);
3872 return IRQ_HANDLED;
3873}
3874
3875static irqreturn_t skd_statec_isr(int irq, void *skd_host_data)
3876{
3877 struct skd_device *skdev = skd_host_data;
3878 unsigned long flags;
3879
3880 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3881 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3882 skdev->name, __func__, __LINE__,
3883 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3884 SKD_WRITEL(skdev, FIT_ISH_FW_STATE_CHANGE, FIT_INT_STATUS_HOST);
3885 skd_isr_fwstate(skdev);
3886 spin_unlock_irqrestore(&skdev->lock, flags);
3887 return IRQ_HANDLED;
3888}
3889
3890static irqreturn_t skd_comp_q(int irq, void *skd_host_data)
3891{
3892 struct skd_device *skdev = skd_host_data;
3893 unsigned long flags;
3894 int flush_enqueued = 0;
3895 int deferred;
3896
3897 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3898 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3899 skdev->name, __func__, __LINE__,
3900 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3901 SKD_WRITEL(skdev, FIT_ISH_COMPLETION_POSTED, FIT_INT_STATUS_HOST);
3902 deferred = skd_isr_completion_posted(skdev, skd_isr_comp_limit,
3903 &flush_enqueued);
3904
3905 if (flush_enqueued)
3906 skd_request_fn(skdev->queue);
3907
3908 if (deferred)
3909 schedule_work(&skdev->completion_worker);
3910 else if (!flush_enqueued)
3911 skd_request_fn(skdev->queue);
3912
3913 spin_unlock_irqrestore(&skdev->lock, flags);
3914
3915 return IRQ_HANDLED;
3916}
3917
3918static irqreturn_t skd_msg_isr(int irq, void *skd_host_data)
3919{
3920 struct skd_device *skdev = skd_host_data;
3921 unsigned long flags;
3922
3923 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3924 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3925 skdev->name, __func__, __LINE__,
3926 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3927 SKD_WRITEL(skdev, FIT_ISH_MSG_FROM_DEV, FIT_INT_STATUS_HOST);
3928 skd_isr_msg_from_dev(skdev);
3929 spin_unlock_irqrestore(&skdev->lock, flags);
3930 return IRQ_HANDLED;
3931}
3932
3933static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
3934{
3935 struct skd_device *skdev = skd_host_data;
3936 unsigned long flags;
3937
3938 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3939 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3940 skdev->name, __func__, __LINE__,
3941 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3942 SKD_WRITEL(skdev, FIT_INT_QUEUE_FULL, FIT_INT_STATUS_HOST);
3943 spin_unlock_irqrestore(&skdev->lock, flags);
3944 return IRQ_HANDLED;
3945}
3946
3947/*
3948 *****************************************************************************
3949 * PCIe MSI/MSI-X SETUP
3950 *****************************************************************************
3951 */
3952
3953struct skd_msix_entry {
3954 int have_irq;
3955 u32 vector;
3956 u32 entry;
3957 struct skd_device *rsp;
3958 char isr_name[30];
3959};
3960
3961struct skd_init_msix_entry {
3962 const char *name;
3963 irq_handler_t handler;
3964};
3965
3966#define SKD_MAX_MSIX_COUNT 13
3967#define SKD_MIN_MSIX_COUNT 7
3968#define SKD_BASE_MSIX_IRQ 4
3969
3970static struct skd_init_msix_entry msix_entries[SKD_MAX_MSIX_COUNT] = {
3971 { "(DMA 0)", skd_reserved_isr },
3972 { "(DMA 1)", skd_reserved_isr },
3973 { "(DMA 2)", skd_reserved_isr },
3974 { "(DMA 3)", skd_reserved_isr },
3975 { "(State Change)", skd_statec_isr },
3976 { "(COMPL_Q)", skd_comp_q },
3977 { "(MSG)", skd_msg_isr },
3978 { "(Reserved)", skd_reserved_isr },
3979 { "(Reserved)", skd_reserved_isr },
3980 { "(Queue Full 0)", skd_qfull_isr },
3981 { "(Queue Full 1)", skd_qfull_isr },
3982 { "(Queue Full 2)", skd_qfull_isr },
3983 { "(Queue Full 3)", skd_qfull_isr },
3984};
3985
3986static void skd_release_msix(struct skd_device *skdev)
3987{
3988 struct skd_msix_entry *qentry;
3989 int i;
3990
3991 if (skdev->msix_entries == NULL)
3992 return;
3993 for (i = 0; i < skdev->msix_count; i++) {
3994 qentry = &skdev->msix_entries[i];
3995 skdev = qentry->rsp;
3996
3997 if (qentry->have_irq)
3998 devm_free_irq(&skdev->pdev->dev,
3999 qentry->vector, qentry->rsp);
4000 }
4001 pci_disable_msix(skdev->pdev);
4002 kfree(skdev->msix_entries);
4003 skdev->msix_count = 0;
4004 skdev->msix_entries = NULL;
4005}
4006
4007static int skd_acquire_msix(struct skd_device *skdev)
4008{
4009 int i, rc;
4010 struct pci_dev *pdev;
4011 struct msix_entry *entries = NULL;
4012 struct skd_msix_entry *qentry;
4013
4014 pdev = skdev->pdev;
4015 skdev->msix_count = SKD_MAX_MSIX_COUNT;
4016 entries = kzalloc(sizeof(struct msix_entry) * SKD_MAX_MSIX_COUNT,
4017 GFP_KERNEL);
4018 if (!entries)
4019 return -ENOMEM;
4020
4021 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++)
4022 entries[i].entry = i;
4023
4024 rc = pci_enable_msix(pdev, entries, SKD_MAX_MSIX_COUNT);
4025 if (rc < 0)
4026 goto msix_out;
4027 if (rc) {
4028 if (rc < SKD_MIN_MSIX_COUNT) {
4029 pr_err("(%s): failed to enable MSI-X %d\n",
4030 skd_name(skdev), rc);
4031 goto msix_out;
4032 }
2e44b427 4033 pr_debug("%s:%s:%d %s: <%s> allocated %d MSI-X vectors\n",
4034 skdev->name, __func__, __LINE__,
4035 pci_name(pdev), skdev->name, rc);
e67f86b3
AB		 4036
4037 skdev->msix_count = rc;
4038 rc = pci_enable_msix(pdev, entries, skdev->msix_count);
4039 if (rc) {
4040 pr_err("(%s): failed to enable MSI-X "
4041 "support (%d) %d\n",
4042 skd_name(skdev), skdev->msix_count, rc);
4043 goto msix_out;
4044 }
4045 }
4046 skdev->msix_entries = kzalloc(sizeof(struct skd_msix_entry) *
4047 skdev->msix_count, GFP_KERNEL);
4048 if (!skdev->msix_entries) {
4049 rc = -ENOMEM;
4050 skdev->msix_count = 0;
4051 pr_err("(%s): msix table allocation error\n",
4052 skd_name(skdev));
4053 goto msix_out;
4054 }
4055
4056 qentry = skdev->msix_entries;
4057 for (i = 0; i < skdev->msix_count; i++) {
4058 qentry->vector = entries[i].vector;
4059 qentry->entry = entries[i].entry;
4060 qentry->rsp = NULL;
4061 qentry->have_irq = 0;
2e44b427 4062 pr_debug("%s:%s:%d %s: <%s> msix (%d) vec %d, entry %x\n",
4063 skdev->name, __func__, __LINE__,
4064 pci_name(pdev), skdev->name,
4065 i, qentry->vector, qentry->entry);
e67f86b3
AB		 4066 qentry++;
4067 }
4068
4069 /* Enable MSI-X vectors for the base queue */
4070 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
4071 qentry = &skdev->msix_entries[i];
4072 snprintf(qentry->isr_name, sizeof(qentry->isr_name),
4073 "%s%d-msix %s", DRV_NAME, skdev->devno,
4074 msix_entries[i].name);
4075 rc = devm_request_irq(&skdev->pdev->dev, qentry->vector,
4076 msix_entries[i].handler, 0,
4077 qentry->isr_name, skdev);
4078 if (rc) {
4079 pr_err("(%s): Unable to register(%d) MSI-X "
4080 "handler %d: %s\n",
4081 skd_name(skdev), rc, i, qentry->isr_name);
4082 goto msix_out;
4083 } else {
4084 qentry->have_irq = 1;
4085 qentry->rsp = skdev;
4086 }
4087 }
2e44b427 4088 pr_debug("%s:%s:%d %s: <%s> msix %d irq(s) enabled\n",
4089 skdev->name, __func__, __LINE__,
4090 pci_name(pdev), skdev->name, skdev->msix_count);
e67f86b3
AB		 4091 return 0;
4092
4093msix_out:
4094 if (entries)
4095 kfree(entries);
4096 skd_release_msix(skdev);
4097 return rc;
4098}
4099
4100static int skd_acquire_irq(struct skd_device *skdev)
4101{
4102 int rc;
4103 struct pci_dev *pdev;
4104
4105 pdev = skdev->pdev;
4106 skdev->msix_count = 0;
4107
4108RETRY_IRQ_TYPE:
4109 switch (skdev->irq_type) {
4110 case SKD_IRQ_MSIX:
4111 rc = skd_acquire_msix(skdev);
4112 if (!rc)
4113 pr_info("(%s): MSI-X %d irqs enabled\n",
4114 skd_name(skdev), skdev->msix_count);
4115 else {
4116 pr_err(
4117 "(%s): failed to enable MSI-X, re-trying with MSI %d\n",
4118 skd_name(skdev), rc);
4119 skdev->irq_type = SKD_IRQ_MSI;
4120 goto RETRY_IRQ_TYPE;
4121 }
4122 break;
4123 case SKD_IRQ_MSI:
4124 snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d-msi",
4125 DRV_NAME, skdev->devno);
4126 rc = pci_enable_msi(pdev);
4127 if (!rc) {
4128 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr, 0,
4129 skdev->isr_name, skdev);
4130 if (rc) {
4131 pci_disable_msi(pdev);
4132 pr_err(
4133 "(%s): failed to allocate the MSI interrupt %d\n",
4134 skd_name(skdev), rc);
4135 goto RETRY_IRQ_LEGACY;
4136 }
4137 pr_info("(%s): MSI irq %d enabled\n",
4138 skd_name(skdev), pdev->irq);
4139 } else {
4140RETRY_IRQ_LEGACY:
4141 pr_err(
4142 "(%s): failed to enable MSI, re-trying with LEGACY %d\n",
4143 skd_name(skdev), rc);
4144 skdev->irq_type = SKD_IRQ_LEGACY;
4145 goto RETRY_IRQ_TYPE;
4146 }
4147 break;
4148 case SKD_IRQ_LEGACY:
4149 snprintf(skdev->isr_name, sizeof(skdev->isr_name),
4150 "%s%d-legacy", DRV_NAME, skdev->devno);
4151 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
4152 IRQF_SHARED, skdev->isr_name, skdev);
4153 if (!rc)
4154 pr_info("(%s): LEGACY irq %d enabled\n",
4155 skd_name(skdev), pdev->irq);
4156 else
4157 pr_err("(%s): request LEGACY irq error %d\n",
4158 skd_name(skdev), rc);
4159 break;
4160 default:
4161 pr_info("(%s): irq_type %d invalid, re-set to %d\n",
4162 skd_name(skdev), skdev->irq_type, SKD_IRQ_DEFAULT);
4163 skdev->irq_type = SKD_IRQ_LEGACY;
4164 goto RETRY_IRQ_TYPE;
4165 }
4166 return rc;
4167}
4168
4169static void skd_release_irq(struct skd_device *skdev)
4170{
4171 switch (skdev->irq_type) {
4172 case SKD_IRQ_MSIX:
4173 skd_release_msix(skdev);
4174 break;
4175 case SKD_IRQ_MSI:
4176 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4177 pci_disable_msi(skdev->pdev);
4178 break;
4179 case SKD_IRQ_LEGACY:
4180 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4181 break;
4182 default:
4183 pr_err("(%s): wrong irq type %d!",
4184 skd_name(skdev), skdev->irq_type);
4185 break;
4186 }
4187}
4188
4189/*
4190 *****************************************************************************
4191 * CONSTRUCT
4192 *****************************************************************************
4193 */
4194
4195static int skd_cons_skcomp(struct skd_device *skdev);
4196static int skd_cons_skmsg(struct skd_device *skdev);
4197static int skd_cons_skreq(struct skd_device *skdev);
4198static int skd_cons_skspcl(struct skd_device *skdev);
4199static int skd_cons_sksb(struct skd_device *skdev);
4200static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4201 u32 n_sg,
4202 dma_addr_t *ret_dma_addr);
4203static int skd_cons_disk(struct skd_device *skdev);
4204
4205#define SKD_N_DEV_TABLE 16u
4206static u32 skd_next_devno;
4207
4208static struct skd_device *skd_construct(struct pci_dev *pdev)
4209{
4210 struct skd_device *skdev;
4211 int blk_major = skd_major;
4212 int rc;
4213
4214 skdev = kzalloc(sizeof(*skdev), GFP_KERNEL);
4215
4216 if (!skdev) {
4217 pr_err(PFX "(%s): memory alloc failure\n",
4218 pci_name(pdev));
4219 return NULL;
4220 }
4221
4222 skdev->state = SKD_DRVR_STATE_LOAD;
4223 skdev->pdev = pdev;
4224 skdev->devno = skd_next_devno++;
4225 skdev->major = blk_major;
4226 skdev->irq_type = skd_isr_type;
4227 sprintf(skdev->name, DRV_NAME "%d", skdev->devno);
4228 skdev->dev_max_queue_depth = 0;
4229
4230 skdev->num_req_context = skd_max_queue_depth;
4231 skdev->num_fitmsg_context = skd_max_queue_depth;
4232 skdev->n_special = skd_max_pass_thru;
4233 skdev->cur_max_queue_depth = 1;
4234 skdev->queue_low_water_mark = 1;
4235 skdev->proto_ver = 99;
4236 skdev->sgs_per_request = skd_sgs_per_request;
4237 skdev->dbg_level = skd_dbg_level;
4238
e67f86b3
AB		 4239 atomic_set(&skdev->device_count, 0);
4240
4241 spin_lock_init(&skdev->lock);
4242
4243 INIT_WORK(&skdev->completion_worker, skd_completion_worker);
e67f86b3 4244
2e44b427 4245 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4246 rc = skd_cons_skcomp(skdev);
4247 if (rc < 0)
4248 goto err_out;
4249
2e44b427 4250 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4251 rc = skd_cons_skmsg(skdev);
4252 if (rc < 0)
4253 goto err_out;
4254
2e44b427 4255 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4256 rc = skd_cons_skreq(skdev);
4257 if (rc < 0)
4258 goto err_out;
4259
2e44b427 4260 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4261 rc = skd_cons_skspcl(skdev);
4262 if (rc < 0)
4263 goto err_out;
4264
2e44b427 4265 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4266 rc = skd_cons_sksb(skdev);
4267 if (rc < 0)
4268 goto err_out;
4269
2e44b427 4270 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4271 rc = skd_cons_disk(skdev);
4272 if (rc < 0)
4273 goto err_out;
4274
2e44b427 4275 pr_debug("%s:%s:%d VICTORY\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4276 return skdev;
4277
4278err_out:
2e44b427 4279 pr_debug("%s:%s:%d construct failed\n",
4280 skdev->name, __func__, __LINE__);
e67f86b3
AB		 4281 skd_destruct(skdev);
4282 return NULL;
4283}
4284
4285static int skd_cons_skcomp(struct skd_device *skdev)
4286{
4287 int rc = 0;
4288 struct fit_completion_entry_v1 *skcomp;
4289 u32 nbytes;
4290
4291 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
4292 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
4293
2e44b427 4294 pr_debug("%s:%s:%d comp pci_alloc, total bytes %d entries %d\n",
4295 skdev->name, __func__, __LINE__,
4296 nbytes, SKD_N_COMPLETION_ENTRY);
e67f86b3
AB		 4297
4298 skcomp = pci_alloc_consistent(skdev->pdev, nbytes,
4299 &skdev->cq_dma_address);
4300
4301 if (skcomp == NULL) {
4302 rc = -ENOMEM;
4303 goto err_out;
4304 }
4305
4306 memset(skcomp, 0, nbytes);
4307
4308 skdev->skcomp_table = skcomp;
4309 skdev->skerr_table = (struct fit_comp_error_info *)((char *)skcomp +
4310 sizeof(*skcomp) *
4311 SKD_N_COMPLETION_ENTRY);
4312
4313err_out:
4314 return rc;
4315}
4316
4317static int skd_cons_skmsg(struct skd_device *skdev)
4318{
4319 int rc = 0;
4320 u32 i;
4321
2e44b427 4322 pr_debug("%s:%s:%d skmsg_table kzalloc, struct %lu, count %u total %lu\n",
4323 skdev->name, __func__, __LINE__,
4324 sizeof(struct skd_fitmsg_context),
4325 skdev->num_fitmsg_context,
4326 sizeof(struct skd_fitmsg_context) * skdev->num_fitmsg_context);
e67f86b3
AB		 4327
4328 skdev->skmsg_table = kzalloc(sizeof(struct skd_fitmsg_context)
4329 *skdev->num_fitmsg_context, GFP_KERNEL);
4330 if (skdev->skmsg_table == NULL) {
4331 rc = -ENOMEM;
4332 goto err_out;
4333 }
4334
4335 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4336 struct skd_fitmsg_context *skmsg;
4337
4338 skmsg = &skdev->skmsg_table[i];
4339
4340 skmsg->id = i + SKD_ID_FIT_MSG;
4341
4342 skmsg->state = SKD_MSG_STATE_IDLE;
4343 skmsg->msg_buf = pci_alloc_consistent(skdev->pdev,
4344 SKD_N_FITMSG_BYTES + 64,
4345 &skmsg->mb_dma_address);
4346
4347 if (skmsg->msg_buf == NULL) {
4348 rc = -ENOMEM;
4349 goto err_out;
4350 }
4351
4352 skmsg->offset = (u32)((u64)skmsg->msg_buf &
4353 (~FIT_QCMD_BASE_ADDRESS_MASK));
4354 skmsg->msg_buf += ~FIT_QCMD_BASE_ADDRESS_MASK;
4355 skmsg->msg_buf = (u8 *)((u64)skmsg->msg_buf &
4356 FIT_QCMD_BASE_ADDRESS_MASK);
4357 skmsg->mb_dma_address += ~FIT_QCMD_BASE_ADDRESS_MASK;
4358 skmsg->mb_dma_address &= FIT_QCMD_BASE_ADDRESS_MASK;
4359 memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
4360
4361 skmsg->next = &skmsg[1];
4362 }
4363
4364 /* Free list is in order starting with the 0th entry. */
4365 skdev->skmsg_table[i - 1].next = NULL;
4366 skdev->skmsg_free_list = skdev->skmsg_table;
4367
4368err_out:
4369 return rc;
4370}
4371
4372static int skd_cons_skreq(struct skd_device *skdev)
4373{
4374 int rc = 0;
4375 u32 i;
4376
2e44b427 4377 pr_debug("%s:%s:%d skreq_table kzalloc, struct %lu, count %u total %lu\n",
4378 skdev->name, __func__, __LINE__,
4379 sizeof(struct skd_request_context),
4380 skdev->num_req_context,
4381 sizeof(struct skd_request_context) * skdev->num_req_context);
e67f86b3
AB		 4382
4383 skdev->skreq_table = kzalloc(sizeof(struct skd_request_context)
4384 * skdev->num_req_context, GFP_KERNEL);
4385 if (skdev->skreq_table == NULL) {
4386 rc = -ENOMEM;
4387 goto err_out;
4388 }
4389
2e44b427 4390 pr_debug("%s:%s:%d alloc sg_table sg_per_req %u scatlist %lu total %lu\n",
4391 skdev->name, __func__, __LINE__,
4392 skdev->sgs_per_request, sizeof(struct scatterlist),
4393 skdev->sgs_per_request * sizeof(struct scatterlist));
e67f86b3
AB		 4394
4395 for (i = 0; i < skdev->num_req_context; i++) {
4396 struct skd_request_context *skreq;
4397
4398 skreq = &skdev->skreq_table[i];
4399
4400 skreq->id = i + SKD_ID_RW_REQUEST;
4401 skreq->state = SKD_REQ_STATE_IDLE;
4402
4403 skreq->sg = kzalloc(sizeof(struct scatterlist) *
4404 skdev->sgs_per_request, GFP_KERNEL);
4405 if (skreq->sg == NULL) {
4406 rc = -ENOMEM;
4407 goto err_out;
4408 }
4409 sg_init_table(skreq->sg, skdev->sgs_per_request);
4410
4411 skreq->sksg_list = skd_cons_sg_list(skdev,
4412 skdev->sgs_per_request,
4413 &skreq->sksg_dma_address);
4414
4415 if (skreq->sksg_list == NULL) {
4416 rc = -ENOMEM;
4417 goto err_out;
4418 }
4419
4420 skreq->next = &skreq[1];
4421 }
4422
4423 /* Free list is in order starting with the 0th entry. */
4424 skdev->skreq_table[i - 1].next = NULL;
4425 skdev->skreq_free_list = skdev->skreq_table;
4426
4427err_out:
4428 return rc;
4429}
4430
4431static int skd_cons_skspcl(struct skd_device *skdev)
4432{
4433 int rc = 0;
4434 u32 i, nbytes;
4435
2e44b427 4436 pr_debug("%s:%s:%d skspcl_table kzalloc, struct %lu, count %u total %lu\n",
4437 skdev->name, __func__, __LINE__,
4438 sizeof(struct skd_special_context),
4439 skdev->n_special,
4440 sizeof(struct skd_special_context) * skdev->n_special);
e67f86b3
AB		 4441
4442 skdev->skspcl_table = kzalloc(sizeof(struct skd_special_context)
4443 * skdev->n_special, GFP_KERNEL);
4444 if (skdev->skspcl_table == NULL) {
4445 rc = -ENOMEM;
4446 goto err_out;
4447 }
4448
4449 for (i = 0; i < skdev->n_special; i++) {
4450 struct skd_special_context *skspcl;
4451
4452 skspcl = &skdev->skspcl_table[i];
4453
4454 skspcl->req.id = i + SKD_ID_SPECIAL_REQUEST;
4455 skspcl->req.state = SKD_REQ_STATE_IDLE;
4456
4457 skspcl->req.next = &skspcl[1].req;
4458
4459 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4460
4461 skspcl->msg_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4462 &skspcl->mb_dma_address);
4463 if (skspcl->msg_buf == NULL) {
4464 rc = -ENOMEM;
4465 goto err_out;
4466 }
4467
4468 memset(skspcl->msg_buf, 0, nbytes);
4469
4470 skspcl->req.sg = kzalloc(sizeof(struct scatterlist) *
4471 SKD_N_SG_PER_SPECIAL, GFP_KERNEL);
4472 if (skspcl->req.sg == NULL) {
4473 rc = -ENOMEM;
4474 goto err_out;
4475 }
4476
4477 skspcl->req.sksg_list = skd_cons_sg_list(skdev,
4478 SKD_N_SG_PER_SPECIAL,
4479 &skspcl->req.
4480 sksg_dma_address);
4481 if (skspcl->req.sksg_list == NULL) {
4482 rc = -ENOMEM;
4483 goto err_out;
4484 }
4485 }
4486
4487 /* Free list is in order starting with the 0th entry. */
4488 skdev->skspcl_table[i - 1].req.next = NULL;
4489 skdev->skspcl_free_list = skdev->skspcl_table;
4490
4491 return rc;
4492
4493err_out:
4494 return rc;
4495}
4496
4497static int skd_cons_sksb(struct skd_device *skdev)
4498{
4499 int rc = 0;
4500 struct skd_special_context *skspcl;
4501 u32 nbytes;
4502
4503 skspcl = &skdev->internal_skspcl;
4504
4505 skspcl->req.id = 0 + SKD_ID_INTERNAL;
4506 skspcl->req.state = SKD_REQ_STATE_IDLE;
4507
4508 nbytes = SKD_N_INTERNAL_BYTES;
4509
4510 skspcl->data_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4511 &skspcl->db_dma_address);
4512 if (skspcl->data_buf == NULL) {
4513 rc = -ENOMEM;
4514 goto err_out;
4515 }
4516
4517 memset(skspcl->data_buf, 0, nbytes);
4518
4519 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4520 skspcl->msg_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4521 &skspcl->mb_dma_address);
4522 if (skspcl->msg_buf == NULL) {
4523 rc = -ENOMEM;
4524 goto err_out;
4525 }
4526
4527 memset(skspcl->msg_buf, 0, nbytes);
4528
4529 skspcl->req.sksg_list = skd_cons_sg_list(skdev, 1,
4530 &skspcl->req.sksg_dma_address);
4531 if (skspcl->req.sksg_list == NULL) {
4532 rc = -ENOMEM;
4533 goto err_out;
4534 }
4535
4536 if (!skd_format_internal_skspcl(skdev)) {
4537 rc = -EINVAL;
4538 goto err_out;
4539 }
4540
4541err_out:
4542 return rc;
4543}
4544
4545static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4546 u32 n_sg,
4547 dma_addr_t *ret_dma_addr)
4548{
4549 struct fit_sg_descriptor *sg_list;
4550 u32 nbytes;
4551
4552 nbytes = sizeof(*sg_list) * n_sg;
4553
4554 sg_list = pci_alloc_consistent(skdev->pdev, nbytes, ret_dma_addr);
4555
4556 if (sg_list != NULL) {
4557 uint64_t dma_address = *ret_dma_addr;
4558 u32 i;
4559
4560 memset(sg_list, 0, nbytes);
4561
4562 for (i = 0; i < n_sg - 1; i++) {
4563 uint64_t ndp_off;
4564 ndp_off = (i + 1) * sizeof(struct fit_sg_descriptor);
4565
4566 sg_list[i].next_desc_ptr = dma_address + ndp_off;
4567 }
4568 sg_list[i].next_desc_ptr = 0LL;
4569 }
4570
4571 return sg_list;
4572}
4573
4574static int skd_cons_disk(struct skd_device *skdev)
4575{
4576 int rc = 0;
4577 struct gendisk *disk;
4578 struct request_queue *q;
4579 unsigned long flags;
4580
4581 disk = alloc_disk(SKD_MINORS_PER_DEVICE);
4582 if (!disk) {
4583 rc = -ENOMEM;
4584 goto err_out;
4585 }
4586
4587 skdev->disk = disk;
4588 sprintf(disk->disk_name, DRV_NAME "%u", skdev->devno);
4589
4590 disk->major = skdev->major;
4591 disk->first_minor = skdev->devno * SKD_MINORS_PER_DEVICE;
4592 disk->fops = &skd_blockdev_ops;
4593 disk->private_data = skdev;
4594
fcd37eb3 4595 q = blk_init_queue(skd_request_fn, &skdev->lock);
e67f86b3
AB		 4596 if (!q) {
4597 rc = -ENOMEM;
4598 goto err_out;
4599 }
4600
4601 skdev->queue = q;
4602 disk->queue = q;
4603 q->queuedata = skdev;
4604
e67f86b3
AB		 4605 blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
4606 blk_queue_max_segments(q, skdev->sgs_per_request);
4607 blk_queue_max_hw_sectors(q, SKD_N_MAX_SECTORS);
4608
4609 /* set sysfs ptimal_io_size to 8K */
4610 blk_queue_io_opt(q, 8192);
4611
4612 /* DISCARD Flag initialization. */
4613 q->limits.discard_granularity = 8192;
4614 q->limits.discard_alignment = 0;
4615 q->limits.max_discard_sectors = UINT_MAX >> 9;
4616 q->limits.discard_zeroes_data = 1;
4617 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
4618 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
4619
4620 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 4621 pr_debug("%s:%s:%d stopping %s queue\n",
4622 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 4623 skd_stop_queue(skdev);
4624 spin_unlock_irqrestore(&skdev->lock, flags);
4625
4626err_out:
4627 return rc;
4628}
4629
4630/*
4631 *****************************************************************************
4632 * DESTRUCT (FREE)
4633 *****************************************************************************
4634 */
4635
4636static void skd_free_skcomp(struct skd_device *skdev);
4637static void skd_free_skmsg(struct skd_device *skdev);
4638static void skd_free_skreq(struct skd_device *skdev);
4639static void skd_free_skspcl(struct skd_device *skdev);
4640static void skd_free_sksb(struct skd_device *skdev);
4641static void skd_free_sg_list(struct skd_device *skdev,
4642 struct fit_sg_descriptor *sg_list,
4643 u32 n_sg, dma_addr_t dma_addr);
4644static void skd_free_disk(struct skd_device *skdev);
4645
4646static void skd_destruct(struct skd_device *skdev)
4647{
4648 if (skdev == NULL)
4649 return;
4650
4651
2e44b427 4652 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4653 skd_free_disk(skdev);
4654
2e44b427 4655 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4656 skd_free_sksb(skdev);
4657
2e44b427 4658 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4659 skd_free_skspcl(skdev);
4660
2e44b427 4661 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4662 skd_free_skreq(skdev);
4663
2e44b427 4664 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4665 skd_free_skmsg(skdev);
4666
2e44b427 4667 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4668 skd_free_skcomp(skdev);
4669
2e44b427 4670 pr_debug("%s:%s:%d skdev\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4671 kfree(skdev);
4672}
4673
4674static void skd_free_skcomp(struct skd_device *skdev)
4675{
4676 if (skdev->skcomp_table != NULL) {
4677 u32 nbytes;
4678
4679 nbytes = sizeof(skdev->skcomp_table[0]) *
4680 SKD_N_COMPLETION_ENTRY;
4681 pci_free_consistent(skdev->pdev, nbytes,
4682 skdev->skcomp_table, skdev->cq_dma_address);
4683 }
4684
4685 skdev->skcomp_table = NULL;
4686 skdev->cq_dma_address = 0;
4687}
4688
4689static void skd_free_skmsg(struct skd_device *skdev)
4690{
4691 u32 i;
4692
4693 if (skdev->skmsg_table == NULL)
4694 return;
4695
4696 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4697 struct skd_fitmsg_context *skmsg;
4698
4699 skmsg = &skdev->skmsg_table[i];
4700
4701 if (skmsg->msg_buf != NULL) {
4702 skmsg->msg_buf += skmsg->offset;
4703 skmsg->mb_dma_address += skmsg->offset;
4704 pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES,
4705 skmsg->msg_buf,
4706 skmsg->mb_dma_address);
4707 }
4708 skmsg->msg_buf = NULL;
4709 skmsg->mb_dma_address = 0;
4710 }
4711
4712 kfree(skdev->skmsg_table);
4713 skdev->skmsg_table = NULL;
4714}
4715
4716static void skd_free_skreq(struct skd_device *skdev)
4717{
4718 u32 i;
4719
4720 if (skdev->skreq_table == NULL)
4721 return;
4722
4723 for (i = 0; i < skdev->num_req_context; i++) {
4724 struct skd_request_context *skreq;
4725
4726 skreq = &skdev->skreq_table[i];
4727
4728 skd_free_sg_list(skdev, skreq->sksg_list,
4729 skdev->sgs_per_request,
4730 skreq->sksg_dma_address);
4731
4732 skreq->sksg_list = NULL;
4733 skreq->sksg_dma_address = 0;
4734
4735 kfree(skreq->sg);
4736 }
4737
4738 kfree(skdev->skreq_table);
4739 skdev->skreq_table = NULL;
4740}
4741
4742static void skd_free_skspcl(struct skd_device *skdev)
4743{
4744 u32 i;
4745 u32 nbytes;
4746
4747 if (skdev->skspcl_table == NULL)
4748 return;
4749
4750 for (i = 0; i < skdev->n_special; i++) {
4751 struct skd_special_context *skspcl;
4752
4753 skspcl = &skdev->skspcl_table[i];
4754
4755 if (skspcl->msg_buf != NULL) {
4756 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4757 pci_free_consistent(skdev->pdev, nbytes,
4758 skspcl->msg_buf,
4759 skspcl->mb_dma_address);
4760 }
4761
4762 skspcl->msg_buf = NULL;
4763 skspcl->mb_dma_address = 0;
4764
4765 skd_free_sg_list(skdev, skspcl->req.sksg_list,
4766 SKD_N_SG_PER_SPECIAL,
4767 skspcl->req.sksg_dma_address);
4768
4769 skspcl->req.sksg_list = NULL;
4770 skspcl->req.sksg_dma_address = 0;
4771
4772 kfree(skspcl->req.sg);
4773 }
4774
4775 kfree(skdev->skspcl_table);
4776 skdev->skspcl_table = NULL;
4777}
4778
4779static void skd_free_sksb(struct skd_device *skdev)
4780{
4781 struct skd_special_context *skspcl;
4782 u32 nbytes;
4783
4784 skspcl = &skdev->internal_skspcl;
4785
4786 if (skspcl->data_buf != NULL) {
4787 nbytes = SKD_N_INTERNAL_BYTES;
4788
4789 pci_free_consistent(skdev->pdev, nbytes,
4790 skspcl->data_buf, skspcl->db_dma_address);
4791 }
4792
4793 skspcl->data_buf = NULL;
4794 skspcl->db_dma_address = 0;
4795
4796 if (skspcl->msg_buf != NULL) {
4797 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4798 pci_free_consistent(skdev->pdev, nbytes,
4799 skspcl->msg_buf, skspcl->mb_dma_address);
4800 }
4801
4802 skspcl->msg_buf = NULL;
4803 skspcl->mb_dma_address = 0;
4804
4805 skd_free_sg_list(skdev, skspcl->req.sksg_list, 1,
4806 skspcl->req.sksg_dma_address);
4807
4808 skspcl->req.sksg_list = NULL;
4809 skspcl->req.sksg_dma_address = 0;
4810}
4811
4812static void skd_free_sg_list(struct skd_device *skdev,
4813 struct fit_sg_descriptor *sg_list,
4814 u32 n_sg, dma_addr_t dma_addr)
4815{
4816 if (sg_list != NULL) {
4817 u32 nbytes;
4818
4819 nbytes = sizeof(*sg_list) * n_sg;
4820
4821 pci_free_consistent(skdev->pdev, nbytes, sg_list, dma_addr);
4822 }
4823}
4824
4825static void skd_free_disk(struct skd_device *skdev)
4826{
4827 struct gendisk *disk = skdev->disk;
4828
4829 if (disk != NULL) {
4830 struct request_queue *q = disk->queue;
4831
4832 if (disk->flags & GENHD_FL_UP)
4833 del_gendisk(disk);
4834 if (q)
4835 blk_cleanup_queue(q);
4836 put_disk(disk);
4837 }
4838 skdev->disk = NULL;
4839}
4840
4841
4842
4843/*
4844 *****************************************************************************
4845 * BLOCK DEVICE (BDEV) GLUE
4846 *****************************************************************************
4847 */
4848
4849static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
4850{
4851 struct skd_device *skdev;
4852 u64 capacity;
4853
4854 skdev = bdev->bd_disk->private_data;
4855
2e44b427 4856 pr_debug("%s:%s:%d %s: CMD[%s] getgeo device\n",
4857 skdev->name, __func__, __LINE__,
4858 bdev->bd_disk->disk_name, current->comm);
e67f86b3
AB		 4859
4860 if (skdev->read_cap_is_valid) {
4861 capacity = get_capacity(skdev->disk);
4862 geo->heads = 64;
4863 geo->sectors = 255;
4864 geo->cylinders = (capacity) / (255 * 64);
4865
4866 return 0;
4867 }
4868 return -EIO;
4869}
4870
4871static int skd_bdev_attach(struct skd_device *skdev)
4872{
2e44b427 4873 pr_debug("%s:%s:%d add_disk\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4874 add_disk(skdev->disk);
4875 return 0;
4876}
4877
4878static const struct block_device_operations skd_blockdev_ops = {
4879 .owner = THIS_MODULE,
4880 .ioctl = skd_bdev_ioctl,
4881 .getgeo = skd_bdev_getgeo,
4882};
4883
4884
4885/*
4886 *****************************************************************************
4887 * PCIe DRIVER GLUE
4888 *****************************************************************************
4889 */
4890
4891static DEFINE_PCI_DEVICE_TABLE(skd_pci_tbl) = {
4892 { PCI_VENDOR_ID_STEC, PCI_DEVICE_ID_S1120,
4893 PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
4894 { 0 } /* terminate list */
4895};
4896
4897MODULE_DEVICE_TABLE(pci, skd_pci_tbl);
4898
4899static char *skd_pci_info(struct skd_device *skdev, char *str)
4900{
4901 int pcie_reg;
4902
4903 strcpy(str, "PCIe (");
4904 pcie_reg = pci_find_capability(skdev->pdev, PCI_CAP_ID_EXP);
4905
4906 if (pcie_reg) {
4907
4908 char lwstr[6];
4909 uint16_t pcie_lstat, lspeed, lwidth;
4910
4911 pcie_reg += 0x12;
4912 pci_read_config_word(skdev->pdev, pcie_reg, &pcie_lstat);
4913 lspeed = pcie_lstat & (0xF);
4914 lwidth = (pcie_lstat & 0x3F0) >> 4;
4915
4916 if (lspeed == 1)
4917 strcat(str, "2.5GT/s ");
4918 else if (lspeed == 2)
4919 strcat(str, "5.0GT/s ");
4920 else
4921 strcat(str, "<unknown> ");
4922 snprintf(lwstr, sizeof(lwstr), "%dX)", lwidth);
4923 strcat(str, lwstr);
4924 }
4925 return str;
4926}
4927
4928static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4929{
4930 int i;
4931 int rc = 0;
4932 char pci_str[32];
4933 struct skd_device *skdev;
4934
4935 pr_info("STEC s1120 Driver(%s) version %s-b%s\n",
4936 DRV_NAME, DRV_VERSION, DRV_BUILD_ID);
4937 pr_info("(skd?:??:[%s]): vendor=%04X device=%04x\n",
4938 pci_name(pdev), pdev->vendor, pdev->device);
4939
4940 rc = pci_enable_device(pdev);
4941 if (rc)
4942 return rc;
4943 rc = pci_request_regions(pdev, DRV_NAME);
4944 if (rc)
4945 goto err_out;
4946 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
4947 if (!rc) {
4948 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
4949
4950 pr_err("(%s): consistent DMA mask error %d\n",
4951 pci_name(pdev), rc);
4952 }
4953 } else {
4954 (rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)));
4955 if (rc) {
4956
4957 pr_err("(%s): DMA mask error %d\n",
4958 pci_name(pdev), rc);
4959 goto err_out_regions;
4960 }
4961 }
4962
4963 skdev = skd_construct(pdev);
1762b57f
WY		 4964 if (skdev == NULL) {
4965 rc = -ENOMEM;
e67f86b3 4966 goto err_out_regions;
1762b57f 4967 }
e67f86b3
AB		 4968
4969 skd_pci_info(skdev, pci_str);
4970 pr_info("(%s): %s 64bit\n", skd_name(skdev), pci_str);
4971
4972 pci_set_master(pdev);
4973 rc = pci_enable_pcie_error_reporting(pdev);
4974 if (rc) {
4975 pr_err(
4976 "(%s): bad enable of PCIe error reporting rc=%d\n",
4977 skd_name(skdev), rc);
4978 skdev->pcie_error_reporting_is_enabled = 0;
4979 } else
4980 skdev->pcie_error_reporting_is_enabled = 1;
4981
4982
4983 pci_set_drvdata(pdev, skdev);
4984 skdev->pdev = pdev;
4985 skdev->disk->driverfs_dev = &pdev->dev;
4986
4987 for (i = 0; i < SKD_MAX_BARS; i++) {
4988 skdev->mem_phys[i] = pci_resource_start(pdev, i);
4989 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
4990 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
4991 skdev->mem_size[i]);
4992 if (!skdev->mem_map[i]) {
4993 pr_err("(%s): Unable to map adapter memory!\n",
4994 skd_name(skdev));
4995 rc = -ENODEV;
4996 goto err_out_iounmap;
4997 }
2e44b427 4998 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
4999 skdev->name, __func__, __LINE__,
5000 skdev->mem_map[i],
5001 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
e67f86b3
AB		 5002 }
5003
5004 rc = skd_acquire_irq(skdev);
5005 if (rc) {
5006 pr_err("(%s): interrupt resource error %d\n",
5007 skd_name(skdev), rc);
5008 goto err_out_iounmap;
5009 }
5010
5011 rc = skd_start_timer(skdev);
5012 if (rc)
5013 goto err_out_timer;
5014
5015 init_waitqueue_head(&skdev->waitq);
5016
5017 skd_start_device(skdev);
5018
5019 rc = wait_event_interruptible_timeout(skdev->waitq,
5020 (skdev->gendisk_on),
5021 (SKD_START_WAIT_SECONDS * HZ));
5022 if (skdev->gendisk_on > 0) {
5023 /* device came on-line after reset */
5024 skd_bdev_attach(skdev);
5025 rc = 0;
5026 } else {
5027 /* we timed out, something is wrong with the device,
5028 don't add the disk structure */
5029 pr_err(
5030 "(%s): error: waiting for s1120 timed out %d!\n",
5031 skd_name(skdev), rc);
5032 /* in case of no error; we timeout with ENXIO */
5033 if (!rc)
5034 rc = -ENXIO;
5035 goto err_out_timer;
5036 }
5037
5038
5039#ifdef SKD_VMK_POLL_HANDLER
5040 if (skdev->irq_type == SKD_IRQ_MSIX) {
5041 /* MSIX completion handler is being used for coredump */
5042 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
5043 skdev->msix_entries[5].vector,
5044 skd_comp_q, skdev);
5045 } else {
5046 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
5047 skdev->pdev->irq, skd_isr,
5048 skdev);
5049 }
5050#endif /* SKD_VMK_POLL_HANDLER */
5051
5052 return rc;
5053
5054err_out_timer:
5055 skd_stop_device(skdev);
5056 skd_release_irq(skdev);
5057
5058err_out_iounmap:
5059 for (i = 0; i < SKD_MAX_BARS; i++)
5060 if (skdev->mem_map[i])
5061 iounmap(skdev->mem_map[i]);
5062
5063 if (skdev->pcie_error_reporting_is_enabled)
5064 pci_disable_pcie_error_reporting(pdev);
5065
5066 skd_destruct(skdev);
5067
5068err_out_regions:
5069 pci_release_regions(pdev);
5070
5071err_out:
5072 pci_disable_device(pdev);
5073 pci_set_drvdata(pdev, NULL);
5074 return rc;
5075}
5076
5077static void skd_pci_remove(struct pci_dev *pdev)
5078{
5079 int i;
5080 struct skd_device *skdev;
5081
5082 skdev = pci_get_drvdata(pdev);
5083 if (!skdev) {
5084 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5085 return;
5086 }
5087 skd_stop_device(skdev);
5088 skd_release_irq(skdev);
5089
5090 for (i = 0; i < SKD_MAX_BARS; i++)
5091 if (skdev->mem_map[i])
5092 iounmap((u32 *)skdev->mem_map[i]);
5093
5094 if (skdev->pcie_error_reporting_is_enabled)
5095 pci_disable_pcie_error_reporting(pdev);
5096
5097 skd_destruct(skdev);
5098
5099 pci_release_regions(pdev);
5100 pci_disable_device(pdev);
5101 pci_set_drvdata(pdev, NULL);
5102
5103 return;
5104}
5105
5106static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
5107{
5108 int i;
5109 struct skd_device *skdev;
5110
5111 skdev = pci_get_drvdata(pdev);
5112 if (!skdev) {
5113 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5114 return -EIO;
5115 }
5116
5117 skd_stop_device(skdev);
5118
5119 skd_release_irq(skdev);
5120
5121 for (i = 0; i < SKD_MAX_BARS; i++)
5122 if (skdev->mem_map[i])
5123 iounmap((u32 *)skdev->mem_map[i]);
5124
5125 if (skdev->pcie_error_reporting_is_enabled)
5126 pci_disable_pcie_error_reporting(pdev);
5127
5128 pci_release_regions(pdev);
5129 pci_save_state(pdev);
5130 pci_disable_device(pdev);
5131 pci_set_power_state(pdev, pci_choose_state(pdev, state));
5132 return 0;
5133}
5134
5135static int skd_pci_resume(struct pci_dev *pdev)
5136{
5137 int i;
5138 int rc = 0;
5139 struct skd_device *skdev;
5140
5141 skdev = pci_get_drvdata(pdev);
5142 if (!skdev) {
5143 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5144 return -1;
5145 }
5146
5147 pci_set_power_state(pdev, PCI_D0);
5148 pci_enable_wake(pdev, PCI_D0, 0);
5149 pci_restore_state(pdev);
5150
5151 rc = pci_enable_device(pdev);
5152 if (rc)
5153 return rc;
5154 rc = pci_request_regions(pdev, DRV_NAME);
5155 if (rc)
5156 goto err_out;
5157 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
5158 if (!rc) {
5159 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
5160
5161 pr_err("(%s): consistent DMA mask error %d\n",
5162 pci_name(pdev), rc);
5163 }
5164 } else {
5165 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5166 if (rc) {
5167
5168 pr_err("(%s): DMA mask error %d\n",
5169 pci_name(pdev), rc);
5170 goto err_out_regions;
5171 }
5172 }
5173
5174 pci_set_master(pdev);
5175 rc = pci_enable_pcie_error_reporting(pdev);
5176 if (rc) {
5177 pr_err("(%s): bad enable of PCIe error reporting rc=%d\n",
5178 skdev->name, rc);
5179 skdev->pcie_error_reporting_is_enabled = 0;
5180 } else
5181 skdev->pcie_error_reporting_is_enabled = 1;
5182
5183 for (i = 0; i < SKD_MAX_BARS; i++) {
5184
5185 skdev->mem_phys[i] = pci_resource_start(pdev, i);
5186 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
5187 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
5188 skdev->mem_size[i]);
5189 if (!skdev->mem_map[i]) {
5190 pr_err("(%s): Unable to map adapter memory!\n",
5191 skd_name(skdev));
5192 rc = -ENODEV;
5193 goto err_out_iounmap;
5194 }
2e44b427 5195 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
5196 skdev->name, __func__, __LINE__,
5197 skdev->mem_map[i],
5198 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
e67f86b3
AB		 5199 }
5200 rc = skd_acquire_irq(skdev);
5201 if (rc) {
5202
5203 pr_err("(%s): interrupt resource error %d\n",
5204 pci_name(pdev), rc);
5205 goto err_out_iounmap;
5206 }
5207
5208 rc = skd_start_timer(skdev);
5209 if (rc)
5210 goto err_out_timer;
5211
5212 init_waitqueue_head(&skdev->waitq);
5213
5214 skd_start_device(skdev);
5215
5216 return rc;
5217
5218err_out_timer:
5219 skd_stop_device(skdev);
5220 skd_release_irq(skdev);
5221
5222err_out_iounmap:
5223 for (i = 0; i < SKD_MAX_BARS; i++)
5224 if (skdev->mem_map[i])
5225 iounmap(skdev->mem_map[i]);
5226
5227 if (skdev->pcie_error_reporting_is_enabled)
5228 pci_disable_pcie_error_reporting(pdev);
5229
5230err_out_regions:
5231 pci_release_regions(pdev);
5232
5233err_out:
5234 pci_disable_device(pdev);
5235 return rc;
5236}
5237
5238static void skd_pci_shutdown(struct pci_dev *pdev)
5239{
5240 struct skd_device *skdev;
5241
5242 pr_err("skd_pci_shutdown called\n");
5243
5244 skdev = pci_get_drvdata(pdev);
5245 if (!skdev) {
5246 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5247 return;
5248 }
5249
5250 pr_err("%s: calling stop\n", skd_name(skdev));
5251 skd_stop_device(skdev);
5252}
5253
5254static struct pci_driver skd_driver = {
5255 .name = DRV_NAME,
5256 .id_table = skd_pci_tbl,
5257 .probe = skd_pci_probe,
5258 .remove = skd_pci_remove,
5259 .suspend = skd_pci_suspend,
5260 .resume = skd_pci_resume,
5261 .shutdown = skd_pci_shutdown,
5262};
5263
5264/*
5265 *****************************************************************************
5266 * LOGGING SUPPORT
5267 *****************************************************************************
5268 */
5269
5270static const char *skd_name(struct skd_device *skdev)
5271{
5272 memset(skdev->id_str, 0, sizeof(skdev->id_str));
5273
5274 if (skdev->inquiry_is_valid)
5275 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:%s:[%s]",
5276 skdev->name, skdev->inq_serial_num,
5277 pci_name(skdev->pdev));
5278 else
5279 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:??:[%s]",
5280 skdev->name, pci_name(skdev->pdev));
5281
5282 return skdev->id_str;
5283}
5284
5285const char *skd_drive_state_to_str(int state)
5286{
5287 switch (state) {
5288 case FIT_SR_DRIVE_OFFLINE:
5289 return "OFFLINE";
5290 case FIT_SR_DRIVE_INIT:
5291 return "INIT";
5292 case FIT_SR_DRIVE_ONLINE:
5293 return "ONLINE";
5294 case FIT_SR_DRIVE_BUSY:
5295 return "BUSY";
5296 case FIT_SR_DRIVE_FAULT:
5297 return "FAULT";
5298 case FIT_SR_DRIVE_DEGRADED:
5299 return "DEGRADED";
5300 case FIT_SR_PCIE_LINK_DOWN:
5301 return "INK_DOWN";
5302 case FIT_SR_DRIVE_SOFT_RESET:
5303 return "SOFT_RESET";
5304 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
5305 return "NEED_FW";
5306 case FIT_SR_DRIVE_INIT_FAULT:
5307 return "INIT_FAULT";
5308 case FIT_SR_DRIVE_BUSY_SANITIZE:
5309 return "BUSY_SANITIZE";
5310 case FIT_SR_DRIVE_BUSY_ERASE:
5311 return "BUSY_ERASE";
5312 case FIT_SR_DRIVE_FW_BOOTING:
5313 return "FW_BOOTING";
5314 default:
5315 return "???";
5316 }
5317}
5318
5319const char *skd_skdev_state_to_str(enum skd_drvr_state state)
5320{
5321 switch (state) {
5322 case SKD_DRVR_STATE_LOAD:
5323 return "LOAD";
5324 case SKD_DRVR_STATE_IDLE:
5325 return "IDLE";
5326 case SKD_DRVR_STATE_BUSY:
5327 return "BUSY";
5328 case SKD_DRVR_STATE_STARTING:
5329 return "STARTING";
5330 case SKD_DRVR_STATE_ONLINE:
5331 return "ONLINE";
5332 case SKD_DRVR_STATE_PAUSING:
5333 return "PAUSING";
5334 case SKD_DRVR_STATE_PAUSED:
5335 return "PAUSED";
5336 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
5337 return "DRAINING_TIMEOUT";
5338 case SKD_DRVR_STATE_RESTARTING:
5339 return "RESTARTING";
5340 case SKD_DRVR_STATE_RESUMING:
5341 return "RESUMING";
5342 case SKD_DRVR_STATE_STOPPING:
5343 return "STOPPING";
5344 case SKD_DRVR_STATE_SYNCING:
5345 return "SYNCING";
5346 case SKD_DRVR_STATE_FAULT:
5347 return "FAULT";
5348 case SKD_DRVR_STATE_DISAPPEARED:
5349 return "DISAPPEARED";
5350 case SKD_DRVR_STATE_BUSY_ERASE:
5351 return "BUSY_ERASE";
5352 case SKD_DRVR_STATE_BUSY_SANITIZE:
5353 return "BUSY_SANITIZE";
5354 case SKD_DRVR_STATE_BUSY_IMMINENT:
5355 return "BUSY_IMMINENT";
5356 case SKD_DRVR_STATE_WAIT_BOOT:
5357 return "WAIT_BOOT";
5358
5359 default:
5360 return "???";
5361 }
5362}
5363
5364const char *skd_skmsg_state_to_str(enum skd_fit_msg_state state)
5365{
5366 switch (state) {
5367 case SKD_MSG_STATE_IDLE:
5368 return "IDLE";
5369 case SKD_MSG_STATE_BUSY:
5370 return "BUSY";
5371 default:
5372 return "???";
5373 }
5374}
5375
5376const char *skd_skreq_state_to_str(enum skd_req_state state)
5377{
5378 switch (state) {
5379 case SKD_REQ_STATE_IDLE:
5380 return "IDLE";
5381 case SKD_REQ_STATE_SETUP:
5382 return "SETUP";
5383 case SKD_REQ_STATE_BUSY:
5384 return "BUSY";
5385 case SKD_REQ_STATE_COMPLETED:
5386 return "COMPLETED";
5387 case SKD_REQ_STATE_TIMEOUT:
5388 return "TIMEOUT";
5389 case SKD_REQ_STATE_ABORTED:
5390 return "ABORTED";
5391 default:
5392 return "???";
5393 }
5394}
5395
5396static void skd_log_skdev(struct skd_device *skdev, const char *event)
5397{
2e44b427 5398 pr_debug("%s:%s:%d (%s) skdev=%p event='%s'\n",
5399 skdev->name, __func__, __LINE__, skdev->name, skdev, event);
5400 pr_debug("%s:%s:%d drive_state=%s(%d) driver_state=%s(%d)\n",
5401 skdev->name, __func__, __LINE__,
5402 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
5403 skd_skdev_state_to_str(skdev->state), skdev->state);
5404 pr_debug("%s:%s:%d busy=%d limit=%d dev=%d lowat=%d\n",
5405 skdev->name, __func__, __LINE__,
5406 skdev->in_flight, skdev->cur_max_queue_depth,
5407 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
5408 pr_debug("%s:%s:%d timestamp=0x%x cycle=%d cycle_ix=%d\n",
5409 skdev->name, __func__, __LINE__,
5410 skdev->timeout_stamp, skdev->skcomp_cycle, skdev->skcomp_ix);
e67f86b3
AB		 5411}
5412
5413static void skd_log_skmsg(struct skd_device *skdev,
5414 struct skd_fitmsg_context *skmsg, const char *event)
5415{
2e44b427 5416 pr_debug("%s:%s:%d (%s) skmsg=%p event='%s'\n",
5417 skdev->name, __func__, __LINE__, skdev->name, skmsg, event);
5418 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x length=%d\n",
5419 skdev->name, __func__, __LINE__,
5420 skd_skmsg_state_to_str(skmsg->state), skmsg->state,
5421 skmsg->id, skmsg->length);
e67f86b3
AB		 5422}
5423
5424static void skd_log_skreq(struct skd_device *skdev,
5425 struct skd_request_context *skreq, const char *event)
5426{
2e44b427 5427 pr_debug("%s:%s:%d (%s) skreq=%p event='%s'\n",
5428 skdev->name, __func__, __LINE__, skdev->name, skreq, event);
5429 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
5430 skdev->name, __func__, __LINE__,
5431 skd_skreq_state_to_str(skreq->state), skreq->state,
5432 skreq->id, skreq->fitmsg_id);
5433 pr_debug("%s:%s:%d timo=0x%x sg_dir=%d n_sg=%d\n",
5434 skdev->name, __func__, __LINE__,
5435 skreq->timeout_stamp, skreq->sg_data_dir, skreq->n_sg);
e67f86b3 5436
fcd37eb3
JA		 5437 if (skreq->req != NULL) {
5438 struct request *req = skreq->req;
5439 u32 lba = (u32)blk_rq_pos(req);
5440 u32 count = blk_rq_sectors(req);
e67f86b3 5441
fcd37eb3
JA		 5442 pr_debug("%s:%s:%d "
5443 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
5444 skdev->name, __func__, __LINE__,
5445 req, lba, lba, count, count,
5446 (int)rq_data_dir(req));
5447 } else
5448 pr_debug("%s:%s:%d req=NULL\n",
5449 skdev->name, __func__, __LINE__);
e67f86b3
AB		 5450}
5451
5452/*
5453 *****************************************************************************
5454 * MODULE GLUE
5455 *****************************************************************************
5456 */
5457
5458static int __init skd_init(void)
5459{
5460 int rc = 0;
5461
5462 pr_info(PFX " v%s-b%s loaded\n", DRV_VERSION, DRV_BUILD_ID);
5463
5464 switch (skd_isr_type) {
5465 case SKD_IRQ_LEGACY:
5466 case SKD_IRQ_MSI:
5467 case SKD_IRQ_MSIX:
5468 break;
5469 default:
5470 pr_info("skd_isr_type %d invalid, re-set to %d\n",
5471 skd_isr_type, SKD_IRQ_DEFAULT);
5472 skd_isr_type = SKD_IRQ_DEFAULT;
5473 }
5474
5475 skd_flush_slab = kmem_cache_create(SKD_FLUSH_JOB,
5476 sizeof(struct skd_flush_cmd),
5477 0, 0, NULL);
5478
5479 if (!skd_flush_slab) {
5480 pr_err("failed to allocated flush slab.\n");
5481 return -ENOMEM;
5482 }
5483
5484 if (skd_max_queue_depth < 1
5485 || skd_max_queue_depth > SKD_MAX_QUEUE_DEPTH) {
5486 pr_info(
5487 "skd_max_queue_depth %d invalid, re-set to %d\n",
5488 skd_max_queue_depth, SKD_MAX_QUEUE_DEPTH_DEFAULT);
5489 skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
5490 }
5491
5492 if (skd_max_req_per_msg < 1 || skd_max_req_per_msg > 14) {
5493 pr_info(
5494 "skd_max_req_per_msg %d invalid, re-set to %d\n",
5495 skd_max_req_per_msg, SKD_MAX_REQ_PER_MSG_DEFAULT);
5496 skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
5497 }
5498
5499 if (skd_sgs_per_request < 1 || skd_sgs_per_request > 4096) {
5500 pr_info(
5501 "skd_sg_per_request %d invalid, re-set to %d\n",
5502 skd_sgs_per_request, SKD_N_SG_PER_REQ_DEFAULT);
5503 skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
5504 }
5505
5506 if (skd_dbg_level < 0 || skd_dbg_level > 2) {
5507 pr_info("skd_dbg_level %d invalid, re-set to %d\n",
5508 skd_dbg_level, 0);
5509 skd_dbg_level = 0;
5510 }
5511
5512 if (skd_isr_comp_limit < 0) {
5513 pr_info("skd_isr_comp_limit %d invalid, set to %d\n",
5514 skd_isr_comp_limit, 0);
5515 skd_isr_comp_limit = 0;
5516 }
5517
5518 if (skd_max_pass_thru < 1 || skd_max_pass_thru > 50) {
5519 pr_info("skd_max_pass_thru %d invalid, re-set to %d\n",
5520 skd_max_pass_thru, SKD_N_SPECIAL_CONTEXT);
5521 skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
5522 }
5523
5524 /* Obtain major device number. */
5525 rc = register_blkdev(0, DRV_NAME);
5526 if (rc < 0)
5527 return rc;
5528
5529 skd_major = rc;
5530
5531 return pci_register_driver(&skd_driver);
5532
5533}
5534
5535static void __exit skd_exit(void)
5536{
5537 pr_info(PFX " v%s-b%s unloading\n", DRV_VERSION, DRV_BUILD_ID);
5538
5539 unregister_blkdev(skd_major, DRV_NAME);
5540 pci_unregister_driver(&skd_driver);
5541
5542 kmem_cache_destroy(skd_flush_slab);
5543}
5544
e67f86b3
AB		 5545module_init(skd_init);
5546module_exit(skd_exit);
Ubuntu Artful kernel mirror