]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/block/skd_main.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
block: set REQ_SYNC if we clear REQ_FUA|REQ_PREFLUSH
[mirror_ubuntu-artful-kernel.git] / drivers / block / skd_main.c
CommitLineData
e67f86b3
AB		 1/* Copyright 2012 STEC, Inc.
2 *
3 * This file is licensed under the terms of the 3-clause
4 * BSD License (http://opensource.org/licenses/BSD-3-Clause)
5 * or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
6 * at your option. Both licenses are also available in the LICENSE file
7 * distributed with this project. This file may not be copied, modified,
8 * or distributed except in accordance with those terms.
9 * Gordoni Waidhofer <gwaidhofer@stec-inc.com>
10 * Initial Driver Design!
11 * Thomas Swann <tswann@stec-inc.com>
12 * Interrupt handling.
13 * Ramprasad Chinthekindi <rchinthekindi@stec-inc.com>
14 * biomode implementation.
15 * Akhil Bhansali <abhansali@stec-inc.com>
16 * Added support for DISCARD / FLUSH and FUA.
17 */
18
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/init.h>
22#include <linux/pci.h>
23#include <linux/slab.h>
24#include <linux/spinlock.h>
25#include <linux/blkdev.h>
26#include <linux/sched.h>
27#include <linux/interrupt.h>
28#include <linux/compiler.h>
29#include <linux/workqueue.h>
30#include <linux/bitops.h>
31#include <linux/delay.h>
32#include <linux/time.h>
33#include <linux/hdreg.h>
34#include <linux/dma-mapping.h>
35#include <linux/completion.h>
36#include <linux/scatterlist.h>
37#include <linux/version.h>
38#include <linux/err.h>
39#include <linux/scatterlist.h>
40#include <linux/aer.h>
41#include <linux/ctype.h>
42#include <linux/wait.h>
43#include <linux/uio.h>
44#include <scsi/scsi.h>
e67f86b3
AB		 45#include <scsi/sg.h>
46#include <linux/io.h>
47#include <linux/uaccess.h>
4ca90b53 48#include <asm/unaligned.h>
e67f86b3
AB		 49
50#include "skd_s1120.h"
51
52static int skd_dbg_level;
53static int skd_isr_comp_limit = 4;
54
55enum {
56 STEC_LINK_2_5GTS = 0,
57 STEC_LINK_5GTS = 1,
58 STEC_LINK_8GTS = 2,
59 STEC_LINK_UNKNOWN = 0xFF
60};
61
62enum {
63 SKD_FLUSH_INITIALIZER,
64 SKD_FLUSH_ZERO_SIZE_FIRST,
65 SKD_FLUSH_DATA_SECOND,
66};
67
e67f86b3
AB		 68#define SKD_ASSERT(expr) \
69 do { \
70 if (unlikely(!(expr))) { \
71 pr_err("Assertion failed! %s,%s,%s,line=%d\n", \
72 # expr, __FILE__, __func__, __LINE__); \
73 } \
74 } while (0)
75
e67f86b3
AB		 76#define DRV_NAME "skd"
77#define DRV_VERSION "2.2.1"
78#define DRV_BUILD_ID "0260"
79#define PFX DRV_NAME ": "
80#define DRV_BIN_VERSION 0x100
81#define DRV_VER_COMPL "2.2.1." DRV_BUILD_ID
82
83MODULE_AUTHOR("bug-reports: support@stec-inc.com");
84MODULE_LICENSE("Dual BSD/GPL");
85
38d4a1bb 86MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver (b" DRV_BUILD_ID ")");
e67f86b3
AB		 87MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
88
89#define PCI_VENDOR_ID_STEC 0x1B39
90#define PCI_DEVICE_ID_S1120 0x0001
91
92#define SKD_FUA_NV (1 << 1)
93#define SKD_MINORS_PER_DEVICE 16
94
95#define SKD_MAX_QUEUE_DEPTH 200u
96
97#define SKD_PAUSE_TIMEOUT (5 * 1000)
98
99#define SKD_N_FITMSG_BYTES (512u)
100
101#define SKD_N_SPECIAL_CONTEXT 32u
102#define SKD_N_SPECIAL_FITMSG_BYTES (128u)
103
104/* SG elements are 32 bytes, so we can make this 4096 and still be under the
105 * 128KB limit. That allows 4096*4K = 16M xfer size
106 */
107#define SKD_N_SG_PER_REQ_DEFAULT 256u
108#define SKD_N_SG_PER_SPECIAL 256u
109
110#define SKD_N_COMPLETION_ENTRY 256u
111#define SKD_N_READ_CAP_BYTES (8u)
112
113#define SKD_N_INTERNAL_BYTES (512u)
114
115/* 5 bits of uniqifier, 0xF800 */
116#define SKD_ID_INCR (0x400)
117#define SKD_ID_TABLE_MASK (3u << 8u)
118#define SKD_ID_RW_REQUEST (0u << 8u)
119#define SKD_ID_INTERNAL (1u << 8u)
120#define SKD_ID_SPECIAL_REQUEST (2u << 8u)
121#define SKD_ID_FIT_MSG (3u << 8u)
122#define SKD_ID_SLOT_MASK 0x00FFu
123#define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
124
125#define SKD_N_TIMEOUT_SLOT 4u
126#define SKD_TIMEOUT_SLOT_MASK 3u
127
128#define SKD_N_MAX_SECTORS 2048u
129
130#define SKD_MAX_RETRIES 2u
131
132#define SKD_TIMER_SECONDS(seconds) (seconds)
133#define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
134
135#define INQ_STD_NBYTES 36
e67f86b3
AB		 136
137enum skd_drvr_state {
138 SKD_DRVR_STATE_LOAD,
139 SKD_DRVR_STATE_IDLE,
140 SKD_DRVR_STATE_BUSY,
141 SKD_DRVR_STATE_STARTING,
142 SKD_DRVR_STATE_ONLINE,
143 SKD_DRVR_STATE_PAUSING,
144 SKD_DRVR_STATE_PAUSED,
145 SKD_DRVR_STATE_DRAINING_TIMEOUT,
146 SKD_DRVR_STATE_RESTARTING,
147 SKD_DRVR_STATE_RESUMING,
148 SKD_DRVR_STATE_STOPPING,
149 SKD_DRVR_STATE_FAULT,
150 SKD_DRVR_STATE_DISAPPEARED,
151 SKD_DRVR_STATE_PROTOCOL_MISMATCH,
152 SKD_DRVR_STATE_BUSY_ERASE,
153 SKD_DRVR_STATE_BUSY_SANITIZE,
154 SKD_DRVR_STATE_BUSY_IMMINENT,
155 SKD_DRVR_STATE_WAIT_BOOT,
156 SKD_DRVR_STATE_SYNCING,
157};
158
159#define SKD_WAIT_BOOT_TIMO SKD_TIMER_SECONDS(90u)
160#define SKD_STARTING_TIMO SKD_TIMER_SECONDS(8u)
161#define SKD_RESTARTING_TIMO SKD_TIMER_MINUTES(4u)
162#define SKD_DRAINING_TIMO SKD_TIMER_SECONDS(6u)
163#define SKD_BUSY_TIMO SKD_TIMER_MINUTES(20u)
164#define SKD_STARTED_BUSY_TIMO SKD_TIMER_SECONDS(60u)
165#define SKD_START_WAIT_SECONDS 90u
166
167enum skd_req_state {
168 SKD_REQ_STATE_IDLE,
169 SKD_REQ_STATE_SETUP,
170 SKD_REQ_STATE_BUSY,
171 SKD_REQ_STATE_COMPLETED,
172 SKD_REQ_STATE_TIMEOUT,
173 SKD_REQ_STATE_ABORTED,
174};
175
176enum skd_fit_msg_state {
177 SKD_MSG_STATE_IDLE,
178 SKD_MSG_STATE_BUSY,
179};
180
181enum skd_check_status_action {
182 SKD_CHECK_STATUS_REPORT_GOOD,
183 SKD_CHECK_STATUS_REPORT_SMART_ALERT,
184 SKD_CHECK_STATUS_REQUEUE_REQUEST,
185 SKD_CHECK_STATUS_REPORT_ERROR,
186 SKD_CHECK_STATUS_BUSY_IMMINENT,
187};
188
189struct skd_fitmsg_context {
190 enum skd_fit_msg_state state;
191
192 struct skd_fitmsg_context *next;
193
194 u32 id;
195 u16 outstanding;
196
197 u32 length;
198 u32 offset;
199
200 u8 *msg_buf;
201 dma_addr_t mb_dma_address;
202};
203
204struct skd_request_context {
205 enum skd_req_state state;
206
207 struct skd_request_context *next;
208
209 u16 id;
210 u32 fitmsg_id;
211
212 struct request *req;
e67f86b3 213 u8 flush_cmd;
e67f86b3
AB		 214
215 u32 timeout_stamp;
216 u8 sg_data_dir;
217 struct scatterlist *sg;
218 u32 n_sg;
219 u32 sg_byte_count;
220
221 struct fit_sg_descriptor *sksg_list;
222 dma_addr_t sksg_dma_address;
223
224 struct fit_completion_entry_v1 completion;
225
226 struct fit_comp_error_info err_info;
227
228};
229#define SKD_DATA_DIR_HOST_TO_CARD 1
230#define SKD_DATA_DIR_CARD_TO_HOST 2
e67f86b3
AB		 231
232struct skd_special_context {
233 struct skd_request_context req;
234
235 u8 orphaned;
236
237 void *data_buf;
238 dma_addr_t db_dma_address;
239
240 u8 *msg_buf;
241 dma_addr_t mb_dma_address;
242};
243
244struct skd_sg_io {
245 fmode_t mode;
246 void __user *argp;
247
248 struct sg_io_hdr sg;
249
250 u8 cdb[16];
251
252 u32 dxfer_len;
253 u32 iovcnt;
254 struct sg_iovec *iov;
255 struct sg_iovec no_iov_iov;
256
257 struct skd_special_context *skspcl;
258};
259
260typedef enum skd_irq_type {
261 SKD_IRQ_LEGACY,
262 SKD_IRQ_MSI,
263 SKD_IRQ_MSIX
264} skd_irq_type_t;
265
266#define SKD_MAX_BARS 2
267
268struct skd_device {
269 volatile void __iomem *mem_map[SKD_MAX_BARS];
270 resource_size_t mem_phys[SKD_MAX_BARS];
271 u32 mem_size[SKD_MAX_BARS];
272
e67f86b3
AB		 273 struct skd_msix_entry *msix_entries;
274
275 struct pci_dev *pdev;
276 int pcie_error_reporting_is_enabled;
277
278 spinlock_t lock;
279 struct gendisk *disk;
280 struct request_queue *queue;
281 struct device *class_dev;
282 int gendisk_on;
283 int sync_done;
284
285 atomic_t device_count;
286 u32 devno;
287 u32 major;
288 char name[32];
289 char isr_name[30];
290
291 enum skd_drvr_state state;
292 u32 drive_state;
293
294 u32 in_flight;
295 u32 cur_max_queue_depth;
296 u32 queue_low_water_mark;
297 u32 dev_max_queue_depth;
298
299 u32 num_fitmsg_context;
300 u32 num_req_context;
301
302 u32 timeout_slot[SKD_N_TIMEOUT_SLOT];
303 u32 timeout_stamp;
304 struct skd_fitmsg_context *skmsg_free_list;
305 struct skd_fitmsg_context *skmsg_table;
306
307 struct skd_request_context *skreq_free_list;
308 struct skd_request_context *skreq_table;
309
310 struct skd_special_context *skspcl_free_list;
311 struct skd_special_context *skspcl_table;
312
313 struct skd_special_context internal_skspcl;
314 u32 read_cap_blocksize;
315 u32 read_cap_last_lba;
316 int read_cap_is_valid;
317 int inquiry_is_valid;
318 u8 inq_serial_num[13]; /*12 chars plus null term */
319 u8 id_str[80]; /* holds a composite name (pci + sernum) */
320
321 u8 skcomp_cycle;
322 u32 skcomp_ix;
323 struct fit_completion_entry_v1 *skcomp_table;
324 struct fit_comp_error_info *skerr_table;
325 dma_addr_t cq_dma_address;
326
327 wait_queue_head_t waitq;
328
329 struct timer_list timer;
330 u32 timer_countdown;
331 u32 timer_substate;
332
333 int n_special;
334 int sgs_per_request;
335 u32 last_mtd;
336
337 u32 proto_ver;
338
339 int dbg_level;
340 u32 connect_time_stamp;
341 int connect_retries;
342#define SKD_MAX_CONNECT_RETRIES 16
343 u32 drive_jiffies;
344
345 u32 timo_slot;
346
e67f86b3 347
38d4a1bb 348 struct work_struct completion_worker;
e67f86b3
AB		 349};
350
351#define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
352#define SKD_READL(DEV, OFF) skd_reg_read32(DEV, OFF)
353#define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
354
355static inline u32 skd_reg_read32(struct skd_device *skdev, u32 offset)
356{
357 u32 val;
358
359 if (likely(skdev->dbg_level < 2))
360 return readl(skdev->mem_map[1] + offset);
361 else {
362 barrier();
363 val = readl(skdev->mem_map[1] + offset);
364 barrier();
2e44b427 365 pr_debug("%s:%s:%d offset %x = %x\n",
366 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 367 return val;
368 }
369
370}
371
372static inline void skd_reg_write32(struct skd_device *skdev, u32 val,
373 u32 offset)
374{
375 if (likely(skdev->dbg_level < 2)) {
376 writel(val, skdev->mem_map[1] + offset);
377 barrier();
e67f86b3
AB		 378 } else {
379 barrier();
380 writel(val, skdev->mem_map[1] + offset);
381 barrier();
2e44b427 382 pr_debug("%s:%s:%d offset %x = %x\n",
383 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 384 }
385}
386
387static inline void skd_reg_write64(struct skd_device *skdev, u64 val,
388 u32 offset)
389{
390 if (likely(skdev->dbg_level < 2)) {
391 writeq(val, skdev->mem_map[1] + offset);
392 barrier();
e67f86b3
AB		 393 } else {
394 barrier();
395 writeq(val, skdev->mem_map[1] + offset);
396 barrier();
2e44b427 397 pr_debug("%s:%s:%d offset %x = %016llx\n",
398 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 399 }
400}
401
402
403#define SKD_IRQ_DEFAULT SKD_IRQ_MSI
404static int skd_isr_type = SKD_IRQ_DEFAULT;
405
406module_param(skd_isr_type, int, 0444);
407MODULE_PARM_DESC(skd_isr_type, "Interrupt type capability."
408 " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
409
410#define SKD_MAX_REQ_PER_MSG_DEFAULT 1
411static int skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
412
413module_param(skd_max_req_per_msg, int, 0444);
414MODULE_PARM_DESC(skd_max_req_per_msg,
415 "Maximum SCSI requests packed in a single message."
416 " (1-14, default==1)");
417
418#define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
419#define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
420static int skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
421
422module_param(skd_max_queue_depth, int, 0444);
423MODULE_PARM_DESC(skd_max_queue_depth,
424 "Maximum SCSI requests issued to s1120."
425 " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR ")");
426
427static int skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
428module_param(skd_sgs_per_request, int, 0444);
429MODULE_PARM_DESC(skd_sgs_per_request,
430 "Maximum SG elements per block request."
431 " (1-4096, default==256)");
432
433static int skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
434module_param(skd_max_pass_thru, int, 0444);
435MODULE_PARM_DESC(skd_max_pass_thru,
436 "Maximum SCSI pass-thru at a time." " (1-50, default==32)");
437
438module_param(skd_dbg_level, int, 0444);
439MODULE_PARM_DESC(skd_dbg_level, "s1120 debug level (0,1,2)");
440
441module_param(skd_isr_comp_limit, int, 0444);
442MODULE_PARM_DESC(skd_isr_comp_limit, "s1120 isr comp limit (0=none) default=4");
443
e67f86b3
AB		 444/* Major device number dynamically assigned. */
445static u32 skd_major;
446
e67f86b3
AB		 447static void skd_destruct(struct skd_device *skdev);
448static const struct block_device_operations skd_blockdev_ops;
449static void skd_send_fitmsg(struct skd_device *skdev,
450 struct skd_fitmsg_context *skmsg);
451static void skd_send_special_fitmsg(struct skd_device *skdev,
452 struct skd_special_context *skspcl);
453static void skd_request_fn(struct request_queue *rq);
454static void skd_end_request(struct skd_device *skdev,
455 struct skd_request_context *skreq, int error);
456static int skd_preop_sg_list(struct skd_device *skdev,
457 struct skd_request_context *skreq);
458static void skd_postop_sg_list(struct skd_device *skdev,
459 struct skd_request_context *skreq);
460
461static void skd_restart_device(struct skd_device *skdev);
462static int skd_quiesce_dev(struct skd_device *skdev);
463static int skd_unquiesce_dev(struct skd_device *skdev);
464static void skd_release_special(struct skd_device *skdev,
465 struct skd_special_context *skspcl);
466static void skd_disable_interrupts(struct skd_device *skdev);
467static void skd_isr_fwstate(struct skd_device *skdev);
468static void skd_recover_requests(struct skd_device *skdev, int requeue);
469static void skd_soft_reset(struct skd_device *skdev);
470
471static const char *skd_name(struct skd_device *skdev);
472const char *skd_drive_state_to_str(int state);
473const char *skd_skdev_state_to_str(enum skd_drvr_state state);
474static void skd_log_skdev(struct skd_device *skdev, const char *event);
475static void skd_log_skmsg(struct skd_device *skdev,
476 struct skd_fitmsg_context *skmsg, const char *event);
477static void skd_log_skreq(struct skd_device *skdev,
478 struct skd_request_context *skreq, const char *event);
479
e67f86b3
AB		 480/*
481 *****************************************************************************
482 * READ/WRITE REQUESTS
483 *****************************************************************************
484 */
fcd37eb3 485static void skd_fail_all_pending(struct skd_device *skdev)
e67f86b3
AB		 486{
487 struct request_queue *q = skdev->queue;
488 struct request *req;
489
490 for (;; ) {
491 req = blk_peek_request(q);
492 if (req == NULL)
493 break;
494 blk_start_request(req);
495 __blk_end_request_all(req, -EIO);
496 }
497}
498
e67f86b3
AB		 499static void
500skd_prep_rw_cdb(struct skd_scsi_request *scsi_req,
501 int data_dir, unsigned lba,
502 unsigned count)
503{
504 if (data_dir == READ)
505 scsi_req->cdb[0] = 0x28;
506 else
507 scsi_req->cdb[0] = 0x2a;
508
509 scsi_req->cdb[1] = 0;
510 scsi_req->cdb[2] = (lba & 0xff000000) >> 24;
511 scsi_req->cdb[3] = (lba & 0xff0000) >> 16;
512 scsi_req->cdb[4] = (lba & 0xff00) >> 8;
513 scsi_req->cdb[5] = (lba & 0xff);
514 scsi_req->cdb[6] = 0;
515 scsi_req->cdb[7] = (count & 0xff00) >> 8;
516 scsi_req->cdb[8] = count & 0xff;
517 scsi_req->cdb[9] = 0;
518}
519
520static void
521skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
38d4a1bb 522 struct skd_request_context *skreq)
e67f86b3
AB		 523{
524 skreq->flush_cmd = 1;
525
526 scsi_req->cdb[0] = 0x35;
527 scsi_req->cdb[1] = 0;
528 scsi_req->cdb[2] = 0;
529 scsi_req->cdb[3] = 0;
530 scsi_req->cdb[4] = 0;
531 scsi_req->cdb[5] = 0;
532 scsi_req->cdb[6] = 0;
533 scsi_req->cdb[7] = 0;
534 scsi_req->cdb[8] = 0;
535 scsi_req->cdb[9] = 0;
536}
537
e67f86b3
AB		 538static void skd_request_fn_not_online(struct request_queue *q);
539
540static void skd_request_fn(struct request_queue *q)
541{
542 struct skd_device *skdev = q->queuedata;
543 struct skd_fitmsg_context *skmsg = NULL;
544 struct fit_msg_hdr *fmh = NULL;
545 struct skd_request_context *skreq;
546 struct request *req = NULL;
e67f86b3 547 struct skd_scsi_request *scsi_req;
e67f86b3
AB		 548 unsigned long io_flags;
549 int error;
550 u32 lba;
551 u32 count;
552 int data_dir;
553 u32 be_lba;
554 u32 be_count;
555 u64 be_dmaa;
556 u64 cmdctxt;
557 u32 timo_slot;
558 void *cmd_ptr;
559 int flush, fua;
560
561 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
562 skd_request_fn_not_online(q);
563 return;
564 }
565
6a5ec65b 566 if (blk_queue_stopped(skdev->queue)) {
e67f86b3
AB		 567 if (skdev->skmsg_free_list == NULL ||
568 skdev->skreq_free_list == NULL ||
569 skdev->in_flight >= skdev->queue_low_water_mark)
570 /* There is still some kind of shortage */
571 return;
572
6a5ec65b 573 queue_flag_clear(QUEUE_FLAG_STOPPED, skdev->queue);
e67f86b3
AB		 574 }
575
576 /*
577 * Stop conditions:
578 * - There are no more native requests
579 * - There are already the maximum number of requests in progress
580 * - There are no more skd_request_context entries
581 * - There are no more FIT msg buffers
582 */
583 for (;; ) {
584
585 flush = fua = 0;
586
fcd37eb3 587 req = blk_peek_request(q);
e67f86b3 588
fcd37eb3
JA		 589 /* Are there any native requests to start? */
590 if (req == NULL)
591 break;
e67f86b3 592
fcd37eb3
JA		 593 lba = (u32)blk_rq_pos(req);
594 count = blk_rq_sectors(req);
595 data_dir = rq_data_dir(req);
596 io_flags = req->cmd_flags;
e67f86b3 597
3a5e02ce 598 if (req_op(req) == REQ_OP_FLUSH)
fcd37eb3 599 flush++;
e67f86b3 600
fcd37eb3
JA		 601 if (io_flags & REQ_FUA)
602 fua++;
e67f86b3 603
fcd37eb3
JA		 604 pr_debug("%s:%s:%d new req=%p lba=%u(0x%x) "
605 "count=%u(0x%x) dir=%d\n",
606 skdev->name, __func__, __LINE__,
607 req, lba, lba, count, count, data_dir);
e67f86b3 608
38d4a1bb 609 /* At this point we know there is a request */
e67f86b3
AB		 610
611 /* Are too many requets already in progress? */
612 if (skdev->in_flight >= skdev->cur_max_queue_depth) {
2e44b427 613 pr_debug("%s:%s:%d qdepth %d, limit %d\n",
614 skdev->name, __func__, __LINE__,
615 skdev->in_flight, skdev->cur_max_queue_depth);
e67f86b3
AB		 616 break;
617 }
618
619 /* Is a skd_request_context available? */
620 skreq = skdev->skreq_free_list;
621 if (skreq == NULL) {
2e44b427 622 pr_debug("%s:%s:%d Out of req=%p\n",
623 skdev->name, __func__, __LINE__, q);
e67f86b3
AB		 624 break;
625 }
626 SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
627 SKD_ASSERT((skreq->id & SKD_ID_INCR) == 0);
628
629 /* Now we check to see if we can get a fit msg */
630 if (skmsg == NULL) {
631 if (skdev->skmsg_free_list == NULL) {
2e44b427 632 pr_debug("%s:%s:%d Out of msg\n",
633 skdev->name, __func__, __LINE__);
e67f86b3
AB		 634 break;
635 }
636 }
637
638 skreq->flush_cmd = 0;
639 skreq->n_sg = 0;
640 skreq->sg_byte_count = 0;
e67f86b3
AB		 641
642 /*
38d4a1bb 643 * OK to now dequeue request from q.
e67f86b3
AB		 644 *
645 * At this point we are comitted to either start or reject
646 * the native request. Note that skd_request_context is
647 * available but is still at the head of the free list.
648 */
fcd37eb3
JA		 649 blk_start_request(req);
650 skreq->req = req;
651 skreq->fitmsg_id = 0;
e67f86b3
AB		 652
653 /* Either a FIT msg is in progress or we have to start one. */
654 if (skmsg == NULL) {
655 /* Are there any FIT msg buffers available? */
656 skmsg = skdev->skmsg_free_list;
657 if (skmsg == NULL) {
2e44b427 658 pr_debug("%s:%s:%d Out of msg skdev=%p\n",
659 skdev->name, __func__, __LINE__,
660 skdev);
e67f86b3
AB		 661 break;
662 }
663 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_IDLE);
664 SKD_ASSERT((skmsg->id & SKD_ID_INCR) == 0);
665
666 skdev->skmsg_free_list = skmsg->next;
667
668 skmsg->state = SKD_MSG_STATE_BUSY;
669 skmsg->id += SKD_ID_INCR;
670
671 /* Initialize the FIT msg header */
672 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
673 memset(fmh, 0, sizeof(*fmh));
674 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
675 skmsg->length = sizeof(*fmh);
676 }
677
678 skreq->fitmsg_id = skmsg->id;
679
680 /*
681 * Note that a FIT msg may have just been started
682 * but contains no SoFIT requests yet.
683 */
684
685 /*
686 * Transcode the request, checking as we go. The outcome of
687 * the transcoding is represented by the error variable.
688 */
689 cmd_ptr = &skmsg->msg_buf[skmsg->length];
690 memset(cmd_ptr, 0, 32);
691
692 be_lba = cpu_to_be32(lba);
693 be_count = cpu_to_be32(count);
694 be_dmaa = cpu_to_be64((u64)skreq->sksg_dma_address);
695 cmdctxt = skreq->id + SKD_ID_INCR;
696
697 scsi_req = cmd_ptr;
698 scsi_req->hdr.tag = cmdctxt;
699 scsi_req->hdr.sg_list_dma_address = be_dmaa;
700
701 if (data_dir == READ)
702 skreq->sg_data_dir = SKD_DATA_DIR_CARD_TO_HOST;
703 else
704 skreq->sg_data_dir = SKD_DATA_DIR_HOST_TO_CARD;
705
49bdedb3 706 if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
e67f86b3
AB		 707 skd_prep_zerosize_flush_cdb(scsi_req, skreq);
708 SKD_ASSERT(skreq->flush_cmd == 1);
709
710 } else {
711 skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
712 }
713
714 if (fua)
715 scsi_req->cdb[1] |= SKD_FUA_NV;
716
fcd37eb3 717 if (!req->bio)
e67f86b3
AB		 718 goto skip_sg;
719
720 error = skd_preop_sg_list(skdev, skreq);
721
722 if (error != 0) {
723 /*
724 * Complete the native request with error.
725 * Note that the request context is still at the
726 * head of the free list, and that the SoFIT request
727 * was encoded into the FIT msg buffer but the FIT
728 * msg length has not been updated. In short, the
729 * only resource that has been allocated but might
730 * not be used is that the FIT msg could be empty.
731 */
2e44b427 732 pr_debug("%s:%s:%d error Out\n",
733 skdev->name, __func__, __LINE__);
e67f86b3
AB		 734 skd_end_request(skdev, skreq, error);
735 continue;
736 }
737
738skip_sg:
739 scsi_req->hdr.sg_list_len_bytes =
740 cpu_to_be32(skreq->sg_byte_count);
741
742 /* Complete resource allocations. */
743 skdev->skreq_free_list = skreq->next;
744 skreq->state = SKD_REQ_STATE_BUSY;
745 skreq->id += SKD_ID_INCR;
746
747 skmsg->length += sizeof(struct skd_scsi_request);
748 fmh->num_protocol_cmds_coalesced++;
749
750 /*
751 * Update the active request counts.
752 * Capture the timeout timestamp.
753 */
754 skreq->timeout_stamp = skdev->timeout_stamp;
755 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
756 skdev->timeout_slot[timo_slot]++;
757 skdev->in_flight++;
2e44b427 758 pr_debug("%s:%s:%d req=0x%x busy=%d\n",
759 skdev->name, __func__, __LINE__,
760 skreq->id, skdev->in_flight);
e67f86b3
AB		 761
762 /*
763 * If the FIT msg buffer is full send it.
764 */
765 if (skmsg->length >= SKD_N_FITMSG_BYTES ||
766 fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
767 skd_send_fitmsg(skdev, skmsg);
768 skmsg = NULL;
769 fmh = NULL;
770 }
771 }
772
773 /*
774 * Is a FIT msg in progress? If it is empty put the buffer back
775 * on the free list. If it is non-empty send what we got.
776 * This minimizes latency when there are fewer requests than
777 * what fits in a FIT msg.
778 */
779 if (skmsg != NULL) {
780 /* Bigger than just a FIT msg header? */
781 if (skmsg->length > sizeof(struct fit_msg_hdr)) {
2e44b427 782 pr_debug("%s:%s:%d sending msg=%p, len %d\n",
783 skdev->name, __func__, __LINE__,
784 skmsg, skmsg->length);
e67f86b3
AB		 785 skd_send_fitmsg(skdev, skmsg);
786 } else {
787 /*
788 * The FIT msg is empty. It means we got started
789 * on the msg, but the requests were rejected.
790 */
791 skmsg->state = SKD_MSG_STATE_IDLE;
792 skmsg->id += SKD_ID_INCR;
793 skmsg->next = skdev->skmsg_free_list;
794 skdev->skmsg_free_list = skmsg;
795 }
796 skmsg = NULL;
797 fmh = NULL;
798 }
799
800 /*
801 * If req is non-NULL it means there is something to do but
802 * we are out of a resource.
803 */
fcd37eb3 804 if (req)
6a5ec65b 805 blk_stop_queue(skdev->queue);
e67f86b3
AB		 806}
807
38d4a1bb
MS		 808static void skd_end_request(struct skd_device *skdev,
809 struct skd_request_context *skreq, int error)
e67f86b3 810{
e67f86b3
AB		 811 if (unlikely(error)) {
812 struct request *req = skreq->req;
813 char *cmd = (rq_data_dir(req) == READ) ? "read" : "write";
814 u32 lba = (u32)blk_rq_pos(req);
815 u32 count = blk_rq_sectors(req);
816
817 pr_err("(%s): Error cmd=%s sect=%u count=%u id=0x%x\n",
818 skd_name(skdev), cmd, lba, count, skreq->id);
819 } else
2e44b427 820 pr_debug("%s:%s:%d id=0x%x error=%d\n",
821 skdev->name, __func__, __LINE__, skreq->id, error);
e67f86b3
AB		 822
823 __blk_end_request_all(skreq->req, error);
824}
825
fcd37eb3 826static int skd_preop_sg_list(struct skd_device *skdev,
38d4a1bb 827 struct skd_request_context *skreq)
e67f86b3
AB		 828{
829 struct request *req = skreq->req;
830 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
831 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
832 struct scatterlist *sg = &skreq->sg[0];
833 int n_sg;
834 int i;
835
836 skreq->sg_byte_count = 0;
837
838 /* SKD_ASSERT(skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD ||
839 skreq->sg_data_dir == SKD_DATA_DIR_CARD_TO_HOST); */
840
841 n_sg = blk_rq_map_sg(skdev->queue, req, sg);
842 if (n_sg <= 0)
843 return -EINVAL;
844
845 /*
846 * Map scatterlist to PCI bus addresses.
847 * Note PCI might change the number of entries.
848 */
849 n_sg = pci_map_sg(skdev->pdev, sg, n_sg, pci_dir);
850 if (n_sg <= 0)
851 return -EINVAL;
852
853 SKD_ASSERT(n_sg <= skdev->sgs_per_request);
854
855 skreq->n_sg = n_sg;
856
857 for (i = 0; i < n_sg; i++) {
858 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
859 u32 cnt = sg_dma_len(&sg[i]);
860 uint64_t dma_addr = sg_dma_address(&sg[i]);
861
862 sgd->control = FIT_SGD_CONTROL_NOT_LAST;
863 sgd->byte_count = cnt;
864 skreq->sg_byte_count += cnt;
865 sgd->host_side_addr = dma_addr;
866 sgd->dev_side_addr = 0;
867 }
868
869 skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;
870 skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
871
872 if (unlikely(skdev->dbg_level > 1)) {
2e44b427 873 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
874 skdev->name, __func__, __LINE__,
875 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 876 for (i = 0; i < n_sg; i++) {
877 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
2e44b427 878 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
879 "addr=0x%llx next=0x%llx\n",
880 skdev->name, __func__, __LINE__,
881 i, sgd->byte_count, sgd->control,
882 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 883 }
884 }
885
886 return 0;
887}
888
fcd37eb3 889static void skd_postop_sg_list(struct skd_device *skdev,
38d4a1bb 890 struct skd_request_context *skreq)
e67f86b3
AB		 891{
892 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
893 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
894
895 /*
896 * restore the next ptr for next IO request so we
897 * don't have to set it every time.
898 */
899 skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
900 skreq->sksg_dma_address +
901 ((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
902 pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, pci_dir);
903}
904
e67f86b3
AB		 905static void skd_request_fn_not_online(struct request_queue *q)
906{
907 struct skd_device *skdev = q->queuedata;
908 int error;
909
910 SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
911
912 skd_log_skdev(skdev, "req_not_online");
913 switch (skdev->state) {
914 case SKD_DRVR_STATE_PAUSING:
915 case SKD_DRVR_STATE_PAUSED:
916 case SKD_DRVR_STATE_STARTING:
917 case SKD_DRVR_STATE_RESTARTING:
918 case SKD_DRVR_STATE_WAIT_BOOT:
919 /* In case of starting, we haven't started the queue,
920 * so we can't get here... but requests are
921 * possibly hanging out waiting for us because we
922 * reported the dev/skd0 already. They'll wait
923 * forever if connect doesn't complete.
924 * What to do??? delay dev/skd0 ??
925 */
926 case SKD_DRVR_STATE_BUSY:
927 case SKD_DRVR_STATE_BUSY_IMMINENT:
928 case SKD_DRVR_STATE_BUSY_ERASE:
929 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
930 return;
931
932 case SKD_DRVR_STATE_BUSY_SANITIZE:
933 case SKD_DRVR_STATE_STOPPING:
934 case SKD_DRVR_STATE_SYNCING:
935 case SKD_DRVR_STATE_FAULT:
936 case SKD_DRVR_STATE_DISAPPEARED:
937 default:
938 error = -EIO;
939 break;
940 }
941
942 /* If we get here, terminate all pending block requeusts
943 * with EIO and any scsi pass thru with appropriate sense
944 */
945
946 skd_fail_all_pending(skdev);
947}
948
949/*
950 *****************************************************************************
951 * TIMER
952 *****************************************************************************
953 */
954
955static void skd_timer_tick_not_online(struct skd_device *skdev);
956
957static void skd_timer_tick(ulong arg)
958{
959 struct skd_device *skdev = (struct skd_device *)arg;
960
961 u32 timo_slot;
962 u32 overdue_timestamp;
963 unsigned long reqflags;
964 u32 state;
965
966 if (skdev->state == SKD_DRVR_STATE_FAULT)
967 /* The driver has declared fault, and we want it to
968 * stay that way until driver is reloaded.
969 */
970 return;
971
972 spin_lock_irqsave(&skdev->lock, reqflags);
973
974 state = SKD_READL(skdev, FIT_STATUS);
975 state &= FIT_SR_DRIVE_STATE_MASK;
976 if (state != skdev->drive_state)
977 skd_isr_fwstate(skdev);
978
979 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
980 skd_timer_tick_not_online(skdev);
981 goto timer_func_out;
982 }
983 skdev->timeout_stamp++;
984 timo_slot = skdev->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
985
986 /*
987 * All requests that happened during the previous use of
988 * this slot should be done by now. The previous use was
989 * over 7 seconds ago.
990 */
991 if (skdev->timeout_slot[timo_slot] == 0)
992 goto timer_func_out;
993
994 /* Something is overdue */
995 overdue_timestamp = skdev->timeout_stamp - SKD_N_TIMEOUT_SLOT;
996
2e44b427 997 pr_debug("%s:%s:%d found %d timeouts, draining busy=%d\n",
998 skdev->name, __func__, __LINE__,
999 skdev->timeout_slot[timo_slot], skdev->in_flight);
e67f86b3
AB		 1000 pr_err("(%s): Overdue IOs (%d), busy %d\n",
1001 skd_name(skdev), skdev->timeout_slot[timo_slot],
1002 skdev->in_flight);
1003
1004 skdev->timer_countdown = SKD_DRAINING_TIMO;
1005 skdev->state = SKD_DRVR_STATE_DRAINING_TIMEOUT;
1006 skdev->timo_slot = timo_slot;
6a5ec65b 1007 blk_stop_queue(skdev->queue);
e67f86b3
AB		 1008
1009timer_func_out:
1010 mod_timer(&skdev->timer, (jiffies + HZ));
1011
1012 spin_unlock_irqrestore(&skdev->lock, reqflags);
1013}
1014
1015static void skd_timer_tick_not_online(struct skd_device *skdev)
1016{
1017 switch (skdev->state) {
1018 case SKD_DRVR_STATE_IDLE:
1019 case SKD_DRVR_STATE_LOAD:
1020 break;
1021 case SKD_DRVR_STATE_BUSY_SANITIZE:
2e44b427 1022 pr_debug("%s:%s:%d drive busy sanitize[%x], driver[%x]\n",
1023 skdev->name, __func__, __LINE__,
1024 skdev->drive_state, skdev->state);
e67f86b3
AB		 1025 /* If we've been in sanitize for 3 seconds, we figure we're not
1026 * going to get anymore completions, so recover requests now
1027 */
1028 if (skdev->timer_countdown > 0) {
1029 skdev->timer_countdown--;
1030 return;
1031 }
1032 skd_recover_requests(skdev, 0);
1033 break;
1034
1035 case SKD_DRVR_STATE_BUSY:
1036 case SKD_DRVR_STATE_BUSY_IMMINENT:
1037 case SKD_DRVR_STATE_BUSY_ERASE:
2e44b427 1038 pr_debug("%s:%s:%d busy[%x], countdown=%d\n",
1039 skdev->name, __func__, __LINE__,
1040 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 1041 if (skdev->timer_countdown > 0) {
1042 skdev->timer_countdown--;
1043 return;
1044 }
2e44b427 1045 pr_debug("%s:%s:%d busy[%x], timedout=%d, restarting device.",
1046 skdev->name, __func__, __LINE__,
1047 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 1048 skd_restart_device(skdev);
1049 break;
1050
1051 case SKD_DRVR_STATE_WAIT_BOOT:
1052 case SKD_DRVR_STATE_STARTING:
1053 if (skdev->timer_countdown > 0) {
1054 skdev->timer_countdown--;
1055 return;
1056 }
1057 /* For now, we fault the drive. Could attempt resets to
1058 * revcover at some point. */
1059 skdev->state = SKD_DRVR_STATE_FAULT;
1060
1061 pr_err("(%s): DriveFault Connect Timeout (%x)\n",
1062 skd_name(skdev), skdev->drive_state);
1063
1064 /*start the queue so we can respond with error to requests */
1065 /* wakeup anyone waiting for startup complete */
6a5ec65b 1066 blk_start_queue(skdev->queue);
e67f86b3
AB		 1067 skdev->gendisk_on = -1;
1068 wake_up_interruptible(&skdev->waitq);
1069 break;
1070
1071 case SKD_DRVR_STATE_ONLINE:
1072 /* shouldn't get here. */
1073 break;
1074
1075 case SKD_DRVR_STATE_PAUSING:
1076 case SKD_DRVR_STATE_PAUSED:
1077 break;
1078
1079 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
2e44b427 1080 pr_debug("%s:%s:%d "
1081 "draining busy [%d] tick[%d] qdb[%d] tmls[%d]\n",
1082 skdev->name, __func__, __LINE__,
1083 skdev->timo_slot,
1084 skdev->timer_countdown,
1085 skdev->in_flight,
1086 skdev->timeout_slot[skdev->timo_slot]);
e67f86b3
AB		 1087 /* if the slot has cleared we can let the I/O continue */
1088 if (skdev->timeout_slot[skdev->timo_slot] == 0) {
2e44b427 1089 pr_debug("%s:%s:%d Slot drained, starting queue.\n",
1090 skdev->name, __func__, __LINE__);
e67f86b3 1091 skdev->state = SKD_DRVR_STATE_ONLINE;
6a5ec65b 1092 blk_start_queue(skdev->queue);
e67f86b3
AB		 1093 return;
1094 }
1095 if (skdev->timer_countdown > 0) {
1096 skdev->timer_countdown--;
1097 return;
1098 }
1099 skd_restart_device(skdev);
1100 break;
1101
1102 case SKD_DRVR_STATE_RESTARTING:
1103 if (skdev->timer_countdown > 0) {
1104 skdev->timer_countdown--;
1105 return;
1106 }
1107 /* For now, we fault the drive. Could attempt resets to
1108 * revcover at some point. */
1109 skdev->state = SKD_DRVR_STATE_FAULT;
1110 pr_err("(%s): DriveFault Reconnect Timeout (%x)\n",
1111 skd_name(skdev), skdev->drive_state);
1112
1113 /*
1114 * Recovering does two things:
1115 * 1. completes IO with error
1116 * 2. reclaims dma resources
1117 * When is it safe to recover requests?
1118 * - if the drive state is faulted
1119 * - if the state is still soft reset after out timeout
1120 * - if the drive registers are dead (state = FF)
1121 * If it is "unsafe", we still need to recover, so we will
1122 * disable pci bus mastering and disable our interrupts.
1123 */
1124
1125 if ((skdev->drive_state == FIT_SR_DRIVE_SOFT_RESET) ||
1126 (skdev->drive_state == FIT_SR_DRIVE_FAULT) ||
1127 (skdev->drive_state == FIT_SR_DRIVE_STATE_MASK))
1128 /* It never came out of soft reset. Try to
1129 * recover the requests and then let them
1130 * fail. This is to mitigate hung processes. */
1131 skd_recover_requests(skdev, 0);
1132 else {
1133 pr_err("(%s): Disable BusMaster (%x)\n",
1134 skd_name(skdev), skdev->drive_state);
1135 pci_disable_device(skdev->pdev);
1136 skd_disable_interrupts(skdev);
1137 skd_recover_requests(skdev, 0);
1138 }
1139
1140 /*start the queue so we can respond with error to requests */
1141 /* wakeup anyone waiting for startup complete */
6a5ec65b 1142 blk_start_queue(skdev->queue);
e67f86b3
AB		 1143 skdev->gendisk_on = -1;
1144 wake_up_interruptible(&skdev->waitq);
1145 break;
1146
1147 case SKD_DRVR_STATE_RESUMING:
1148 case SKD_DRVR_STATE_STOPPING:
1149 case SKD_DRVR_STATE_SYNCING:
1150 case SKD_DRVR_STATE_FAULT:
1151 case SKD_DRVR_STATE_DISAPPEARED:
1152 default:
1153 break;
1154 }
1155}
1156
1157static int skd_start_timer(struct skd_device *skdev)
1158{
1159 int rc;
1160
1161 init_timer(&skdev->timer);
1162 setup_timer(&skdev->timer, skd_timer_tick, (ulong)skdev);
1163
1164 rc = mod_timer(&skdev->timer, (jiffies + HZ));
1165 if (rc)
1166 pr_err("%s: failed to start timer %d\n",
1167 __func__, rc);
1168 return rc;
1169}
1170
1171static void skd_kill_timer(struct skd_device *skdev)
1172{
1173 del_timer_sync(&skdev->timer);
1174}
1175
1176/*
1177 *****************************************************************************
1178 * IOCTL
1179 *****************************************************************************
1180 */
1181static int skd_ioctl_sg_io(struct skd_device *skdev,
1182 fmode_t mode, void __user *argp);
1183static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1184 struct skd_sg_io *sksgio);
1185static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1186 struct skd_sg_io *sksgio);
1187static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1188 struct skd_sg_io *sksgio);
1189static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1190 struct skd_sg_io *sksgio, int dxfer_dir);
1191static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1192 struct skd_sg_io *sksgio);
1193static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio);
1194static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1195 struct skd_sg_io *sksgio);
1196static int skd_sg_io_put_status(struct skd_device *skdev,
1197 struct skd_sg_io *sksgio);
1198
1199static void skd_complete_special(struct skd_device *skdev,
1200 volatile struct fit_completion_entry_v1
1201 *skcomp,
1202 volatile struct fit_comp_error_info *skerr,
1203 struct skd_special_context *skspcl);
1204
1205static int skd_bdev_ioctl(struct block_device *bdev, fmode_t mode,
1206 uint cmd_in, ulong arg)
1207{
1208 int rc = 0;
1209 struct gendisk *disk = bdev->bd_disk;
1210 struct skd_device *skdev = disk->private_data;
1211 void __user *p = (void *)arg;
1212
2e44b427 1213 pr_debug("%s:%s:%d %s: CMD[%s] ioctl mode 0x%x, cmd 0x%x arg %0lx\n",
1214 skdev->name, __func__, __LINE__,
1215 disk->disk_name, current->comm, mode, cmd_in, arg);
e67f86b3
AB		 1216
1217 if (!capable(CAP_SYS_ADMIN))
1218 return -EPERM;
1219
1220 switch (cmd_in) {
1221 case SG_SET_TIMEOUT:
1222 case SG_GET_TIMEOUT:
1223 case SG_GET_VERSION_NUM:
1224 rc = scsi_cmd_ioctl(disk->queue, disk, mode, cmd_in, p);
1225 break;
1226 case SG_IO:
1227 rc = skd_ioctl_sg_io(skdev, mode, p);
1228 break;
1229
1230 default:
1231 rc = -ENOTTY;
1232 break;
1233 }
1234
2e44b427 1235 pr_debug("%s:%s:%d %s: completion rc %d\n",
1236 skdev->name, __func__, __LINE__, disk->disk_name, rc);
e67f86b3
AB		 1237 return rc;
1238}
1239
1240static int skd_ioctl_sg_io(struct skd_device *skdev, fmode_t mode,
1241 void __user *argp)
1242{
1243 int rc;
1244 struct skd_sg_io sksgio;
1245
1246 memset(&sksgio, 0, sizeof(sksgio));
1247 sksgio.mode = mode;
1248 sksgio.argp = argp;
1249 sksgio.iov = &sksgio.no_iov_iov;
1250
1251 switch (skdev->state) {
1252 case SKD_DRVR_STATE_ONLINE:
1253 case SKD_DRVR_STATE_BUSY_IMMINENT:
1254 break;
1255
1256 default:
2e44b427 1257 pr_debug("%s:%s:%d drive not online\n",
1258 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1259 rc = -ENXIO;
1260 goto out;
1261 }
1262
f721bb0d
AB		 1263 rc = skd_sg_io_get_and_check_args(skdev, &sksgio);
1264 if (rc)
1265 goto out;
1266
1267 rc = skd_sg_io_obtain_skspcl(skdev, &sksgio);
1268 if (rc)
1269 goto out;
1270
1271 rc = skd_sg_io_prep_buffering(skdev, &sksgio);
1272 if (rc)
1273 goto out;
1274
1275 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_TO_DEV);
1276 if (rc)
e67f86b3
AB		 1277 goto out;
1278
f721bb0d
AB		 1279 rc = skd_sg_io_send_fitmsg(skdev, &sksgio);
1280 if (rc)
e67f86b3
AB		 1281 goto out;
1282
f721bb0d
AB		 1283 rc = skd_sg_io_await(skdev, &sksgio);
1284 if (rc)
1285 goto out;
1286
1287 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_FROM_DEV);
1288 if (rc)
1289 goto out;
1290
1291 rc = skd_sg_io_put_status(skdev, &sksgio);
1292 if (rc)
e67f86b3
AB		 1293 goto out;
1294
1295 rc = 0;
1296
1297out:
1298 skd_sg_io_release_skspcl(skdev, &sksgio);
1299
1300 if (sksgio.iov != NULL && sksgio.iov != &sksgio.no_iov_iov)
1301 kfree(sksgio.iov);
1302 return rc;
1303}
1304
1305static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1306 struct skd_sg_io *sksgio)
1307{
1308 struct sg_io_hdr *sgp = &sksgio->sg;
1309 int i, acc;
1310
1311 if (!access_ok(VERIFY_WRITE, sksgio->argp, sizeof(sg_io_hdr_t))) {
2e44b427 1312 pr_debug("%s:%s:%d access sg failed %p\n",
1313 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1314 return -EFAULT;
1315 }
1316
1317 if (__copy_from_user(sgp, sksgio->argp, sizeof(sg_io_hdr_t))) {
2e44b427 1318 pr_debug("%s:%s:%d copy_from_user sg failed %p\n",
1319 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1320 return -EFAULT;
1321 }
1322
1323 if (sgp->interface_id != SG_INTERFACE_ID_ORIG) {
2e44b427 1324 pr_debug("%s:%s:%d interface_id invalid 0x%x\n",
1325 skdev->name, __func__, __LINE__, sgp->interface_id);
e67f86b3
AB		 1326 return -EINVAL;
1327 }
1328
1329 if (sgp->cmd_len > sizeof(sksgio->cdb)) {
2e44b427 1330 pr_debug("%s:%s:%d cmd_len invalid %d\n",
1331 skdev->name, __func__, __LINE__, sgp->cmd_len);
e67f86b3
AB		 1332 return -EINVAL;
1333 }
1334
1335 if (sgp->iovec_count > 256) {
2e44b427 1336 pr_debug("%s:%s:%d iovec_count invalid %d\n",
1337 skdev->name, __func__, __LINE__, sgp->iovec_count);
e67f86b3
AB		 1338 return -EINVAL;
1339 }
1340
1341 if (sgp->dxfer_len > (PAGE_SIZE * SKD_N_SG_PER_SPECIAL)) {
2e44b427 1342 pr_debug("%s:%s:%d dxfer_len invalid %d\n",
1343 skdev->name, __func__, __LINE__, sgp->dxfer_len);
e67f86b3
AB		 1344 return -EINVAL;
1345 }
1346
1347 switch (sgp->dxfer_direction) {
1348 case SG_DXFER_NONE:
1349 acc = -1;
1350 break;
1351
1352 case SG_DXFER_TO_DEV:
1353 acc = VERIFY_READ;
1354 break;
1355
1356 case SG_DXFER_FROM_DEV:
1357 case SG_DXFER_TO_FROM_DEV:
1358 acc = VERIFY_WRITE;
1359 break;
1360
1361 default:
2e44b427 1362 pr_debug("%s:%s:%d dxfer_dir invalid %d\n",
1363 skdev->name, __func__, __LINE__, sgp->dxfer_direction);
e67f86b3
AB		 1364 return -EINVAL;
1365 }
1366
1367 if (copy_from_user(sksgio->cdb, sgp->cmdp, sgp->cmd_len)) {
2e44b427 1368 pr_debug("%s:%s:%d copy_from_user cmdp failed %p\n",
1369 skdev->name, __func__, __LINE__, sgp->cmdp);
e67f86b3
AB		 1370 return -EFAULT;
1371 }
1372
1373 if (sgp->mx_sb_len != 0) {
1374 if (!access_ok(VERIFY_WRITE, sgp->sbp, sgp->mx_sb_len)) {
2e44b427 1375 pr_debug("%s:%s:%d access sbp failed %p\n",
1376 skdev->name, __func__, __LINE__, sgp->sbp);
e67f86b3
AB		 1377 return -EFAULT;
1378 }
1379 }
1380
1381 if (sgp->iovec_count == 0) {
1382 sksgio->iov[0].iov_base = sgp->dxferp;
1383 sksgio->iov[0].iov_len = sgp->dxfer_len;
1384 sksgio->iovcnt = 1;
1385 sksgio->dxfer_len = sgp->dxfer_len;
1386 } else {
1387 struct sg_iovec *iov;
1388 uint nbytes = sizeof(*iov) * sgp->iovec_count;
1389 size_t iov_data_len;
1390
1391 iov = kmalloc(nbytes, GFP_KERNEL);
1392 if (iov == NULL) {
2e44b427 1393 pr_debug("%s:%s:%d alloc iovec failed %d\n",
1394 skdev->name, __func__, __LINE__,
1395 sgp->iovec_count);
e67f86b3
AB		 1396 return -ENOMEM;
1397 }
1398 sksgio->iov = iov;
1399 sksgio->iovcnt = sgp->iovec_count;
1400
1401 if (copy_from_user(iov, sgp->dxferp, nbytes)) {
2e44b427 1402 pr_debug("%s:%s:%d copy_from_user iovec failed %p\n",
1403 skdev->name, __func__, __LINE__, sgp->dxferp);
e67f86b3
AB		 1404 return -EFAULT;
1405 }
1406
1407 /*
1408 * Sum up the vecs, making sure they don't overflow
1409 */
1410 iov_data_len = 0;
1411 for (i = 0; i < sgp->iovec_count; i++) {
1412 if (iov_data_len + iov[i].iov_len < iov_data_len)
1413 return -EINVAL;
1414 iov_data_len += iov[i].iov_len;
1415 }
1416
1417 /* SG_IO howto says that the shorter of the two wins */
1418 if (sgp->dxfer_len < iov_data_len) {
1419 sksgio->iovcnt = iov_shorten((struct iovec *)iov,
1420 sgp->iovec_count,
1421 sgp->dxfer_len);
1422 sksgio->dxfer_len = sgp->dxfer_len;
1423 } else
1424 sksgio->dxfer_len = iov_data_len;
1425 }
1426
1427 if (sgp->dxfer_direction != SG_DXFER_NONE) {
1428 struct sg_iovec *iov = sksgio->iov;
1429 for (i = 0; i < sksgio->iovcnt; i++, iov++) {
1430 if (!access_ok(acc, iov->iov_base, iov->iov_len)) {
2e44b427 1431 pr_debug("%s:%s:%d access data failed %p/%d\n",
1432 skdev->name, __func__, __LINE__,
1433 iov->iov_base, (int)iov->iov_len);
e67f86b3
AB		 1434 return -EFAULT;
1435 }
1436 }
1437 }
1438
1439 return 0;
1440}
1441
1442static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1443 struct skd_sg_io *sksgio)
1444{
1445 struct skd_special_context *skspcl = NULL;
1446 int rc;
1447
38d4a1bb 1448 for (;;) {
e67f86b3
AB		 1449 ulong flags;
1450
1451 spin_lock_irqsave(&skdev->lock, flags);
1452 skspcl = skdev->skspcl_free_list;
1453 if (skspcl != NULL) {
1454 skdev->skspcl_free_list =
1455 (struct skd_special_context *)skspcl->req.next;
1456 skspcl->req.id += SKD_ID_INCR;
1457 skspcl->req.state = SKD_REQ_STATE_SETUP;
1458 skspcl->orphaned = 0;
1459 skspcl->req.n_sg = 0;
1460 }
1461 spin_unlock_irqrestore(&skdev->lock, flags);
1462
1463 if (skspcl != NULL) {
1464 rc = 0;
1465 break;
1466 }
1467
2e44b427 1468 pr_debug("%s:%s:%d blocking\n",
1469 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1470
1471 rc = wait_event_interruptible_timeout(
1472 skdev->waitq,
1473 (skdev->skspcl_free_list != NULL),
1474 msecs_to_jiffies(sksgio->sg.timeout));
1475
2e44b427 1476 pr_debug("%s:%s:%d unblocking, rc=%d\n",
1477 skdev->name, __func__, __LINE__, rc);
e67f86b3
AB		 1478
1479 if (rc <= 0) {
1480 if (rc == 0)
1481 rc = -ETIMEDOUT;
1482 else
1483 rc = -EINTR;
1484 break;
1485 }
1486 /*
1487 * If we get here rc > 0 meaning the timeout to
1488 * wait_event_interruptible_timeout() had time left, hence the
1489 * sought event -- non-empty free list -- happened.
1490 * Retry the allocation.
1491 */
1492 }
1493 sksgio->skspcl = skspcl;
1494
1495 return rc;
1496}
1497
1498static int skd_skreq_prep_buffering(struct skd_device *skdev,
1499 struct skd_request_context *skreq,
1500 u32 dxfer_len)
1501{
1502 u32 resid = dxfer_len;
1503
1504 /*
1505 * The DMA engine must have aligned addresses and byte counts.
1506 */
1507 resid += (-resid) & 3;
1508 skreq->sg_byte_count = resid;
1509
1510 skreq->n_sg = 0;
1511
1512 while (resid > 0) {
1513 u32 nbytes = PAGE_SIZE;
1514 u32 ix = skreq->n_sg;
1515 struct scatterlist *sg = &skreq->sg[ix];
1516 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1517 struct page *page;
1518
1519 if (nbytes > resid)
1520 nbytes = resid;
1521
1522 page = alloc_page(GFP_KERNEL);
1523 if (page == NULL)
1524 return -ENOMEM;
1525
1526 sg_set_page(sg, page, nbytes, 0);
1527
1528 /* TODO: This should be going through a pci_???()
1529 * routine to do proper mapping. */
1530 sksg->control = FIT_SGD_CONTROL_NOT_LAST;
1531 sksg->byte_count = nbytes;
1532
1533 sksg->host_side_addr = sg_phys(sg);
1534
1535 sksg->dev_side_addr = 0;
1536 sksg->next_desc_ptr = skreq->sksg_dma_address +
1537 (ix + 1) * sizeof(*sksg);
1538
1539 skreq->n_sg++;
1540 resid -= nbytes;
1541 }
1542
1543 if (skreq->n_sg > 0) {
1544 u32 ix = skreq->n_sg - 1;
1545 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1546
1547 sksg->control = FIT_SGD_CONTROL_LAST;
1548 sksg->next_desc_ptr = 0;
1549 }
1550
1551 if (unlikely(skdev->dbg_level > 1)) {
1552 u32 i;
1553
2e44b427 1554 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
1555 skdev->name, __func__, __LINE__,
1556 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 1557 for (i = 0; i < skreq->n_sg; i++) {
1558 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1559
2e44b427 1560 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
1561 "addr=0x%llx next=0x%llx\n",
1562 skdev->name, __func__, __LINE__,
1563 i, sgd->byte_count, sgd->control,
1564 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 1565 }
1566 }
1567
1568 return 0;
1569}
1570
1571static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1572 struct skd_sg_io *sksgio)
1573{
1574 struct skd_special_context *skspcl = sksgio->skspcl;
1575 struct skd_request_context *skreq = &skspcl->req;
1576 u32 dxfer_len = sksgio->dxfer_len;
1577 int rc;
1578
1579 rc = skd_skreq_prep_buffering(skdev, skreq, dxfer_len);
1580 /*
1581 * Eventually, errors or not, skd_release_special() is called
1582 * to recover allocations including partial allocations.
1583 */
1584 return rc;
1585}
1586
1587static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1588 struct skd_sg_io *sksgio, int dxfer_dir)
1589{
1590 struct skd_special_context *skspcl = sksgio->skspcl;
1591 u32 iov_ix = 0;
1592 struct sg_iovec curiov;
1593 u32 sksg_ix = 0;
1594 u8 *bufp = NULL;
1595 u32 buf_len = 0;
1596 u32 resid = sksgio->dxfer_len;
1597 int rc;
1598
1599 curiov.iov_len = 0;
1600 curiov.iov_base = NULL;
1601
1602 if (dxfer_dir != sksgio->sg.dxfer_direction) {
1603 if (dxfer_dir != SG_DXFER_TO_DEV ||
1604 sksgio->sg.dxfer_direction != SG_DXFER_TO_FROM_DEV)
1605 return 0;
1606 }
1607
1608 while (resid > 0) {
1609 u32 nbytes = PAGE_SIZE;
1610
1611 if (curiov.iov_len == 0) {
1612 curiov = sksgio->iov[iov_ix++];
1613 continue;
1614 }
1615
1616 if (buf_len == 0) {
1617 struct page *page;
1618 page = sg_page(&skspcl->req.sg[sksg_ix++]);
1619 bufp = page_address(page);
1620 buf_len = PAGE_SIZE;
1621 }
1622
1623 nbytes = min_t(u32, nbytes, resid);
1624 nbytes = min_t(u32, nbytes, curiov.iov_len);
1625 nbytes = min_t(u32, nbytes, buf_len);
1626
1627 if (dxfer_dir == SG_DXFER_TO_DEV)
1628 rc = __copy_from_user(bufp, curiov.iov_base, nbytes);
1629 else
1630 rc = __copy_to_user(curiov.iov_base, bufp, nbytes);
1631
1632 if (rc)
1633 return -EFAULT;
1634
1635 resid -= nbytes;
1636 curiov.iov_len -= nbytes;
1637 curiov.iov_base += nbytes;
1638 buf_len -= nbytes;
1639 }
1640
1641 return 0;
1642}
1643
1644static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1645 struct skd_sg_io *sksgio)
1646{
1647 struct skd_special_context *skspcl = sksgio->skspcl;
1648 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
1649 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
1650
1651 memset(skspcl->msg_buf, 0, SKD_N_SPECIAL_FITMSG_BYTES);
1652
1653 /* Initialize the FIT msg header */
1654 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1655 fmh->num_protocol_cmds_coalesced = 1;
1656
1657 /* Initialize the SCSI request */
1658 if (sksgio->sg.dxfer_direction != SG_DXFER_NONE)
1659 scsi_req->hdr.sg_list_dma_address =
1660 cpu_to_be64(skspcl->req.sksg_dma_address);
1661 scsi_req->hdr.tag = skspcl->req.id;
1662 scsi_req->hdr.sg_list_len_bytes =
1663 cpu_to_be32(skspcl->req.sg_byte_count);
1664 memcpy(scsi_req->cdb, sksgio->cdb, sizeof(scsi_req->cdb));
1665
1666 skspcl->req.state = SKD_REQ_STATE_BUSY;
1667 skd_send_special_fitmsg(skdev, skspcl);
1668
1669 return 0;
1670}
1671
1672static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio)
1673{
1674 unsigned long flags;
1675 int rc;
1676
1677 rc = wait_event_interruptible_timeout(skdev->waitq,
1678 (sksgio->skspcl->req.state !=
1679 SKD_REQ_STATE_BUSY),
1680 msecs_to_jiffies(sksgio->sg.
1681 timeout));
1682
1683 spin_lock_irqsave(&skdev->lock, flags);
1684
1685 if (sksgio->skspcl->req.state == SKD_REQ_STATE_ABORTED) {
2e44b427 1686 pr_debug("%s:%s:%d skspcl %p aborted\n",
1687 skdev->name, __func__, __LINE__, sksgio->skspcl);
e67f86b3
AB		 1688
1689 /* Build check cond, sense and let command finish. */
1690 /* For a timeout, we must fabricate completion and sense
1691 * data to complete the command */
1692 sksgio->skspcl->req.completion.status =
1693 SAM_STAT_CHECK_CONDITION;
1694
1695 memset(&sksgio->skspcl->req.err_info, 0,
1696 sizeof(sksgio->skspcl->req.err_info));
1697 sksgio->skspcl->req.err_info.type = 0x70;
1698 sksgio->skspcl->req.err_info.key = ABORTED_COMMAND;
1699 sksgio->skspcl->req.err_info.code = 0x44;
1700 sksgio->skspcl->req.err_info.qual = 0;
1701 rc = 0;
1702 } else if (sksgio->skspcl->req.state != SKD_REQ_STATE_BUSY)
1703 /* No longer on the adapter. We finish. */
1704 rc = 0;
1705 else {
1706 /* Something's gone wrong. Still busy. Timeout or
1707 * user interrupted (control-C). Mark as an orphan
1708 * so it will be disposed when completed. */
1709 sksgio->skspcl->orphaned = 1;
1710 sksgio->skspcl = NULL;
1711 if (rc == 0) {
2e44b427 1712 pr_debug("%s:%s:%d timed out %p (%u ms)\n",
1713 skdev->name, __func__, __LINE__,
1714 sksgio, sksgio->sg.timeout);
e67f86b3
AB		 1715 rc = -ETIMEDOUT;
1716 } else {
2e44b427 1717 pr_debug("%s:%s:%d cntlc %p\n",
1718 skdev->name, __func__, __LINE__, sksgio);
e67f86b3
AB		 1719 rc = -EINTR;
1720 }
1721 }
1722
1723 spin_unlock_irqrestore(&skdev->lock, flags);
1724
1725 return rc;
1726}
1727
1728static int skd_sg_io_put_status(struct skd_device *skdev,
1729 struct skd_sg_io *sksgio)
1730{
1731 struct sg_io_hdr *sgp = &sksgio->sg;
1732 struct skd_special_context *skspcl = sksgio->skspcl;
1733 int resid = 0;
1734
1735 u32 nb = be32_to_cpu(skspcl->req.completion.num_returned_bytes);
1736
1737 sgp->status = skspcl->req.completion.status;
1738 resid = sksgio->dxfer_len - nb;
1739
1740 sgp->masked_status = sgp->status & STATUS_MASK;
1741 sgp->msg_status = 0;
1742 sgp->host_status = 0;
1743 sgp->driver_status = 0;
1744 sgp->resid = resid;
1745 if (sgp->masked_status || sgp->host_status || sgp->driver_status)
1746 sgp->info |= SG_INFO_CHECK;
1747
2e44b427 1748 pr_debug("%s:%s:%d status %x masked %x resid 0x%x\n",
1749 skdev->name, __func__, __LINE__,
1750 sgp->status, sgp->masked_status, sgp->resid);
e67f86b3
AB		 1751
1752 if (sgp->masked_status == SAM_STAT_CHECK_CONDITION) {
1753 if (sgp->mx_sb_len > 0) {
1754 struct fit_comp_error_info *ei = &skspcl->req.err_info;
1755 u32 nbytes = sizeof(*ei);
1756
1757 nbytes = min_t(u32, nbytes, sgp->mx_sb_len);
1758
1759 sgp->sb_len_wr = nbytes;
1760
1761 if (__copy_to_user(sgp->sbp, ei, nbytes)) {
2e44b427 1762 pr_debug("%s:%s:%d copy_to_user sense failed %p\n",
1763 skdev->name, __func__, __LINE__,
1764 sgp->sbp);
e67f86b3
AB		 1765 return -EFAULT;
1766 }
1767 }
1768 }
1769
1770 if (__copy_to_user(sksgio->argp, sgp, sizeof(sg_io_hdr_t))) {
2e44b427 1771 pr_debug("%s:%s:%d copy_to_user sg failed %p\n",
1772 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1773 return -EFAULT;
1774 }
1775
1776 return 0;
1777}
1778
1779static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1780 struct skd_sg_io *sksgio)
1781{
1782 struct skd_special_context *skspcl = sksgio->skspcl;
1783
1784 if (skspcl != NULL) {
1785 ulong flags;
1786
1787 sksgio->skspcl = NULL;
1788
1789 spin_lock_irqsave(&skdev->lock, flags);
1790 skd_release_special(skdev, skspcl);
1791 spin_unlock_irqrestore(&skdev->lock, flags);
1792 }
1793
1794 return 0;
1795}
1796
1797/*
1798 *****************************************************************************
1799 * INTERNAL REQUESTS -- generated by driver itself
1800 *****************************************************************************
1801 */
1802
1803static int skd_format_internal_skspcl(struct skd_device *skdev)
1804{
1805 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1806 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1807 struct fit_msg_hdr *fmh;
1808 uint64_t dma_address;
1809 struct skd_scsi_request *scsi;
1810
1811 fmh = (struct fit_msg_hdr *)&skspcl->msg_buf[0];
1812 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1813 fmh->num_protocol_cmds_coalesced = 1;
1814
1815 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1816 memset(scsi, 0, sizeof(*scsi));
1817 dma_address = skspcl->req.sksg_dma_address;
1818 scsi->hdr.sg_list_dma_address = cpu_to_be64(dma_address);
1819 sgd->control = FIT_SGD_CONTROL_LAST;
1820 sgd->byte_count = 0;
1821 sgd->host_side_addr = skspcl->db_dma_address;
1822 sgd->dev_side_addr = 0;
1823 sgd->next_desc_ptr = 0LL;
1824
1825 return 1;
1826}
1827
1828#define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
1829
1830static void skd_send_internal_skspcl(struct skd_device *skdev,
1831 struct skd_special_context *skspcl,
1832 u8 opcode)
1833{
1834 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1835 struct skd_scsi_request *scsi;
1836 unsigned char *buf = skspcl->data_buf;
1837 int i;
1838
1839 if (skspcl->req.state != SKD_REQ_STATE_IDLE)
1840 /*
1841 * A refresh is already in progress.
1842 * Just wait for it to finish.
1843 */
1844 return;
1845
1846 SKD_ASSERT((skspcl->req.id & SKD_ID_INCR) == 0);
1847 skspcl->req.state = SKD_REQ_STATE_BUSY;
1848 skspcl->req.id += SKD_ID_INCR;
1849
1850 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1851 scsi->hdr.tag = skspcl->req.id;
1852
1853 memset(scsi->cdb, 0, sizeof(scsi->cdb));
1854
1855 switch (opcode) {
1856 case TEST_UNIT_READY:
1857 scsi->cdb[0] = TEST_UNIT_READY;
1858 sgd->byte_count = 0;
1859 scsi->hdr.sg_list_len_bytes = 0;
1860 break;
1861
1862 case READ_CAPACITY:
1863 scsi->cdb[0] = READ_CAPACITY;
1864 sgd->byte_count = SKD_N_READ_CAP_BYTES;
1865 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1866 break;
1867
1868 case INQUIRY:
1869 scsi->cdb[0] = INQUIRY;
1870 scsi->cdb[1] = 0x01; /* evpd */
1871 scsi->cdb[2] = 0x80; /* serial number page */
1872 scsi->cdb[4] = 0x10;
1873 sgd->byte_count = 16;
1874 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1875 break;
1876
1877 case SYNCHRONIZE_CACHE:
1878 scsi->cdb[0] = SYNCHRONIZE_CACHE;
1879 sgd->byte_count = 0;
1880 scsi->hdr.sg_list_len_bytes = 0;
1881 break;
1882
1883 case WRITE_BUFFER:
1884 scsi->cdb[0] = WRITE_BUFFER;
1885 scsi->cdb[1] = 0x02;
1886 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1887 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1888 sgd->byte_count = WR_BUF_SIZE;
1889 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1890 /* fill incrementing byte pattern */
1891 for (i = 0; i < sgd->byte_count; i++)
1892 buf[i] = i & 0xFF;
1893 break;
1894
1895 case READ_BUFFER:
1896 scsi->cdb[0] = READ_BUFFER;
1897 scsi->cdb[1] = 0x02;
1898 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1899 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1900 sgd->byte_count = WR_BUF_SIZE;
1901 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1902 memset(skspcl->data_buf, 0, sgd->byte_count);
1903 break;
1904
1905 default:
1906 SKD_ASSERT("Don't know what to send");
1907 return;
1908
1909 }
1910 skd_send_special_fitmsg(skdev, skspcl);
1911}
1912
1913static void skd_refresh_device_data(struct skd_device *skdev)
1914{
1915 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1916
1917 skd_send_internal_skspcl(skdev, skspcl, TEST_UNIT_READY);
1918}
1919
1920static int skd_chk_read_buf(struct skd_device *skdev,
1921 struct skd_special_context *skspcl)
1922{
1923 unsigned char *buf = skspcl->data_buf;
1924 int i;
1925
1926 /* check for incrementing byte pattern */
1927 for (i = 0; i < WR_BUF_SIZE; i++)
1928 if (buf[i] != (i & 0xFF))
1929 return 1;
1930
1931 return 0;
1932}
1933
1934static void skd_log_check_status(struct skd_device *skdev, u8 status, u8 key,
1935 u8 code, u8 qual, u8 fruc)
1936{
1937 /* If the check condition is of special interest, log a message */
1938 if ((status == SAM_STAT_CHECK_CONDITION) && (key == 0x02)
1939 && (code == 0x04) && (qual == 0x06)) {
1940 pr_err("(%s): *** LOST_WRITE_DATA ERROR *** key/asc/"
1941 "ascq/fruc %02x/%02x/%02x/%02x\n",
1942 skd_name(skdev), key, code, qual, fruc);
1943 }
1944}
1945
1946static void skd_complete_internal(struct skd_device *skdev,
1947 volatile struct fit_completion_entry_v1
1948 *skcomp,
1949 volatile struct fit_comp_error_info *skerr,
1950 struct skd_special_context *skspcl)
1951{
1952 u8 *buf = skspcl->data_buf;
1953 u8 status;
1954 int i;
1955 struct skd_scsi_request *scsi =
1956 (struct skd_scsi_request *)&skspcl->msg_buf[64];
1957
1958 SKD_ASSERT(skspcl == &skdev->internal_skspcl);
1959
2e44b427 1960 pr_debug("%s:%s:%d complete internal %x\n",
1961 skdev->name, __func__, __LINE__, scsi->cdb[0]);
e67f86b3
AB		 1962
1963 skspcl->req.completion = *skcomp;
1964 skspcl->req.state = SKD_REQ_STATE_IDLE;
1965 skspcl->req.id += SKD_ID_INCR;
1966
1967 status = skspcl->req.completion.status;
1968
1969 skd_log_check_status(skdev, status, skerr->key, skerr->code,
1970 skerr->qual, skerr->fruc);
1971
1972 switch (scsi->cdb[0]) {
1973 case TEST_UNIT_READY:
1974 if (status == SAM_STAT_GOOD)
1975 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
1976 else if ((status == SAM_STAT_CHECK_CONDITION) &&
1977 (skerr->key == MEDIUM_ERROR))
1978 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
1979 else {
1980 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 1981 pr_debug("%s:%s:%d TUR failed, don't send anymore state 0x%x\n",
1982 skdev->name, __func__, __LINE__,
1983 skdev->state);
e67f86b3
AB		 1984 return;
1985 }
2e44b427 1986 pr_debug("%s:%s:%d **** TUR failed, retry skerr\n",
1987 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1988 skd_send_internal_skspcl(skdev, skspcl, 0x00);
1989 }
1990 break;
1991
1992 case WRITE_BUFFER:
1993 if (status == SAM_STAT_GOOD)
1994 skd_send_internal_skspcl(skdev, skspcl, READ_BUFFER);
1995 else {
1996 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 1997 pr_debug("%s:%s:%d write buffer failed, don't send anymore state 0x%x\n",
1998 skdev->name, __func__, __LINE__,
1999 skdev->state);
e67f86b3
AB		 2000 return;
2001 }
2e44b427 2002 pr_debug("%s:%s:%d **** write buffer failed, retry skerr\n",
2003 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2004 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2005 }
2006 break;
2007
2008 case READ_BUFFER:
2009 if (status == SAM_STAT_GOOD) {
2010 if (skd_chk_read_buf(skdev, skspcl) == 0)
2011 skd_send_internal_skspcl(skdev, skspcl,
2012 READ_CAPACITY);
2013 else {
2014 pr_err(
2015 "(%s):*** W/R Buffer mismatch %d ***\n",
2016 skd_name(skdev), skdev->connect_retries);
2017 if (skdev->connect_retries <
2018 SKD_MAX_CONNECT_RETRIES) {
2019 skdev->connect_retries++;
2020 skd_soft_reset(skdev);
2021 } else {
2022 pr_err(
2023 "(%s): W/R Buffer Connect Error\n",
2024 skd_name(skdev));
2025 return;
2026 }
2027 }
2028
2029 } else {
2030 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2031 pr_debug("%s:%s:%d "
2032 "read buffer failed, don't send anymore state 0x%x\n",
2033 skdev->name, __func__, __LINE__,
2034 skdev->state);
e67f86b3
AB		 2035 return;
2036 }
2e44b427 2037 pr_debug("%s:%s:%d "
2038 "**** read buffer failed, retry skerr\n",
2039 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2040 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2041 }
2042 break;
2043
2044 case READ_CAPACITY:
2045 skdev->read_cap_is_valid = 0;
2046 if (status == SAM_STAT_GOOD) {
2047 skdev->read_cap_last_lba =
2048 (buf[0] << 24) | (buf[1] << 16) |
2049 (buf[2] << 8) | buf[3];
2050 skdev->read_cap_blocksize =
2051 (buf[4] << 24) | (buf[5] << 16) |
2052 (buf[6] << 8) | buf[7];
2053
2e44b427 2054 pr_debug("%s:%s:%d last lba %d, bs %d\n",
2055 skdev->name, __func__, __LINE__,
2056 skdev->read_cap_last_lba,
2057 skdev->read_cap_blocksize);
e67f86b3
AB		 2058
2059 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2060
2061 skdev->read_cap_is_valid = 1;
2062
2063 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2064 } else if ((status == SAM_STAT_CHECK_CONDITION) &&
2065 (skerr->key == MEDIUM_ERROR)) {
2066 skdev->read_cap_last_lba = ~0;
2067 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2e44b427 2068 pr_debug("%s:%s:%d "
2069 "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n",
2070 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2071 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2072 } else {
2e44b427 2073 pr_debug("%s:%s:%d **** READCAP failed, retry TUR\n",
2074 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2075 skd_send_internal_skspcl(skdev, skspcl,
2076 TEST_UNIT_READY);
2077 }
2078 break;
2079
2080 case INQUIRY:
2081 skdev->inquiry_is_valid = 0;
2082 if (status == SAM_STAT_GOOD) {
2083 skdev->inquiry_is_valid = 1;
2084
2085 for (i = 0; i < 12; i++)
2086 skdev->inq_serial_num[i] = buf[i + 4];
2087 skdev->inq_serial_num[12] = 0;
2088 }
2089
2090 if (skd_unquiesce_dev(skdev) < 0)
2e44b427 2091 pr_debug("%s:%s:%d **** failed, to ONLINE device\n",
2092 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2093 /* connection is complete */
2094 skdev->connect_retries = 0;
2095 break;
2096
2097 case SYNCHRONIZE_CACHE:
2098 if (status == SAM_STAT_GOOD)
2099 skdev->sync_done = 1;
2100 else
2101 skdev->sync_done = -1;
2102 wake_up_interruptible(&skdev->waitq);
2103 break;
2104
2105 default:
2106 SKD_ASSERT("we didn't send this");
2107 }
2108}
2109
2110/*
2111 *****************************************************************************
2112 * FIT MESSAGES
2113 *****************************************************************************
2114 */
2115
2116static void skd_send_fitmsg(struct skd_device *skdev,
2117 struct skd_fitmsg_context *skmsg)
2118{
2119 u64 qcmd;
2120 struct fit_msg_hdr *fmh;
2121
2e44b427 2122 pr_debug("%s:%s:%d dma address 0x%llx, busy=%d\n",
2123 skdev->name, __func__, __LINE__,
2124 skmsg->mb_dma_address, skdev->in_flight);
2125 pr_debug("%s:%s:%d msg_buf 0x%p, offset %x\n",
2126 skdev->name, __func__, __LINE__,
2127 skmsg->msg_buf, skmsg->offset);
e67f86b3
AB		 2128
2129 qcmd = skmsg->mb_dma_address;
2130 qcmd |= FIT_QCMD_QID_NORMAL;
2131
2132 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
2133 skmsg->outstanding = fmh->num_protocol_cmds_coalesced;
2134
2135 if (unlikely(skdev->dbg_level > 1)) {
2136 u8 *bp = (u8 *)skmsg->msg_buf;
2137 int i;
2138 for (i = 0; i < skmsg->length; i += 8) {
8e1de26c
AS		 2139 pr_debug("%s:%s:%d msg[%2d] %8ph\n",
2140 skdev->name, __func__, __LINE__, i, &bp[i]);
e67f86b3
AB		 2141 if (i == 0)
2142 i = 64 - 8;
2143 }
2144 }
2145
2146 if (skmsg->length > 256)
2147 qcmd |= FIT_QCMD_MSGSIZE_512;
2148 else if (skmsg->length > 128)
2149 qcmd |= FIT_QCMD_MSGSIZE_256;
2150 else if (skmsg->length > 64)
2151 qcmd |= FIT_QCMD_MSGSIZE_128;
2152 else
2153 /*
2154 * This makes no sense because the FIT msg header is
2155 * 64 bytes. If the msg is only 64 bytes long it has
2156 * no payload.
2157 */
2158 qcmd |= FIT_QCMD_MSGSIZE_64;
2159
2160 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
e67f86b3
AB		 2161}
2162
2163static void skd_send_special_fitmsg(struct skd_device *skdev,
2164 struct skd_special_context *skspcl)
2165{
2166 u64 qcmd;
2167
2168 if (unlikely(skdev->dbg_level > 1)) {
2169 u8 *bp = (u8 *)skspcl->msg_buf;
2170 int i;
2171
2172 for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
8e1de26c
AS		 2173 pr_debug("%s:%s:%d spcl[%2d] %8ph\n",
2174 skdev->name, __func__, __LINE__, i, &bp[i]);
e67f86b3
AB		 2175 if (i == 0)
2176 i = 64 - 8;
2177 }
2178
2e44b427 2179 pr_debug("%s:%s:%d skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
2180 skdev->name, __func__, __LINE__,
2181 skspcl, skspcl->req.id, skspcl->req.sksg_list,
2182 skspcl->req.sksg_dma_address);
e67f86b3
AB		 2183 for (i = 0; i < skspcl->req.n_sg; i++) {
2184 struct fit_sg_descriptor *sgd =
2185 &skspcl->req.sksg_list[i];
2186
2e44b427 2187 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
2188 "addr=0x%llx next=0x%llx\n",
2189 skdev->name, __func__, __LINE__,
2190 i, sgd->byte_count, sgd->control,
2191 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 2192 }
2193 }
2194
2195 /*
2196 * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
2197 * and one 64-byte SSDI command.
2198 */
2199 qcmd = skspcl->mb_dma_address;
2200 qcmd |= FIT_QCMD_QID_NORMAL + FIT_QCMD_MSGSIZE_128;
2201
2202 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2203}
2204
2205/*
2206 *****************************************************************************
2207 * COMPLETION QUEUE
2208 *****************************************************************************
2209 */
2210
2211static void skd_complete_other(struct skd_device *skdev,
2212 volatile struct fit_completion_entry_v1 *skcomp,
2213 volatile struct fit_comp_error_info *skerr);
2214
e67f86b3
AB		 2215struct sns_info {
2216 u8 type;
2217 u8 stat;
2218 u8 key;
2219 u8 asc;
2220 u8 ascq;
2221 u8 mask;
2222 enum skd_check_status_action action;
2223};
2224
2225static struct sns_info skd_chkstat_table[] = {
2226 /* Good */
2227 { 0x70, 0x02, RECOVERED_ERROR, 0, 0, 0x1c,
2228 SKD_CHECK_STATUS_REPORT_GOOD },
2229
2230 /* Smart alerts */
2231 { 0x70, 0x02, NO_SENSE, 0x0B, 0x00, 0x1E, /* warnings */
2232 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2233 { 0x70, 0x02, NO_SENSE, 0x5D, 0x00, 0x1E, /* thresholds */
2234 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2235 { 0x70, 0x02, RECOVERED_ERROR, 0x0B, 0x01, 0x1F, /* temperature over trigger */
2236 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2237
2238 /* Retry (with limits) */
2239 { 0x70, 0x02, 0x0B, 0, 0, 0x1C, /* This one is for DMA ERROR */
2240 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2241 { 0x70, 0x02, 0x06, 0x0B, 0x00, 0x1E, /* warnings */
2242 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2243 { 0x70, 0x02, 0x06, 0x5D, 0x00, 0x1E, /* thresholds */
2244 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2245 { 0x70, 0x02, 0x06, 0x80, 0x30, 0x1F, /* backup power */
2246 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2247
2248 /* Busy (or about to be) */
2249 { 0x70, 0x02, 0x06, 0x3f, 0x01, 0x1F, /* fw changed */
2250 SKD_CHECK_STATUS_BUSY_IMMINENT },
2251};
2252
2253/*
2254 * Look up status and sense data to decide how to handle the error
2255 * from the device.
2256 * mask says which fields must match e.g., mask=0x18 means check
2257 * type and stat, ignore key, asc, ascq.
2258 */
2259
38d4a1bb
MS		 2260static enum skd_check_status_action
2261skd_check_status(struct skd_device *skdev,
2262 u8 cmp_status, volatile struct fit_comp_error_info *skerr)
e67f86b3
AB		 2263{
2264 int i, n;
2265
2266 pr_err("(%s): key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
2267 skd_name(skdev), skerr->key, skerr->code, skerr->qual,
2268 skerr->fruc);
2269
2e44b427 2270 pr_debug("%s:%s:%d stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
2271 skdev->name, __func__, __LINE__, skerr->type, cmp_status,
2272 skerr->key, skerr->code, skerr->qual, skerr->fruc);
e67f86b3
AB		 2273
2274 /* Does the info match an entry in the good category? */
2275 n = sizeof(skd_chkstat_table) / sizeof(skd_chkstat_table[0]);
2276 for (i = 0; i < n; i++) {
2277 struct sns_info *sns = &skd_chkstat_table[i];
2278
2279 if (sns->mask & 0x10)
2280 if (skerr->type != sns->type)
2281 continue;
2282
2283 if (sns->mask & 0x08)
2284 if (cmp_status != sns->stat)
2285 continue;
2286
2287 if (sns->mask & 0x04)
2288 if (skerr->key != sns->key)
2289 continue;
2290
2291 if (sns->mask & 0x02)
2292 if (skerr->code != sns->asc)
2293 continue;
2294
2295 if (sns->mask & 0x01)
2296 if (skerr->qual != sns->ascq)
2297 continue;
2298
2299 if (sns->action == SKD_CHECK_STATUS_REPORT_SMART_ALERT) {
2300 pr_err("(%s): SMART Alert: sense key/asc/ascq "
2301 "%02x/%02x/%02x\n",
2302 skd_name(skdev), skerr->key,
2303 skerr->code, skerr->qual);
2304 }
2305 return sns->action;
2306 }
2307
2308 /* No other match, so nonzero status means error,
2309 * zero status means good
2310 */
2311 if (cmp_status) {
2e44b427 2312 pr_debug("%s:%s:%d status check: error\n",
2313 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2314 return SKD_CHECK_STATUS_REPORT_ERROR;
2315 }
2316
2e44b427 2317 pr_debug("%s:%s:%d status check good default\n",
2318 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2319 return SKD_CHECK_STATUS_REPORT_GOOD;
2320}
2321
2322static void skd_resolve_req_exception(struct skd_device *skdev,
2323 struct skd_request_context *skreq)
2324{
2325 u8 cmp_status = skreq->completion.status;
2326
2327 switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {
2328 case SKD_CHECK_STATUS_REPORT_GOOD:
2329 case SKD_CHECK_STATUS_REPORT_SMART_ALERT:
2330 skd_end_request(skdev, skreq, 0);
2331 break;
2332
2333 case SKD_CHECK_STATUS_BUSY_IMMINENT:
2334 skd_log_skreq(skdev, skreq, "retry(busy)");
38d4a1bb 2335 blk_requeue_request(skdev->queue, skreq->req);
e67f86b3
AB		 2336 pr_info("(%s) drive BUSY imminent\n", skd_name(skdev));
2337 skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT;
2338 skdev->timer_countdown = SKD_TIMER_MINUTES(20);
2339 skd_quiesce_dev(skdev);
2340 break;
2341
2342 case SKD_CHECK_STATUS_REQUEUE_REQUEST:
fcd37eb3
JA		 2343 if ((unsigned long) ++skreq->req->special < SKD_MAX_RETRIES) {
2344 skd_log_skreq(skdev, skreq, "retry");
38d4a1bb 2345 blk_requeue_request(skdev->queue, skreq->req);
fcd37eb3 2346 break;
e67f86b3
AB		 2347 }
2348 /* fall through to report error */
2349
2350 case SKD_CHECK_STATUS_REPORT_ERROR:
2351 default:
2352 skd_end_request(skdev, skreq, -EIO);
2353 break;
2354 }
2355}
2356
e67f86b3
AB		 2357/* assume spinlock is already held */
2358static void skd_release_skreq(struct skd_device *skdev,
2359 struct skd_request_context *skreq)
2360{
2361 u32 msg_slot;
2362 struct skd_fitmsg_context *skmsg;
2363
2364 u32 timo_slot;
2365
2366 /*
2367 * Reclaim the FIT msg buffer if this is
2368 * the first of the requests it carried to
2369 * be completed. The FIT msg buffer used to
2370 * send this request cannot be reused until
2371 * we are sure the s1120 card has copied
2372 * it to its memory. The FIT msg might have
2373 * contained several requests. As soon as
2374 * any of them are completed we know that
2375 * the entire FIT msg was transferred.
2376 * Only the first completed request will
2377 * match the FIT msg buffer id. The FIT
2378 * msg buffer id is immediately updated.
2379 * When subsequent requests complete the FIT
2380 * msg buffer id won't match, so we know
2381 * quite cheaply that it is already done.
2382 */
2383 msg_slot = skreq->fitmsg_id & SKD_ID_SLOT_MASK;
2384 SKD_ASSERT(msg_slot < skdev->num_fitmsg_context);
2385
2386 skmsg = &skdev->skmsg_table[msg_slot];
2387 if (skmsg->id == skreq->fitmsg_id) {
2388 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_BUSY);
2389 SKD_ASSERT(skmsg->outstanding > 0);
2390 skmsg->outstanding--;
2391 if (skmsg->outstanding == 0) {
2392 skmsg->state = SKD_MSG_STATE_IDLE;
2393 skmsg->id += SKD_ID_INCR;
2394 skmsg->next = skdev->skmsg_free_list;
2395 skdev->skmsg_free_list = skmsg;
2396 }
2397 }
2398
2399 /*
2400 * Decrease the number of active requests.
2401 * Also decrements the count in the timeout slot.
2402 */
2403 SKD_ASSERT(skdev->in_flight > 0);
2404 skdev->in_flight -= 1;
2405
2406 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
2407 SKD_ASSERT(skdev->timeout_slot[timo_slot] > 0);
2408 skdev->timeout_slot[timo_slot] -= 1;
2409
2410 /*
2411 * Reset backpointer
2412 */
fcd37eb3 2413 skreq->req = NULL;
e67f86b3
AB		 2414
2415 /*
2416 * Reclaim the skd_request_context
2417 */
2418 skreq->state = SKD_REQ_STATE_IDLE;
2419 skreq->id += SKD_ID_INCR;
2420 skreq->next = skdev->skreq_free_list;
2421 skdev->skreq_free_list = skreq;
2422}
2423
2424#define DRIVER_INQ_EVPD_PAGE_CODE 0xDA
2425
2426static void skd_do_inq_page_00(struct skd_device *skdev,
2427 volatile struct fit_completion_entry_v1 *skcomp,
2428 volatile struct fit_comp_error_info *skerr,
2429 uint8_t *cdb, uint8_t *buf)
2430{
2431 uint16_t insert_pt, max_bytes, drive_pages, drive_bytes, new_size;
2432
2433 /* Caller requested "supported pages". The driver needs to insert
2434 * its page.
2435 */
2e44b427 2436 pr_debug("%s:%s:%d skd_do_driver_inquiry: modify supported pages.\n",
2437 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2438
2439 /* If the device rejected the request because the CDB was
2440 * improperly formed, then just leave.
2441 */
2442 if (skcomp->status == SAM_STAT_CHECK_CONDITION &&
2443 skerr->key == ILLEGAL_REQUEST && skerr->code == 0x24)
2444 return;
2445
2446 /* Get the amount of space the caller allocated */
2447 max_bytes = (cdb[3] << 8) | cdb[4];
2448
2449 /* Get the number of pages actually returned by the device */
2450 drive_pages = (buf[2] << 8) | buf[3];
2451 drive_bytes = drive_pages + 4;
2452 new_size = drive_pages + 1;
2453
2454 /* Supported pages must be in numerical order, so find where
2455 * the driver page needs to be inserted into the list of
2456 * pages returned by the device.
2457 */
2458 for (insert_pt = 4; insert_pt < drive_bytes; insert_pt++) {
2459 if (buf[insert_pt] == DRIVER_INQ_EVPD_PAGE_CODE)
2460 return; /* Device using this page code. abort */
2461 else if (buf[insert_pt] > DRIVER_INQ_EVPD_PAGE_CODE)
2462 break;
2463 }
2464
2465 if (insert_pt < max_bytes) {
2466 uint16_t u;
2467
2468 /* Shift everything up one byte to make room. */
2469 for (u = new_size + 3; u > insert_pt; u--)
2470 buf[u] = buf[u - 1];
2471 buf[insert_pt] = DRIVER_INQ_EVPD_PAGE_CODE;
2472
2473 /* SCSI byte order increment of num_returned_bytes by 1 */
2474 skcomp->num_returned_bytes =
2475 be32_to_cpu(skcomp->num_returned_bytes) + 1;
2476 skcomp->num_returned_bytes =
2477 be32_to_cpu(skcomp->num_returned_bytes);
2478 }
2479
2480 /* update page length field to reflect the driver's page too */
2481 buf[2] = (uint8_t)((new_size >> 8) & 0xFF);
2482 buf[3] = (uint8_t)((new_size >> 0) & 0xFF);
2483}
2484
2485static void skd_get_link_info(struct pci_dev *pdev, u8 *speed, u8 *width)
2486{
2487 int pcie_reg;
2488 u16 pci_bus_speed;
2489 u8 pci_lanes;
2490
2491 pcie_reg = pci_find_capability(pdev, PCI_CAP_ID_EXP);
2492 if (pcie_reg) {
2493 u16 linksta;
2494 pci_read_config_word(pdev, pcie_reg + PCI_EXP_LNKSTA, &linksta);
2495
2496 pci_bus_speed = linksta & 0xF;
2497 pci_lanes = (linksta & 0x3F0) >> 4;
2498 } else {
2499 *speed = STEC_LINK_UNKNOWN;
2500 *width = 0xFF;
2501 return;
2502 }
2503
2504 switch (pci_bus_speed) {
2505 case 1:
2506 *speed = STEC_LINK_2_5GTS;
2507 break;
2508 case 2:
2509 *speed = STEC_LINK_5GTS;
2510 break;
2511 case 3:
2512 *speed = STEC_LINK_8GTS;
2513 break;
2514 default:
2515 *speed = STEC_LINK_UNKNOWN;
2516 break;
2517 }
2518
2519 if (pci_lanes <= 0x20)
2520 *width = pci_lanes;
2521 else
2522 *width = 0xFF;
2523}
2524
2525static void skd_do_inq_page_da(struct skd_device *skdev,
2526 volatile struct fit_completion_entry_v1 *skcomp,
2527 volatile struct fit_comp_error_info *skerr,
2528 uint8_t *cdb, uint8_t *buf)
2529{
fec23f63 2530 struct pci_dev *pdev = skdev->pdev;
e67f86b3
AB		 2531 unsigned max_bytes;
2532 struct driver_inquiry_data inq;
2533 u16 val;
2534
2e44b427 2535 pr_debug("%s:%s:%d skd_do_driver_inquiry: return driver page\n",
2536 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2537
2538 memset(&inq, 0, sizeof(inq));
2539
2540 inq.page_code = DRIVER_INQ_EVPD_PAGE_CODE;
2541
fec23f63
BZ		 2542 skd_get_link_info(pdev, &inq.pcie_link_speed, &inq.pcie_link_lanes);
2543 inq.pcie_bus_number = cpu_to_be16(pdev->bus->number);
2544 inq.pcie_device_number = PCI_SLOT(pdev->devfn);
2545 inq.pcie_function_number = PCI_FUNC(pdev->devfn);
e67f86b3 2546
fec23f63
BZ		 2547 pci_read_config_word(pdev, PCI_VENDOR_ID, &val);
2548 inq.pcie_vendor_id = cpu_to_be16(val);
e67f86b3 2549
fec23f63
BZ		 2550 pci_read_config_word(pdev, PCI_DEVICE_ID, &val);
2551 inq.pcie_device_id = cpu_to_be16(val);
e67f86b3 2552
fec23f63
BZ		 2553 pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &val);
2554 inq.pcie_subsystem_vendor_id = cpu_to_be16(val);
e67f86b3 2555
fec23f63
BZ		 2556 pci_read_config_word(pdev, PCI_SUBSYSTEM_ID, &val);
2557 inq.pcie_subsystem_device_id = cpu_to_be16(val);
e67f86b3
AB		 2558
2559 /* Driver version, fixed lenth, padded with spaces on the right */
2560 inq.driver_version_length = sizeof(inq.driver_version);
2561 memset(&inq.driver_version, ' ', sizeof(inq.driver_version));
2562 memcpy(inq.driver_version, DRV_VER_COMPL,
2563 min(sizeof(inq.driver_version), strlen(DRV_VER_COMPL)));
2564
2565 inq.page_length = cpu_to_be16((sizeof(inq) - 4));
2566
2567 /* Clear the error set by the device */
2568 skcomp->status = SAM_STAT_GOOD;
2569 memset((void *)skerr, 0, sizeof(*skerr));
2570
2571 /* copy response into output buffer */
2572 max_bytes = (cdb[3] << 8) | cdb[4];
2573 memcpy(buf, &inq, min_t(unsigned, max_bytes, sizeof(inq)));
2574
2575 skcomp->num_returned_bytes =
2576 be32_to_cpu(min_t(uint16_t, max_bytes, sizeof(inq)));
2577}
2578
2579static void skd_do_driver_inq(struct skd_device *skdev,
2580 volatile struct fit_completion_entry_v1 *skcomp,
2581 volatile struct fit_comp_error_info *skerr,
2582 uint8_t *cdb, uint8_t *buf)
2583{
2584 if (!buf)
2585 return;
2586 else if (cdb[0] != INQUIRY)
2587 return; /* Not an INQUIRY */
2588 else if ((cdb[1] & 1) == 0)
2589 return; /* EVPD not set */
2590 else if (cdb[2] == 0)
2591 /* Need to add driver's page to supported pages list */
2592 skd_do_inq_page_00(skdev, skcomp, skerr, cdb, buf);
2593 else if (cdb[2] == DRIVER_INQ_EVPD_PAGE_CODE)
2594 /* Caller requested driver's page */
2595 skd_do_inq_page_da(skdev, skcomp, skerr, cdb, buf);
2596}
2597
2598static unsigned char *skd_sg_1st_page_ptr(struct scatterlist *sg)
2599{
2600 if (!sg)
2601 return NULL;
2602 if (!sg_page(sg))
2603 return NULL;
2604 return sg_virt(sg);
2605}
2606
2607static void skd_process_scsi_inq(struct skd_device *skdev,
2608 volatile struct fit_completion_entry_v1
2609 *skcomp,
2610 volatile struct fit_comp_error_info *skerr,
2611 struct skd_special_context *skspcl)
2612{
2613 uint8_t *buf;
2614 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
2615 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
2616
2617 dma_sync_sg_for_cpu(skdev->class_dev, skspcl->req.sg, skspcl->req.n_sg,
2618 skspcl->req.sg_data_dir);
2619 buf = skd_sg_1st_page_ptr(skspcl->req.sg);
2620
2621 if (buf)
2622 skd_do_driver_inq(skdev, skcomp, skerr, scsi_req->cdb, buf);
2623}
2624
2625
2626static int skd_isr_completion_posted(struct skd_device *skdev,
2627 int limit, int *enqueued)
2628{
2629 volatile struct fit_completion_entry_v1 *skcmp = NULL;
2630 volatile struct fit_comp_error_info *skerr;
2631 u16 req_id;
2632 u32 req_slot;
2633 struct skd_request_context *skreq;
2634 u16 cmp_cntxt = 0;
2635 u8 cmp_status = 0;
2636 u8 cmp_cycle = 0;
2637 u32 cmp_bytes = 0;
2638 int rc = 0;
2639 int processed = 0;
e67f86b3
AB		 2640
2641 for (;; ) {
2642 SKD_ASSERT(skdev->skcomp_ix < SKD_N_COMPLETION_ENTRY);
2643
2644 skcmp = &skdev->skcomp_table[skdev->skcomp_ix];
2645 cmp_cycle = skcmp->cycle;
2646 cmp_cntxt = skcmp->tag;
2647 cmp_status = skcmp->status;
2648 cmp_bytes = be32_to_cpu(skcmp->num_returned_bytes);
2649
2650 skerr = &skdev->skerr_table[skdev->skcomp_ix];
2651
2e44b427 2652 pr_debug("%s:%s:%d "
2653 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d "
2654 "busy=%d rbytes=0x%x proto=%d\n",
2655 skdev->name, __func__, __LINE__, skdev->skcomp_cycle,
2656 skdev->skcomp_ix, cmp_cycle, cmp_cntxt, cmp_status,
2657 skdev->in_flight, cmp_bytes, skdev->proto_ver);
e67f86b3
AB		 2658
2659 if (cmp_cycle != skdev->skcomp_cycle) {
2e44b427 2660 pr_debug("%s:%s:%d end of completions\n",
2661 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2662 break;
2663 }
2664 /*
2665 * Update the completion queue head index and possibly
2666 * the completion cycle count. 8-bit wrap-around.
2667 */
2668 skdev->skcomp_ix++;
2669 if (skdev->skcomp_ix >= SKD_N_COMPLETION_ENTRY) {
2670 skdev->skcomp_ix = 0;
2671 skdev->skcomp_cycle++;
2672 }
2673
2674 /*
2675 * The command context is a unique 32-bit ID. The low order
2676 * bits help locate the request. The request is usually a
2677 * r/w request (see skd_start() above) or a special request.
2678 */
2679 req_id = cmp_cntxt;
2680 req_slot = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
2681
2682 /* Is this other than a r/w request? */
2683 if (req_slot >= skdev->num_req_context) {
2684 /*
2685 * This is not a completion for a r/w request.
2686 */
2687 skd_complete_other(skdev, skcmp, skerr);
2688 continue;
2689 }
2690
2691 skreq = &skdev->skreq_table[req_slot];
2692
2693 /*
2694 * Make sure the request ID for the slot matches.
2695 */
2696 if (skreq->id != req_id) {
2e44b427 2697 pr_debug("%s:%s:%d mismatch comp_id=0x%x req_id=0x%x\n",
2698 skdev->name, __func__, __LINE__,
2699 req_id, skreq->id);
e67f86b3
AB		 2700 {
2701 u16 new_id = cmp_cntxt;
2702 pr_err("(%s): Completion mismatch "
2703 "comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
2704 skd_name(skdev), req_id,
2705 skreq->id, new_id);
2706
2707 continue;
2708 }
2709 }
2710
2711 SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);
2712
2713 if (skreq->state == SKD_REQ_STATE_ABORTED) {
2e44b427 2714 pr_debug("%s:%s:%d reclaim req %p id=%04x\n",
2715 skdev->name, __func__, __LINE__,
2716 skreq, skreq->id);
e67f86b3
AB		 2717 /* a previously timed out command can
2718 * now be cleaned up */
2719 skd_release_skreq(skdev, skreq);
2720 continue;
2721 }
2722
2723 skreq->completion = *skcmp;
2724 if (unlikely(cmp_status == SAM_STAT_CHECK_CONDITION)) {
2725 skreq->err_info = *skerr;
2726 skd_log_check_status(skdev, cmp_status, skerr->key,
2727 skerr->code, skerr->qual,
2728 skerr->fruc);
2729 }
2730 /* Release DMA resources for the request. */
2731 if (skreq->n_sg > 0)
2732 skd_postop_sg_list(skdev, skreq);
2733
fcd37eb3 2734 if (!skreq->req) {
2e44b427 2735 pr_debug("%s:%s:%d NULL backptr skdreq %p, "
2736 "req=0x%x req_id=0x%x\n",
2737 skdev->name, __func__, __LINE__,
2738 skreq, skreq->id, req_id);
e67f86b3
AB		 2739 } else {
2740 /*
2741 * Capture the outcome and post it back to the
2742 * native request.
2743 */
fcd37eb3
JA		 2744 if (likely(cmp_status == SAM_STAT_GOOD))
2745 skd_end_request(skdev, skreq, 0);
2746 else
e67f86b3 2747 skd_resolve_req_exception(skdev, skreq);
e67f86b3
AB		 2748 }
2749
2750 /*
2751 * Release the skreq, its FIT msg (if one), timeout slot,
2752 * and queue depth.
2753 */
2754 skd_release_skreq(skdev, skreq);
2755
2756 /* skd_isr_comp_limit equal zero means no limit */
2757 if (limit) {
2758 if (++processed >= limit) {
2759 rc = 1;
2760 break;
2761 }
2762 }
2763 }
2764
2765 if ((skdev->state == SKD_DRVR_STATE_PAUSING)
2766 && (skdev->in_flight) == 0) {
2767 skdev->state = SKD_DRVR_STATE_PAUSED;
2768 wake_up_interruptible(&skdev->waitq);
2769 }
2770
2771 return rc;
2772}
2773
2774static void skd_complete_other(struct skd_device *skdev,
2775 volatile struct fit_completion_entry_v1 *skcomp,
2776 volatile struct fit_comp_error_info *skerr)
2777{
2778 u32 req_id = 0;
2779 u32 req_table;
2780 u32 req_slot;
2781 struct skd_special_context *skspcl;
2782
2783 req_id = skcomp->tag;
2784 req_table = req_id & SKD_ID_TABLE_MASK;
2785 req_slot = req_id & SKD_ID_SLOT_MASK;
2786
2e44b427 2787 pr_debug("%s:%s:%d table=0x%x id=0x%x slot=%d\n",
2788 skdev->name, __func__, __LINE__,
2789 req_table, req_id, req_slot);
e67f86b3
AB		 2790
2791 /*
2792 * Based on the request id, determine how to dispatch this completion.
2793 * This swich/case is finding the good cases and forwarding the
2794 * completion entry. Errors are reported below the switch.
2795 */
2796 switch (req_table) {
2797 case SKD_ID_RW_REQUEST:
2798 /*
2799 * The caller, skd_completion_posted_isr() above,
2800 * handles r/w requests. The only way we get here
2801 * is if the req_slot is out of bounds.
2802 */
2803 break;
2804
2805 case SKD_ID_SPECIAL_REQUEST:
2806 /*
2807 * Make sure the req_slot is in bounds and that the id
2808 * matches.
2809 */
2810 if (req_slot < skdev->n_special) {
2811 skspcl = &skdev->skspcl_table[req_slot];
2812 if (skspcl->req.id == req_id &&
2813 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2814 skd_complete_special(skdev,
2815 skcomp, skerr, skspcl);
2816 return;
2817 }
2818 }
2819 break;
2820
2821 case SKD_ID_INTERNAL:
2822 if (req_slot == 0) {
2823 skspcl = &skdev->internal_skspcl;
2824 if (skspcl->req.id == req_id &&
2825 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2826 skd_complete_internal(skdev,
2827 skcomp, skerr, skspcl);
2828 return;
2829 }
2830 }
2831 break;
2832
2833 case SKD_ID_FIT_MSG:
2834 /*
2835 * These id's should never appear in a completion record.
2836 */
2837 break;
2838
2839 default:
2840 /*
2841 * These id's should never appear anywhere;
2842 */
2843 break;
2844 }
2845
2846 /*
2847 * If we get here it is a bad or stale id.
2848 */
2849}
2850
2851static void skd_complete_special(struct skd_device *skdev,
2852 volatile struct fit_completion_entry_v1
2853 *skcomp,
2854 volatile struct fit_comp_error_info *skerr,
2855 struct skd_special_context *skspcl)
2856{
2e44b427 2857 pr_debug("%s:%s:%d completing special request %p\n",
2858 skdev->name, __func__, __LINE__, skspcl);
e67f86b3
AB		 2859 if (skspcl->orphaned) {
2860 /* Discard orphaned request */
2861 /* ?: Can this release directly or does it need
2862 * to use a worker? */
2e44b427 2863 pr_debug("%s:%s:%d release orphaned %p\n",
2864 skdev->name, __func__, __LINE__, skspcl);
e67f86b3
AB		 2865 skd_release_special(skdev, skspcl);
2866 return;
2867 }
2868
2869 skd_process_scsi_inq(skdev, skcomp, skerr, skspcl);
2870
2871 skspcl->req.state = SKD_REQ_STATE_COMPLETED;
2872 skspcl->req.completion = *skcomp;
2873 skspcl->req.err_info = *skerr;
2874
2875 skd_log_check_status(skdev, skspcl->req.completion.status, skerr->key,
2876 skerr->code, skerr->qual, skerr->fruc);
2877
2878 wake_up_interruptible(&skdev->waitq);
2879}
2880
2881/* assume spinlock is already held */
2882static void skd_release_special(struct skd_device *skdev,
2883 struct skd_special_context *skspcl)
2884{
2885 int i, was_depleted;
2886
2887 for (i = 0; i < skspcl->req.n_sg; i++) {
e67f86b3
AB		 2888 struct page *page = sg_page(&skspcl->req.sg[i]);
2889 __free_page(page);
2890 }
2891
2892 was_depleted = (skdev->skspcl_free_list == NULL);
2893
2894 skspcl->req.state = SKD_REQ_STATE_IDLE;
2895 skspcl->req.id += SKD_ID_INCR;
2896 skspcl->req.next =
2897 (struct skd_request_context *)skdev->skspcl_free_list;
2898 skdev->skspcl_free_list = (struct skd_special_context *)skspcl;
2899
2900 if (was_depleted) {
2e44b427 2901 pr_debug("%s:%s:%d skspcl was depleted\n",
2902 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2903 /* Free list was depleted. Their might be waiters. */
2904 wake_up_interruptible(&skdev->waitq);
2905 }
2906}
2907
2908static void skd_reset_skcomp(struct skd_device *skdev)
2909{
2910 u32 nbytes;
2911 struct fit_completion_entry_v1 *skcomp;
2912
2913 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
2914 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
2915
2916 memset(skdev->skcomp_table, 0, nbytes);
2917
2918 skdev->skcomp_ix = 0;
2919 skdev->skcomp_cycle = 1;
2920}
2921
2922/*
2923 *****************************************************************************
2924 * INTERRUPTS
2925 *****************************************************************************
2926 */
2927static void skd_completion_worker(struct work_struct *work)
2928{
2929 struct skd_device *skdev =
2930 container_of(work, struct skd_device, completion_worker);
2931 unsigned long flags;
2932 int flush_enqueued = 0;
2933
2934 spin_lock_irqsave(&skdev->lock, flags);
2935
2936 /*
2937 * pass in limit=0, which means no limit..
2938 * process everything in compq
2939 */
2940 skd_isr_completion_posted(skdev, 0, &flush_enqueued);
2941 skd_request_fn(skdev->queue);
2942
2943 spin_unlock_irqrestore(&skdev->lock, flags);
2944}
2945
2946static void skd_isr_msg_from_dev(struct skd_device *skdev);
2947
2948irqreturn_t
2949static skd_isr(int irq, void *ptr)
2950{
2951 struct skd_device *skdev;
2952 u32 intstat;
2953 u32 ack;
2954 int rc = 0;
2955 int deferred = 0;
2956 int flush_enqueued = 0;
2957
2958 skdev = (struct skd_device *)ptr;
2959 spin_lock(&skdev->lock);
2960
2961 for (;; ) {
2962 intstat = SKD_READL(skdev, FIT_INT_STATUS_HOST);
2963
2964 ack = FIT_INT_DEF_MASK;
2965 ack &= intstat;
2966
2e44b427 2967 pr_debug("%s:%s:%d intstat=0x%x ack=0x%x\n",
2968 skdev->name, __func__, __LINE__, intstat, ack);
e67f86b3
AB		 2969
2970 /* As long as there is an int pending on device, keep
2971 * running loop. When none, get out, but if we've never
2972 * done any processing, call completion handler?
2973 */
2974 if (ack == 0) {
2975 /* No interrupts on device, but run the completion
2976 * processor anyway?
2977 */
2978 if (rc == 0)
2979 if (likely (skdev->state
2980 == SKD_DRVR_STATE_ONLINE))
2981 deferred = 1;
2982 break;
2983 }
2984
2985 rc = IRQ_HANDLED;
2986
2987 SKD_WRITEL(skdev, ack, FIT_INT_STATUS_HOST);
2988
2989 if (likely((skdev->state != SKD_DRVR_STATE_LOAD) &&
2990 (skdev->state != SKD_DRVR_STATE_STOPPING))) {
2991 if (intstat & FIT_ISH_COMPLETION_POSTED) {
2992 /*
2993 * If we have already deferred completion
2994 * processing, don't bother running it again
2995 */
2996 if (deferred == 0)
2997 deferred =
2998 skd_isr_completion_posted(skdev,
2999 skd_isr_comp_limit, &flush_enqueued);
3000 }
3001
3002 if (intstat & FIT_ISH_FW_STATE_CHANGE) {
3003 skd_isr_fwstate(skdev);
3004 if (skdev->state == SKD_DRVR_STATE_FAULT ||
3005 skdev->state ==
3006 SKD_DRVR_STATE_DISAPPEARED) {
3007 spin_unlock(&skdev->lock);
3008 return rc;
3009 }
3010 }
3011
3012 if (intstat & FIT_ISH_MSG_FROM_DEV)
3013 skd_isr_msg_from_dev(skdev);
3014 }
3015 }
3016
3017 if (unlikely(flush_enqueued))
3018 skd_request_fn(skdev->queue);
3019
3020 if (deferred)
3021 schedule_work(&skdev->completion_worker);
3022 else if (!flush_enqueued)
3023 skd_request_fn(skdev->queue);
3024
3025 spin_unlock(&skdev->lock);
3026
3027 return rc;
3028}
3029
e67f86b3
AB		 3030static void skd_drive_fault(struct skd_device *skdev)
3031{
3032 skdev->state = SKD_DRVR_STATE_FAULT;
3033 pr_err("(%s): Drive FAULT\n", skd_name(skdev));
3034}
3035
3036static void skd_drive_disappeared(struct skd_device *skdev)
3037{
3038 skdev->state = SKD_DRVR_STATE_DISAPPEARED;
3039 pr_err("(%s): Drive DISAPPEARED\n", skd_name(skdev));
3040}
3041
3042static void skd_isr_fwstate(struct skd_device *skdev)
3043{
3044 u32 sense;
3045 u32 state;
3046 u32 mtd;
3047 int prev_driver_state = skdev->state;
3048
3049 sense = SKD_READL(skdev, FIT_STATUS);
3050 state = sense & FIT_SR_DRIVE_STATE_MASK;
3051
3052 pr_err("(%s): s1120 state %s(%d)=>%s(%d)\n",
3053 skd_name(skdev),
3054 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
3055 skd_drive_state_to_str(state), state);
3056
3057 skdev->drive_state = state;
3058
3059 switch (skdev->drive_state) {
3060 case FIT_SR_DRIVE_INIT:
3061 if (skdev->state == SKD_DRVR_STATE_PROTOCOL_MISMATCH) {
3062 skd_disable_interrupts(skdev);
3063 break;
3064 }
3065 if (skdev->state == SKD_DRVR_STATE_RESTARTING)
3066 skd_recover_requests(skdev, 0);
3067 if (skdev->state == SKD_DRVR_STATE_WAIT_BOOT) {
3068 skdev->timer_countdown = SKD_STARTING_TIMO;
3069 skdev->state = SKD_DRVR_STATE_STARTING;
3070 skd_soft_reset(skdev);
3071 break;
3072 }
3073 mtd = FIT_MXD_CONS(FIT_MTD_FITFW_INIT, 0, 0);
3074 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3075 skdev->last_mtd = mtd;
3076 break;
3077
3078 case FIT_SR_DRIVE_ONLINE:
3079 skdev->cur_max_queue_depth = skd_max_queue_depth;
3080 if (skdev->cur_max_queue_depth > skdev->dev_max_queue_depth)
3081 skdev->cur_max_queue_depth = skdev->dev_max_queue_depth;
3082
3083 skdev->queue_low_water_mark =
3084 skdev->cur_max_queue_depth * 2 / 3 + 1;
3085 if (skdev->queue_low_water_mark < 1)
3086 skdev->queue_low_water_mark = 1;
3087 pr_info(
3088 "(%s): Queue depth limit=%d dev=%d lowat=%d\n",
3089 skd_name(skdev),
3090 skdev->cur_max_queue_depth,
3091 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
3092
3093 skd_refresh_device_data(skdev);
3094 break;
3095
3096 case FIT_SR_DRIVE_BUSY:
3097 skdev->state = SKD_DRVR_STATE_BUSY;
3098 skdev->timer_countdown = SKD_BUSY_TIMO;
3099 skd_quiesce_dev(skdev);
3100 break;
3101 case FIT_SR_DRIVE_BUSY_SANITIZE:
3102 /* set timer for 3 seconds, we'll abort any unfinished
3103 * commands after that expires
3104 */
3105 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3106 skdev->timer_countdown = SKD_TIMER_SECONDS(3);
6a5ec65b 3107 blk_start_queue(skdev->queue);
e67f86b3
AB		 3108 break;
3109 case FIT_SR_DRIVE_BUSY_ERASE:
3110 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3111 skdev->timer_countdown = SKD_BUSY_TIMO;
3112 break;
3113 case FIT_SR_DRIVE_OFFLINE:
3114 skdev->state = SKD_DRVR_STATE_IDLE;
3115 break;
3116 case FIT_SR_DRIVE_SOFT_RESET:
3117 switch (skdev->state) {
3118 case SKD_DRVR_STATE_STARTING:
3119 case SKD_DRVR_STATE_RESTARTING:
3120 /* Expected by a caller of skd_soft_reset() */
3121 break;
3122 default:
3123 skdev->state = SKD_DRVR_STATE_RESTARTING;
3124 break;
3125 }
3126 break;
3127 case FIT_SR_DRIVE_FW_BOOTING:
2e44b427 3128 pr_debug("%s:%s:%d ISR FIT_SR_DRIVE_FW_BOOTING %s\n",
3129 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3130 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3131 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3132 break;
3133
3134 case FIT_SR_DRIVE_DEGRADED:
3135 case FIT_SR_PCIE_LINK_DOWN:
3136 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
3137 break;
3138
3139 case FIT_SR_DRIVE_FAULT:
3140 skd_drive_fault(skdev);
3141 skd_recover_requests(skdev, 0);
6a5ec65b 3142 blk_start_queue(skdev->queue);
e67f86b3
AB		 3143 break;
3144
3145 /* PCIe bus returned all Fs? */
3146 case 0xFF:
3147 pr_info("(%s): state=0x%x sense=0x%x\n",
3148 skd_name(skdev), state, sense);
3149 skd_drive_disappeared(skdev);
3150 skd_recover_requests(skdev, 0);
6a5ec65b 3151 blk_start_queue(skdev->queue);
e67f86b3
AB		 3152 break;
3153 default:
3154 /*
3155 * Uknown FW State. Wait for a state we recognize.
3156 */
3157 break;
3158 }
3159 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3160 skd_name(skdev),
3161 skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
3162 skd_skdev_state_to_str(skdev->state), skdev->state);
3163}
3164
3165static void skd_recover_requests(struct skd_device *skdev, int requeue)
3166{
3167 int i;
3168
3169 for (i = 0; i < skdev->num_req_context; i++) {
3170 struct skd_request_context *skreq = &skdev->skreq_table[i];
3171
3172 if (skreq->state == SKD_REQ_STATE_BUSY) {
3173 skd_log_skreq(skdev, skreq, "recover");
3174
3175 SKD_ASSERT((skreq->id & SKD_ID_INCR) != 0);
fcd37eb3 3176 SKD_ASSERT(skreq->req != NULL);
e67f86b3
AB		 3177
3178 /* Release DMA resources for the request. */
3179 if (skreq->n_sg > 0)
3180 skd_postop_sg_list(skdev, skreq);
3181
fcd37eb3
JA		 3182 if (requeue &&
3183 (unsigned long) ++skreq->req->special <
3184 SKD_MAX_RETRIES)
38d4a1bb 3185 blk_requeue_request(skdev->queue, skreq->req);
fcd37eb3 3186 else
e67f86b3
AB		 3187 skd_end_request(skdev, skreq, -EIO);
3188
fcd37eb3 3189 skreq->req = NULL;
e67f86b3
AB		 3190
3191 skreq->state = SKD_REQ_STATE_IDLE;
3192 skreq->id += SKD_ID_INCR;
e67f86b3
AB		 3193 }
3194 if (i > 0)
3195 skreq[-1].next = skreq;
3196 skreq->next = NULL;
3197 }
3198 skdev->skreq_free_list = skdev->skreq_table;
3199
3200 for (i = 0; i < skdev->num_fitmsg_context; i++) {
3201 struct skd_fitmsg_context *skmsg = &skdev->skmsg_table[i];
3202
3203 if (skmsg->state == SKD_MSG_STATE_BUSY) {
3204 skd_log_skmsg(skdev, skmsg, "salvaged");
3205 SKD_ASSERT((skmsg->id & SKD_ID_INCR) != 0);
3206 skmsg->state = SKD_MSG_STATE_IDLE;
3207 skmsg->id += SKD_ID_INCR;
3208 }
3209 if (i > 0)
3210 skmsg[-1].next = skmsg;
3211 skmsg->next = NULL;
3212 }
3213 skdev->skmsg_free_list = skdev->skmsg_table;
3214
3215 for (i = 0; i < skdev->n_special; i++) {
3216 struct skd_special_context *skspcl = &skdev->skspcl_table[i];
3217
3218 /* If orphaned, reclaim it because it has already been reported
3219 * to the process as an error (it was just waiting for
3220 * a completion that didn't come, and now it will never come)
3221 * If busy, change to a state that will cause it to error
3222 * out in the wait routine and let it do the normal
3223 * reporting and reclaiming
3224 */
3225 if (skspcl->req.state == SKD_REQ_STATE_BUSY) {
3226 if (skspcl->orphaned) {
2e44b427 3227 pr_debug("%s:%s:%d orphaned %p\n",
3228 skdev->name, __func__, __LINE__,
3229 skspcl);
e67f86b3
AB		 3230 skd_release_special(skdev, skspcl);
3231 } else {
2e44b427 3232 pr_debug("%s:%s:%d not orphaned %p\n",
3233 skdev->name, __func__, __LINE__,
3234 skspcl);
e67f86b3
AB		 3235 skspcl->req.state = SKD_REQ_STATE_ABORTED;
3236 }
3237 }
3238 }
3239 skdev->skspcl_free_list = skdev->skspcl_table;
3240
3241 for (i = 0; i < SKD_N_TIMEOUT_SLOT; i++)
3242 skdev->timeout_slot[i] = 0;
3243
3244 skdev->in_flight = 0;
3245}
3246
3247static void skd_isr_msg_from_dev(struct skd_device *skdev)
3248{
3249 u32 mfd;
3250 u32 mtd;
3251 u32 data;
3252
3253 mfd = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3254
2e44b427 3255 pr_debug("%s:%s:%d mfd=0x%x last_mtd=0x%x\n",
3256 skdev->name, __func__, __LINE__, mfd, skdev->last_mtd);
e67f86b3
AB		 3257
3258 /* ignore any mtd that is an ack for something we didn't send */
3259 if (FIT_MXD_TYPE(mfd) != FIT_MXD_TYPE(skdev->last_mtd))
3260 return;
3261
3262 switch (FIT_MXD_TYPE(mfd)) {
3263 case FIT_MTD_FITFW_INIT:
3264 skdev->proto_ver = FIT_PROTOCOL_MAJOR_VER(mfd);
3265
3266 if (skdev->proto_ver != FIT_PROTOCOL_VERSION_1) {
3267 pr_err("(%s): protocol mismatch\n",
3268 skdev->name);
3269 pr_err("(%s): got=%d support=%d\n",
3270 skdev->name, skdev->proto_ver,
3271 FIT_PROTOCOL_VERSION_1);
3272 pr_err("(%s): please upgrade driver\n",
3273 skdev->name);
3274 skdev->state = SKD_DRVR_STATE_PROTOCOL_MISMATCH;
3275 skd_soft_reset(skdev);
3276 break;
3277 }
3278 mtd = FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH, 0, 0);
3279 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3280 skdev->last_mtd = mtd;
3281 break;
3282
3283 case FIT_MTD_GET_CMDQ_DEPTH:
3284 skdev->dev_max_queue_depth = FIT_MXD_DATA(mfd);
3285 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH, 0,
3286 SKD_N_COMPLETION_ENTRY);
3287 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3288 skdev->last_mtd = mtd;
3289 break;
3290
3291 case FIT_MTD_SET_COMPQ_DEPTH:
3292 SKD_WRITEQ(skdev, skdev->cq_dma_address, FIT_MSG_TO_DEVICE_ARG);
3293 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR, 0, 0);
3294 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3295 skdev->last_mtd = mtd;
3296 break;
3297
3298 case FIT_MTD_SET_COMPQ_ADDR:
3299 skd_reset_skcomp(skdev);
3300 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID, 0, skdev->devno);
3301 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3302 skdev->last_mtd = mtd;
3303 break;
3304
3305 case FIT_MTD_CMD_LOG_HOST_ID:
3306 skdev->connect_time_stamp = get_seconds();
3307 data = skdev->connect_time_stamp & 0xFFFF;
3308 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO, 0, data);
3309 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3310 skdev->last_mtd = mtd;
3311 break;
3312
3313 case FIT_MTD_CMD_LOG_TIME_STAMP_LO:
3314 skdev->drive_jiffies = FIT_MXD_DATA(mfd);
3315 data = (skdev->connect_time_stamp >> 16) & 0xFFFF;
3316 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI, 0, data);
3317 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3318 skdev->last_mtd = mtd;
3319 break;
3320
3321 case FIT_MTD_CMD_LOG_TIME_STAMP_HI:
3322 skdev->drive_jiffies |= (FIT_MXD_DATA(mfd) << 16);
3323 mtd = FIT_MXD_CONS(FIT_MTD_ARM_QUEUE, 0, 0);
3324 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3325 skdev->last_mtd = mtd;
3326
3327 pr_err("(%s): Time sync driver=0x%x device=0x%x\n",
3328 skd_name(skdev),
3329 skdev->connect_time_stamp, skdev->drive_jiffies);
3330 break;
3331
3332 case FIT_MTD_ARM_QUEUE:
3333 skdev->last_mtd = 0;
3334 /*
3335 * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
3336 */
3337 break;
3338
3339 default:
3340 break;
3341 }
3342}
3343
3344static void skd_disable_interrupts(struct skd_device *skdev)
3345{
3346 u32 sense;
3347
3348 sense = SKD_READL(skdev, FIT_CONTROL);
3349 sense &= ~FIT_CR_ENABLE_INTERRUPTS;
3350 SKD_WRITEL(skdev, sense, FIT_CONTROL);
2e44b427 3351 pr_debug("%s:%s:%d sense 0x%x\n",
3352 skdev->name, __func__, __LINE__, sense);
e67f86b3
AB		 3353
3354 /* Note that the 1s is written. A 1-bit means
3355 * disable, a 0 means enable.
3356 */
3357 SKD_WRITEL(skdev, ~0, FIT_INT_MASK_HOST);
3358}
3359
3360static void skd_enable_interrupts(struct skd_device *skdev)
3361{
3362 u32 val;
3363
3364 /* unmask interrupts first */
3365 val = FIT_ISH_FW_STATE_CHANGE +
3366 FIT_ISH_COMPLETION_POSTED + FIT_ISH_MSG_FROM_DEV;
3367
3368 /* Note that the compliment of mask is written. A 1-bit means
3369 * disable, a 0 means enable. */
3370 SKD_WRITEL(skdev, ~val, FIT_INT_MASK_HOST);
2e44b427 3371 pr_debug("%s:%s:%d interrupt mask=0x%x\n",
3372 skdev->name, __func__, __LINE__, ~val);
e67f86b3
AB		 3373
3374 val = SKD_READL(skdev, FIT_CONTROL);
3375 val |= FIT_CR_ENABLE_INTERRUPTS;
2e44b427 3376 pr_debug("%s:%s:%d control=0x%x\n",
3377 skdev->name, __func__, __LINE__, val);
e67f86b3
AB		 3378 SKD_WRITEL(skdev, val, FIT_CONTROL);
3379}
3380
3381/*
3382 *****************************************************************************
3383 * START, STOP, RESTART, QUIESCE, UNQUIESCE
3384 *****************************************************************************
3385 */
3386
3387static void skd_soft_reset(struct skd_device *skdev)
3388{
3389 u32 val;
3390
3391 val = SKD_READL(skdev, FIT_CONTROL);
3392 val |= (FIT_CR_SOFT_RESET);
2e44b427 3393 pr_debug("%s:%s:%d control=0x%x\n",
3394 skdev->name, __func__, __LINE__, val);
e67f86b3
AB		 3395 SKD_WRITEL(skdev, val, FIT_CONTROL);
3396}
3397
3398static void skd_start_device(struct skd_device *skdev)
3399{
3400 unsigned long flags;
3401 u32 sense;
3402 u32 state;
3403
3404 spin_lock_irqsave(&skdev->lock, flags);
3405
3406 /* ack all ghost interrupts */
3407 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3408
3409 sense = SKD_READL(skdev, FIT_STATUS);
3410
2e44b427 3411 pr_debug("%s:%s:%d initial status=0x%x\n",
3412 skdev->name, __func__, __LINE__, sense);
e67f86b3
AB		 3413
3414 state = sense & FIT_SR_DRIVE_STATE_MASK;
3415 skdev->drive_state = state;
3416 skdev->last_mtd = 0;
3417
3418 skdev->state = SKD_DRVR_STATE_STARTING;
3419 skdev->timer_countdown = SKD_STARTING_TIMO;
3420
3421 skd_enable_interrupts(skdev);
3422
3423 switch (skdev->drive_state) {
3424 case FIT_SR_DRIVE_OFFLINE:
3425 pr_err("(%s): Drive offline...\n", skd_name(skdev));
3426 break;
3427
3428 case FIT_SR_DRIVE_FW_BOOTING:
2e44b427 3429 pr_debug("%s:%s:%d FIT_SR_DRIVE_FW_BOOTING %s\n",
3430 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3431 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3432 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3433 break;
3434
3435 case FIT_SR_DRIVE_BUSY_SANITIZE:
3436 pr_info("(%s): Start: BUSY_SANITIZE\n",
3437 skd_name(skdev));
3438 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3439 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3440 break;
3441
3442 case FIT_SR_DRIVE_BUSY_ERASE:
3443 pr_info("(%s): Start: BUSY_ERASE\n", skd_name(skdev));
3444 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3445 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3446 break;
3447
3448 case FIT_SR_DRIVE_INIT:
3449 case FIT_SR_DRIVE_ONLINE:
3450 skd_soft_reset(skdev);
3451 break;
3452
3453 case FIT_SR_DRIVE_BUSY:
3454 pr_err("(%s): Drive Busy...\n", skd_name(skdev));
3455 skdev->state = SKD_DRVR_STATE_BUSY;
3456 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3457 break;
3458
3459 case FIT_SR_DRIVE_SOFT_RESET:
3460 pr_err("(%s) drive soft reset in prog\n",
3461 skd_name(skdev));
3462 break;
3463
3464 case FIT_SR_DRIVE_FAULT:
3465 /* Fault state is bad...soft reset won't do it...
3466 * Hard reset, maybe, but does it work on device?
3467 * For now, just fault so the system doesn't hang.
3468 */
3469 skd_drive_fault(skdev);
3470 /*start the queue so we can respond with error to requests */
2e44b427 3471 pr_debug("%s:%s:%d starting %s queue\n",
3472 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3473 blk_start_queue(skdev->queue);
e67f86b3
AB		 3474 skdev->gendisk_on = -1;
3475 wake_up_interruptible(&skdev->waitq);
3476 break;
3477
3478 case 0xFF:
3479 /* Most likely the device isn't there or isn't responding
3480 * to the BAR1 addresses. */
3481 skd_drive_disappeared(skdev);
3482 /*start the queue so we can respond with error to requests */
2e44b427 3483 pr_debug("%s:%s:%d starting %s queue to error-out reqs\n",
3484 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3485 blk_start_queue(skdev->queue);
e67f86b3
AB		 3486 skdev->gendisk_on = -1;
3487 wake_up_interruptible(&skdev->waitq);
3488 break;
3489
3490 default:
3491 pr_err("(%s) Start: unknown state %x\n",
3492 skd_name(skdev), skdev->drive_state);
3493 break;
3494 }
3495
3496 state = SKD_READL(skdev, FIT_CONTROL);
2e44b427 3497 pr_debug("%s:%s:%d FIT Control Status=0x%x\n",
3498 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3499
3500 state = SKD_READL(skdev, FIT_INT_STATUS_HOST);
2e44b427 3501 pr_debug("%s:%s:%d Intr Status=0x%x\n",
3502 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3503
3504 state = SKD_READL(skdev, FIT_INT_MASK_HOST);
2e44b427 3505 pr_debug("%s:%s:%d Intr Mask=0x%x\n",
3506 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3507
3508 state = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
2e44b427 3509 pr_debug("%s:%s:%d Msg from Dev=0x%x\n",
3510 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3511
3512 state = SKD_READL(skdev, FIT_HW_VERSION);
2e44b427 3513 pr_debug("%s:%s:%d HW version=0x%x\n",
3514 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3515
3516 spin_unlock_irqrestore(&skdev->lock, flags);
3517}
3518
3519static void skd_stop_device(struct skd_device *skdev)
3520{
3521 unsigned long flags;
3522 struct skd_special_context *skspcl = &skdev->internal_skspcl;
3523 u32 dev_state;
3524 int i;
3525
3526 spin_lock_irqsave(&skdev->lock, flags);
3527
3528 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
3529 pr_err("(%s): skd_stop_device not online no sync\n",
3530 skd_name(skdev));
3531 goto stop_out;
3532 }
3533
3534 if (skspcl->req.state != SKD_REQ_STATE_IDLE) {
3535 pr_err("(%s): skd_stop_device no special\n",
3536 skd_name(skdev));
3537 goto stop_out;
3538 }
3539
3540 skdev->state = SKD_DRVR_STATE_SYNCING;
3541 skdev->sync_done = 0;
3542
3543 skd_send_internal_skspcl(skdev, skspcl, SYNCHRONIZE_CACHE);
3544
3545 spin_unlock_irqrestore(&skdev->lock, flags);
3546
3547 wait_event_interruptible_timeout(skdev->waitq,
3548 (skdev->sync_done), (10 * HZ));
3549
3550 spin_lock_irqsave(&skdev->lock, flags);
3551
3552 switch (skdev->sync_done) {
3553 case 0:
3554 pr_err("(%s): skd_stop_device no sync\n",
3555 skd_name(skdev));
3556 break;
3557 case 1:
3558 pr_err("(%s): skd_stop_device sync done\n",
3559 skd_name(skdev));
3560 break;
3561 default:
3562 pr_err("(%s): skd_stop_device sync error\n",
3563 skd_name(skdev));
3564 }
3565
3566stop_out:
3567 skdev->state = SKD_DRVR_STATE_STOPPING;
3568 spin_unlock_irqrestore(&skdev->lock, flags);
3569
3570 skd_kill_timer(skdev);
3571
3572 spin_lock_irqsave(&skdev->lock, flags);
3573 skd_disable_interrupts(skdev);
3574
3575 /* ensure all ints on device are cleared */
3576 /* soft reset the device to unload with a clean slate */
3577 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3578 SKD_WRITEL(skdev, FIT_CR_SOFT_RESET, FIT_CONTROL);
3579
3580 spin_unlock_irqrestore(&skdev->lock, flags);
3581
3582 /* poll every 100ms, 1 second timeout */
3583 for (i = 0; i < 10; i++) {
3584 dev_state =
3585 SKD_READL(skdev, FIT_STATUS) & FIT_SR_DRIVE_STATE_MASK;
3586 if (dev_state == FIT_SR_DRIVE_INIT)
3587 break;
3588 set_current_state(TASK_INTERRUPTIBLE);
3589 schedule_timeout(msecs_to_jiffies(100));
3590 }
3591
3592 if (dev_state != FIT_SR_DRIVE_INIT)
3593 pr_err("(%s): skd_stop_device state error 0x%02x\n",
3594 skd_name(skdev), dev_state);
3595}
3596
3597/* assume spinlock is held */
3598static void skd_restart_device(struct skd_device *skdev)
3599{
3600 u32 state;
3601
3602 /* ack all ghost interrupts */
3603 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3604
3605 state = SKD_READL(skdev, FIT_STATUS);
3606
2e44b427 3607 pr_debug("%s:%s:%d drive status=0x%x\n",
3608 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3609
3610 state &= FIT_SR_DRIVE_STATE_MASK;
3611 skdev->drive_state = state;
3612 skdev->last_mtd = 0;
3613
3614 skdev->state = SKD_DRVR_STATE_RESTARTING;
3615 skdev->timer_countdown = SKD_RESTARTING_TIMO;
3616
3617 skd_soft_reset(skdev);
3618}
3619
3620/* assume spinlock is held */
3621static int skd_quiesce_dev(struct skd_device *skdev)
3622{
3623 int rc = 0;
3624
3625 switch (skdev->state) {
3626 case SKD_DRVR_STATE_BUSY:
3627 case SKD_DRVR_STATE_BUSY_IMMINENT:
2e44b427 3628 pr_debug("%s:%s:%d stopping %s queue\n",
3629 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3630 blk_stop_queue(skdev->queue);
e67f86b3
AB		 3631 break;
3632 case SKD_DRVR_STATE_ONLINE:
3633 case SKD_DRVR_STATE_STOPPING:
3634 case SKD_DRVR_STATE_SYNCING:
3635 case SKD_DRVR_STATE_PAUSING:
3636 case SKD_DRVR_STATE_PAUSED:
3637 case SKD_DRVR_STATE_STARTING:
3638 case SKD_DRVR_STATE_RESTARTING:
3639 case SKD_DRVR_STATE_RESUMING:
3640 default:
3641 rc = -EINVAL;
2e44b427 3642 pr_debug("%s:%s:%d state [%d] not implemented\n",
3643 skdev->name, __func__, __LINE__, skdev->state);
e67f86b3
AB		 3644 }
3645 return rc;
3646}
3647
3648/* assume spinlock is held */
3649static int skd_unquiesce_dev(struct skd_device *skdev)
3650{
3651 int prev_driver_state = skdev->state;
3652
3653 skd_log_skdev(skdev, "unquiesce");
3654 if (skdev->state == SKD_DRVR_STATE_ONLINE) {
2e44b427 3655 pr_debug("%s:%s:%d **** device already ONLINE\n",
3656 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3657 return 0;
3658 }
3659 if (skdev->drive_state != FIT_SR_DRIVE_ONLINE) {
3660 /*
3661 * If there has been an state change to other than
3662 * ONLINE, we will rely on controller state change
3663 * to come back online and restart the queue.
3664 * The BUSY state means that driver is ready to
3665 * continue normal processing but waiting for controller
3666 * to become available.
3667 */
3668 skdev->state = SKD_DRVR_STATE_BUSY;
2e44b427 3669 pr_debug("%s:%s:%d drive BUSY state\n",
3670 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3671 return 0;
3672 }
3673
3674 /*
3675 * Drive has just come online, driver is either in startup,
3676 * paused performing a task, or bust waiting for hardware.
3677 */
3678 switch (skdev->state) {
3679 case SKD_DRVR_STATE_PAUSED:
3680 case SKD_DRVR_STATE_BUSY:
3681 case SKD_DRVR_STATE_BUSY_IMMINENT:
3682 case SKD_DRVR_STATE_BUSY_ERASE:
3683 case SKD_DRVR_STATE_STARTING:
3684 case SKD_DRVR_STATE_RESTARTING:
3685 case SKD_DRVR_STATE_FAULT:
3686 case SKD_DRVR_STATE_IDLE:
3687 case SKD_DRVR_STATE_LOAD:
3688 skdev->state = SKD_DRVR_STATE_ONLINE;
3689 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3690 skd_name(skdev),
3691 skd_skdev_state_to_str(prev_driver_state),
3692 prev_driver_state, skd_skdev_state_to_str(skdev->state),
3693 skdev->state);
2e44b427 3694 pr_debug("%s:%s:%d **** device ONLINE...starting block queue\n",
3695 skdev->name, __func__, __LINE__);
3696 pr_debug("%s:%s:%d starting %s queue\n",
3697 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3 3698 pr_info("(%s): STEC s1120 ONLINE\n", skd_name(skdev));
6a5ec65b 3699 blk_start_queue(skdev->queue);
e67f86b3
AB		 3700 skdev->gendisk_on = 1;
3701 wake_up_interruptible(&skdev->waitq);
3702 break;
3703
3704 case SKD_DRVR_STATE_DISAPPEARED:
3705 default:
2e44b427 3706 pr_debug("%s:%s:%d **** driver state %d, not implemented \n",
3707 skdev->name, __func__, __LINE__,
3708 skdev->state);
e67f86b3
AB		 3709 return -EBUSY;
3710 }
3711 return 0;
3712}
3713
3714/*
3715 *****************************************************************************
3716 * PCIe MSI/MSI-X INTERRUPT HANDLERS
3717 *****************************************************************************
3718 */
3719
3720static irqreturn_t skd_reserved_isr(int irq, void *skd_host_data)
3721{
3722 struct skd_device *skdev = skd_host_data;
3723 unsigned long flags;
3724
3725 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3726 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3727 skdev->name, __func__, __LINE__,
3728 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3729 pr_err("(%s): MSIX reserved irq %d = 0x%x\n", skd_name(skdev),
3730 irq, SKD_READL(skdev, FIT_INT_STATUS_HOST));
3731 SKD_WRITEL(skdev, FIT_INT_RESERVED_MASK, FIT_INT_STATUS_HOST);
3732 spin_unlock_irqrestore(&skdev->lock, flags);
3733 return IRQ_HANDLED;
3734}
3735
3736static irqreturn_t skd_statec_isr(int irq, void *skd_host_data)
3737{
3738 struct skd_device *skdev = skd_host_data;
3739 unsigned long flags;
3740
3741 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3742 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3743 skdev->name, __func__, __LINE__,
3744 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3745 SKD_WRITEL(skdev, FIT_ISH_FW_STATE_CHANGE, FIT_INT_STATUS_HOST);
3746 skd_isr_fwstate(skdev);
3747 spin_unlock_irqrestore(&skdev->lock, flags);
3748 return IRQ_HANDLED;
3749}
3750
3751static irqreturn_t skd_comp_q(int irq, void *skd_host_data)
3752{
3753 struct skd_device *skdev = skd_host_data;
3754 unsigned long flags;
3755 int flush_enqueued = 0;
3756 int deferred;
3757
3758 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3759 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3760 skdev->name, __func__, __LINE__,
3761 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3762 SKD_WRITEL(skdev, FIT_ISH_COMPLETION_POSTED, FIT_INT_STATUS_HOST);
3763 deferred = skd_isr_completion_posted(skdev, skd_isr_comp_limit,
3764 &flush_enqueued);
e67f86b3
AB		 3765 if (flush_enqueued)
3766 skd_request_fn(skdev->queue);
3767
3768 if (deferred)
3769 schedule_work(&skdev->completion_worker);
3770 else if (!flush_enqueued)
3771 skd_request_fn(skdev->queue);
3772
3773 spin_unlock_irqrestore(&skdev->lock, flags);
3774
3775 return IRQ_HANDLED;
3776}
3777
3778static irqreturn_t skd_msg_isr(int irq, void *skd_host_data)
3779{
3780 struct skd_device *skdev = skd_host_data;
3781 unsigned long flags;
3782
3783 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3784 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3785 skdev->name, __func__, __LINE__,
3786 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3787 SKD_WRITEL(skdev, FIT_ISH_MSG_FROM_DEV, FIT_INT_STATUS_HOST);
3788 skd_isr_msg_from_dev(skdev);
3789 spin_unlock_irqrestore(&skdev->lock, flags);
3790 return IRQ_HANDLED;
3791}
3792
3793static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
3794{
3795 struct skd_device *skdev = skd_host_data;
3796 unsigned long flags;
3797
3798 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3799 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3800 skdev->name, __func__, __LINE__,
3801 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3802 SKD_WRITEL(skdev, FIT_INT_QUEUE_FULL, FIT_INT_STATUS_HOST);
3803 spin_unlock_irqrestore(&skdev->lock, flags);
3804 return IRQ_HANDLED;
3805}
3806
3807/*
3808 *****************************************************************************
3809 * PCIe MSI/MSI-X SETUP
3810 *****************************************************************************
3811 */
3812
3813struct skd_msix_entry {
e67f86b3
AB		 3814 char isr_name[30];
3815};
3816
3817struct skd_init_msix_entry {
3818 const char *name;
3819 irq_handler_t handler;
3820};
3821
3822#define SKD_MAX_MSIX_COUNT 13
3823#define SKD_MIN_MSIX_COUNT 7
3824#define SKD_BASE_MSIX_IRQ 4
3825
3826static struct skd_init_msix_entry msix_entries[SKD_MAX_MSIX_COUNT] = {
3827 { "(DMA 0)", skd_reserved_isr },
3828 { "(DMA 1)", skd_reserved_isr },
3829 { "(DMA 2)", skd_reserved_isr },
3830 { "(DMA 3)", skd_reserved_isr },
3831 { "(State Change)", skd_statec_isr },
3832 { "(COMPL_Q)", skd_comp_q },
3833 { "(MSG)", skd_msg_isr },
3834 { "(Reserved)", skd_reserved_isr },
3835 { "(Reserved)", skd_reserved_isr },
3836 { "(Queue Full 0)", skd_qfull_isr },
3837 { "(Queue Full 1)", skd_qfull_isr },
3838 { "(Queue Full 2)", skd_qfull_isr },
3839 { "(Queue Full 3)", skd_qfull_isr },
3840};
3841
e67f86b3
AB		 3842static int skd_acquire_msix(struct skd_device *skdev)
3843{
a9df8625 3844 int i, rc;
46817769 3845 struct pci_dev *pdev = skdev->pdev;
e67f86b3 3846
180b0ae7
CH		 3847 rc = pci_alloc_irq_vectors(pdev, SKD_MAX_MSIX_COUNT, SKD_MAX_MSIX_COUNT,
3848 PCI_IRQ_MSIX);
3849 if (rc < 0) {
a9df8625
AG		 3850 pr_err("(%s): failed to enable MSI-X %d\n",
3851 skd_name(skdev), rc);
e67f86b3 3852 goto msix_out;
e67f86b3 3853 }
46817769 3854
180b0ae7
CH		 3855 skdev->msix_entries = kcalloc(SKD_MAX_MSIX_COUNT,
3856 sizeof(struct skd_msix_entry), GFP_KERNEL);
e67f86b3
AB		 3857 if (!skdev->msix_entries) {
3858 rc = -ENOMEM;
e67f86b3
AB		 3859 pr_err("(%s): msix table allocation error\n",
3860 skd_name(skdev));
3861 goto msix_out;
3862 }
3863
e67f86b3 3864 /* Enable MSI-X vectors for the base queue */
180b0ae7
CH		 3865 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
3866 struct skd_msix_entry *qentry = &skdev->msix_entries[i];
3867
e67f86b3
AB		 3868 snprintf(qentry->isr_name, sizeof(qentry->isr_name),
3869 "%s%d-msix %s", DRV_NAME, skdev->devno,
3870 msix_entries[i].name);
180b0ae7
CH		 3871
3872 rc = devm_request_irq(&skdev->pdev->dev,
3873 pci_irq_vector(skdev->pdev, i),
3874 msix_entries[i].handler, 0,
3875 qentry->isr_name, skdev);
e67f86b3
AB		 3876 if (rc) {
3877 pr_err("(%s): Unable to register(%d) MSI-X "
3878 "handler %d: %s\n",
3879 skd_name(skdev), rc, i, qentry->isr_name);
3880 goto msix_out;
e67f86b3
AB		 3881 }
3882 }
180b0ae7 3883
2e44b427 3884 pr_debug("%s:%s:%d %s: <%s> msix %d irq(s) enabled\n",
3885 skdev->name, __func__, __LINE__,
180b0ae7 3886 pci_name(pdev), skdev->name, SKD_MAX_MSIX_COUNT);
e67f86b3
AB		 3887 return 0;
3888
3889msix_out:
180b0ae7
CH		 3890 while (--i >= 0)
3891 devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), skdev);
3892 kfree(skdev->msix_entries);
3893 skdev->msix_entries = NULL;
e67f86b3
AB		 3894 return rc;
3895}
3896
3897static int skd_acquire_irq(struct skd_device *skdev)
3898{
180b0ae7
CH		 3899 struct pci_dev *pdev = skdev->pdev;
3900 unsigned int irq_flag = PCI_IRQ_LEGACY;
e67f86b3 3901 int rc;
e67f86b3 3902
180b0ae7 3903 if (skd_isr_type == SKD_IRQ_MSIX) {
e67f86b3
AB		 3904 rc = skd_acquire_msix(skdev);
3905 if (!rc)
180b0ae7
CH		 3906 return 0;
3907
3908 pr_err("(%s): failed to enable MSI-X, re-trying with MSI %d\n",
3909 skd_name(skdev), rc);
e67f86b3 3910 }
180b0ae7
CH		 3911
3912 snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d", DRV_NAME,
3913 skdev->devno);
3914
3915 if (skd_isr_type != SKD_IRQ_LEGACY)
3916 irq_flag |= PCI_IRQ_MSI;
3917 rc = pci_alloc_irq_vectors(pdev, 1, 1, irq_flag);
3918 if (rc < 0) {
3919 pr_err("(%s): failed to allocate the MSI interrupt %d\n",
3920 skd_name(skdev), rc);
3921 return rc;
3922 }
3923
3924 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
3925 pdev->msi_enabled ? 0 : IRQF_SHARED,
3926 skdev->isr_name, skdev);
3927 if (rc) {
3928 pci_free_irq_vectors(pdev);
3929 pr_err("(%s): failed to allocate interrupt %d\n",
3930 skd_name(skdev), rc);
3931 return rc;
3932 }
3933
3934 return 0;
e67f86b3
AB		 3935}
3936
3937static void skd_release_irq(struct skd_device *skdev)
3938{
180b0ae7
CH		 3939 struct pci_dev *pdev = skdev->pdev;
3940
3941 if (skdev->msix_entries) {
3942 int i;
3943
3944 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
3945 devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i),
3946 skdev);
3947 }
3948
3949 kfree(skdev->msix_entries);
3950 skdev->msix_entries = NULL;
3951 } else {
3952 devm_free_irq(&pdev->dev, pdev->irq, skdev);
e67f86b3 3953 }
180b0ae7
CH		 3954
3955 pci_free_irq_vectors(pdev);
e67f86b3
AB		 3956}
3957
3958/*
3959 *****************************************************************************
3960 * CONSTRUCT
3961 *****************************************************************************
3962 */
3963
e67f86b3
AB		 3964static int skd_cons_skcomp(struct skd_device *skdev)
3965{
3966 int rc = 0;
3967 struct fit_completion_entry_v1 *skcomp;
3968 u32 nbytes;
3969
3970 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
3971 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
3972
2e44b427 3973 pr_debug("%s:%s:%d comp pci_alloc, total bytes %d entries %d\n",
3974 skdev->name, __func__, __LINE__,
3975 nbytes, SKD_N_COMPLETION_ENTRY);
e67f86b3 3976
a5bbf616
JP		 3977 skcomp = pci_zalloc_consistent(skdev->pdev, nbytes,
3978 &skdev->cq_dma_address);
e67f86b3
AB		 3979
3980 if (skcomp == NULL) {
3981 rc = -ENOMEM;
3982 goto err_out;
3983 }
3984
e67f86b3
AB		 3985 skdev->skcomp_table = skcomp;
3986 skdev->skerr_table = (struct fit_comp_error_info *)((char *)skcomp +
3987 sizeof(*skcomp) *
3988 SKD_N_COMPLETION_ENTRY);
3989
3990err_out:
3991 return rc;
3992}
3993
3994static int skd_cons_skmsg(struct skd_device *skdev)
3995{
3996 int rc = 0;
3997 u32 i;
3998
2e44b427 3999 pr_debug("%s:%s:%d skmsg_table kzalloc, struct %lu, count %u total %lu\n",
4000 skdev->name, __func__, __LINE__,
4001 sizeof(struct skd_fitmsg_context),
4002 skdev->num_fitmsg_context,
4003 sizeof(struct skd_fitmsg_context) * skdev->num_fitmsg_context);
e67f86b3
AB		 4004
4005 skdev->skmsg_table = kzalloc(sizeof(struct skd_fitmsg_context)
4006 *skdev->num_fitmsg_context, GFP_KERNEL);
4007 if (skdev->skmsg_table == NULL) {
4008 rc = -ENOMEM;
4009 goto err_out;
4010 }
4011
4012 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4013 struct skd_fitmsg_context *skmsg;
4014
4015 skmsg = &skdev->skmsg_table[i];
4016
4017 skmsg->id = i + SKD_ID_FIT_MSG;
4018
4019 skmsg->state = SKD_MSG_STATE_IDLE;
4020 skmsg->msg_buf = pci_alloc_consistent(skdev->pdev,
4021 SKD_N_FITMSG_BYTES + 64,
4022 &skmsg->mb_dma_address);
4023
4024 if (skmsg->msg_buf == NULL) {
4025 rc = -ENOMEM;
4026 goto err_out;
4027 }
4028
4029 skmsg->offset = (u32)((u64)skmsg->msg_buf &
4030 (~FIT_QCMD_BASE_ADDRESS_MASK));
4031 skmsg->msg_buf += ~FIT_QCMD_BASE_ADDRESS_MASK;
4032 skmsg->msg_buf = (u8 *)((u64)skmsg->msg_buf &
4033 FIT_QCMD_BASE_ADDRESS_MASK);
4034 skmsg->mb_dma_address += ~FIT_QCMD_BASE_ADDRESS_MASK;
4035 skmsg->mb_dma_address &= FIT_QCMD_BASE_ADDRESS_MASK;
4036 memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
4037
4038 skmsg->next = &skmsg[1];
4039 }
4040
4041 /* Free list is in order starting with the 0th entry. */
4042 skdev->skmsg_table[i - 1].next = NULL;
4043 skdev->skmsg_free_list = skdev->skmsg_table;
4044
4045err_out:
4046 return rc;
4047}
4048
542d7b00
BZ		 4049static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4050 u32 n_sg,
4051 dma_addr_t *ret_dma_addr)
4052{
4053 struct fit_sg_descriptor *sg_list;
4054 u32 nbytes;
4055
4056 nbytes = sizeof(*sg_list) * n_sg;
4057
4058 sg_list = pci_alloc_consistent(skdev->pdev, nbytes, ret_dma_addr);
4059
4060 if (sg_list != NULL) {
4061 uint64_t dma_address = *ret_dma_addr;
4062 u32 i;
4063
4064 memset(sg_list, 0, nbytes);
4065
4066 for (i = 0; i < n_sg - 1; i++) {
4067 uint64_t ndp_off;
4068 ndp_off = (i + 1) * sizeof(struct fit_sg_descriptor);
4069
4070 sg_list[i].next_desc_ptr = dma_address + ndp_off;
4071 }
4072 sg_list[i].next_desc_ptr = 0LL;
4073 }
4074
4075 return sg_list;
4076}
4077
e67f86b3
AB		 4078static int skd_cons_skreq(struct skd_device *skdev)
4079{
4080 int rc = 0;
4081 u32 i;
4082
2e44b427 4083 pr_debug("%s:%s:%d skreq_table kzalloc, struct %lu, count %u total %lu\n",
4084 skdev->name, __func__, __LINE__,
4085 sizeof(struct skd_request_context),
4086 skdev->num_req_context,
4087 sizeof(struct skd_request_context) * skdev->num_req_context);
e67f86b3
AB		 4088
4089 skdev->skreq_table = kzalloc(sizeof(struct skd_request_context)
4090 * skdev->num_req_context, GFP_KERNEL);
4091 if (skdev->skreq_table == NULL) {
4092 rc = -ENOMEM;
4093 goto err_out;
4094 }
4095
2e44b427 4096 pr_debug("%s:%s:%d alloc sg_table sg_per_req %u scatlist %lu total %lu\n",
4097 skdev->name, __func__, __LINE__,
4098 skdev->sgs_per_request, sizeof(struct scatterlist),
4099 skdev->sgs_per_request * sizeof(struct scatterlist));
e67f86b3
AB		 4100
4101 for (i = 0; i < skdev->num_req_context; i++) {
4102 struct skd_request_context *skreq;
4103
4104 skreq = &skdev->skreq_table[i];
4105
4106 skreq->id = i + SKD_ID_RW_REQUEST;
4107 skreq->state = SKD_REQ_STATE_IDLE;
4108
4109 skreq->sg = kzalloc(sizeof(struct scatterlist) *
4110 skdev->sgs_per_request, GFP_KERNEL);
4111 if (skreq->sg == NULL) {
4112 rc = -ENOMEM;
4113 goto err_out;
4114 }
4115 sg_init_table(skreq->sg, skdev->sgs_per_request);
4116
4117 skreq->sksg_list = skd_cons_sg_list(skdev,
4118 skdev->sgs_per_request,
4119 &skreq->sksg_dma_address);
4120
4121 if (skreq->sksg_list == NULL) {
4122 rc = -ENOMEM;
4123 goto err_out;
4124 }
4125
4126 skreq->next = &skreq[1];
4127 }
4128
4129 /* Free list is in order starting with the 0th entry. */
4130 skdev->skreq_table[i - 1].next = NULL;
4131 skdev->skreq_free_list = skdev->skreq_table;
4132
4133err_out:
4134 return rc;
4135}
4136
4137static int skd_cons_skspcl(struct skd_device *skdev)
4138{
4139 int rc = 0;
4140 u32 i, nbytes;
4141
2e44b427 4142 pr_debug("%s:%s:%d skspcl_table kzalloc, struct %lu, count %u total %lu\n",
4143 skdev->name, __func__, __LINE__,
4144 sizeof(struct skd_special_context),
4145 skdev->n_special,
4146 sizeof(struct skd_special_context) * skdev->n_special);
e67f86b3
AB		 4147
4148 skdev->skspcl_table = kzalloc(sizeof(struct skd_special_context)
4149 * skdev->n_special, GFP_KERNEL);
4150 if (skdev->skspcl_table == NULL) {
4151 rc = -ENOMEM;
4152 goto err_out;
4153 }
4154
4155 for (i = 0; i < skdev->n_special; i++) {
4156 struct skd_special_context *skspcl;
4157
4158 skspcl = &skdev->skspcl_table[i];
4159
4160 skspcl->req.id = i + SKD_ID_SPECIAL_REQUEST;
4161 skspcl->req.state = SKD_REQ_STATE_IDLE;
4162
4163 skspcl->req.next = &skspcl[1].req;
4164
4165 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4166
a5bbf616
JP		 4167 skspcl->msg_buf =
4168 pci_zalloc_consistent(skdev->pdev, nbytes,
4169 &skspcl->mb_dma_address);
e67f86b3
AB		 4170 if (skspcl->msg_buf == NULL) {
4171 rc = -ENOMEM;
4172 goto err_out;
4173 }
4174
e67f86b3
AB		 4175 skspcl->req.sg = kzalloc(sizeof(struct scatterlist) *
4176 SKD_N_SG_PER_SPECIAL, GFP_KERNEL);
4177 if (skspcl->req.sg == NULL) {
4178 rc = -ENOMEM;
4179 goto err_out;
4180 }
4181
4182 skspcl->req.sksg_list = skd_cons_sg_list(skdev,
4183 SKD_N_SG_PER_SPECIAL,
4184 &skspcl->req.
4185 sksg_dma_address);
4186 if (skspcl->req.sksg_list == NULL) {
4187 rc = -ENOMEM;
4188 goto err_out;
4189 }
4190 }
4191
4192 /* Free list is in order starting with the 0th entry. */
4193 skdev->skspcl_table[i - 1].req.next = NULL;
4194 skdev->skspcl_free_list = skdev->skspcl_table;
4195
4196 return rc;
4197
4198err_out:
4199 return rc;
4200}
4201
4202static int skd_cons_sksb(struct skd_device *skdev)
4203{
4204 int rc = 0;
4205 struct skd_special_context *skspcl;
4206 u32 nbytes;
4207
4208 skspcl = &skdev->internal_skspcl;
4209
4210 skspcl->req.id = 0 + SKD_ID_INTERNAL;
4211 skspcl->req.state = SKD_REQ_STATE_IDLE;
4212
4213 nbytes = SKD_N_INTERNAL_BYTES;
4214
a5bbf616
JP		 4215 skspcl->data_buf = pci_zalloc_consistent(skdev->pdev, nbytes,
4216 &skspcl->db_dma_address);
e67f86b3
AB		 4217 if (skspcl->data_buf == NULL) {
4218 rc = -ENOMEM;
4219 goto err_out;
4220 }
4221
e67f86b3 4222 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
a5bbf616
JP		 4223 skspcl->msg_buf = pci_zalloc_consistent(skdev->pdev, nbytes,
4224 &skspcl->mb_dma_address);
e67f86b3
AB		 4225 if (skspcl->msg_buf == NULL) {
4226 rc = -ENOMEM;
4227 goto err_out;
4228 }
4229
e67f86b3
AB		 4230 skspcl->req.sksg_list = skd_cons_sg_list(skdev, 1,
4231 &skspcl->req.sksg_dma_address);
4232 if (skspcl->req.sksg_list == NULL) {
4233 rc = -ENOMEM;
4234 goto err_out;
4235 }
4236
4237 if (!skd_format_internal_skspcl(skdev)) {
4238 rc = -EINVAL;
4239 goto err_out;
4240 }
4241
4242err_out:
4243 return rc;
4244}
4245
e67f86b3
AB		 4246static int skd_cons_disk(struct skd_device *skdev)
4247{
4248 int rc = 0;
4249 struct gendisk *disk;
4250 struct request_queue *q;
4251 unsigned long flags;
4252
4253 disk = alloc_disk(SKD_MINORS_PER_DEVICE);
4254 if (!disk) {
4255 rc = -ENOMEM;
4256 goto err_out;
4257 }
4258
4259 skdev->disk = disk;
4260 sprintf(disk->disk_name, DRV_NAME "%u", skdev->devno);
4261
4262 disk->major = skdev->major;
4263 disk->first_minor = skdev->devno * SKD_MINORS_PER_DEVICE;
4264 disk->fops = &skd_blockdev_ops;
4265 disk->private_data = skdev;
4266
fcd37eb3 4267 q = blk_init_queue(skd_request_fn, &skdev->lock);
e67f86b3
AB		 4268 if (!q) {
4269 rc = -ENOMEM;
4270 goto err_out;
4271 }
4272
4273 skdev->queue = q;
4274 disk->queue = q;
4275 q->queuedata = skdev;
4276
6975f732 4277 blk_queue_write_cache(q, true, true);
e67f86b3
AB		 4278 blk_queue_max_segments(q, skdev->sgs_per_request);
4279 blk_queue_max_hw_sectors(q, SKD_N_MAX_SECTORS);
4280
4281 /* set sysfs ptimal_io_size to 8K */
4282 blk_queue_io_opt(q, 8192);
4283
e67f86b3 4284 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
b277da0a 4285 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, q);
e67f86b3
AB		 4286
4287 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 4288 pr_debug("%s:%s:%d stopping %s queue\n",
4289 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 4290 blk_stop_queue(skdev->queue);
e67f86b3
AB		 4291 spin_unlock_irqrestore(&skdev->lock, flags);
4292
4293err_out:
4294 return rc;
4295}
4296
542d7b00
BZ		 4297#define SKD_N_DEV_TABLE 16u
4298static u32 skd_next_devno;
e67f86b3 4299
542d7b00 4300static struct skd_device *skd_construct(struct pci_dev *pdev)
e67f86b3 4301{
542d7b00
BZ		 4302 struct skd_device *skdev;
4303 int blk_major = skd_major;
4304 int rc;
e67f86b3 4305
542d7b00 4306 skdev = kzalloc(sizeof(*skdev), GFP_KERNEL);
e67f86b3 4307
542d7b00
BZ		 4308 if (!skdev) {
4309 pr_err(PFX "(%s): memory alloc failure\n",
4310 pci_name(pdev));
4311 return NULL;
4312 }
e67f86b3 4313
542d7b00
BZ		 4314 skdev->state = SKD_DRVR_STATE_LOAD;
4315 skdev->pdev = pdev;
4316 skdev->devno = skd_next_devno++;
4317 skdev->major = blk_major;
542d7b00
BZ		 4318 sprintf(skdev->name, DRV_NAME "%d", skdev->devno);
4319 skdev->dev_max_queue_depth = 0;
e67f86b3 4320
542d7b00
BZ		 4321 skdev->num_req_context = skd_max_queue_depth;
4322 skdev->num_fitmsg_context = skd_max_queue_depth;
4323 skdev->n_special = skd_max_pass_thru;
4324 skdev->cur_max_queue_depth = 1;
4325 skdev->queue_low_water_mark = 1;
4326 skdev->proto_ver = 99;
4327 skdev->sgs_per_request = skd_sgs_per_request;
4328 skdev->dbg_level = skd_dbg_level;
e67f86b3 4329
542d7b00 4330 atomic_set(&skdev->device_count, 0);
e67f86b3 4331
542d7b00
BZ		 4332 spin_lock_init(&skdev->lock);
4333
4334 INIT_WORK(&skdev->completion_worker, skd_completion_worker);
e67f86b3 4335
2e44b427 4336 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
542d7b00
BZ		 4337 rc = skd_cons_skcomp(skdev);
4338 if (rc < 0)
4339 goto err_out;
e67f86b3 4340
542d7b00
BZ		 4341 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
4342 rc = skd_cons_skmsg(skdev);
4343 if (rc < 0)
4344 goto err_out;
4345
4346 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
4347 rc = skd_cons_skreq(skdev);
4348 if (rc < 0)
4349 goto err_out;
4350
4351 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
4352 rc = skd_cons_skspcl(skdev);
4353 if (rc < 0)
4354 goto err_out;
4355
4356 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
4357 rc = skd_cons_sksb(skdev);
4358 if (rc < 0)
4359 goto err_out;
4360
4361 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
4362 rc = skd_cons_disk(skdev);
4363 if (rc < 0)
4364 goto err_out;
4365
4366 pr_debug("%s:%s:%d VICTORY\n", skdev->name, __func__, __LINE__);
4367 return skdev;
4368
4369err_out:
4370 pr_debug("%s:%s:%d construct failed\n",
4371 skdev->name, __func__, __LINE__);
4372 skd_destruct(skdev);
4373 return NULL;
e67f86b3
AB		 4374}
4375
542d7b00
BZ		 4376/*
4377 *****************************************************************************
4378 * DESTRUCT (FREE)
4379 *****************************************************************************
4380 */
4381
e67f86b3
AB		 4382static void skd_free_skcomp(struct skd_device *skdev)
4383{
4384 if (skdev->skcomp_table != NULL) {
4385 u32 nbytes;
4386
4387 nbytes = sizeof(skdev->skcomp_table[0]) *
4388 SKD_N_COMPLETION_ENTRY;
4389 pci_free_consistent(skdev->pdev, nbytes,
4390 skdev->skcomp_table, skdev->cq_dma_address);
4391 }
4392
4393 skdev->skcomp_table = NULL;
4394 skdev->cq_dma_address = 0;
4395}
4396
4397static void skd_free_skmsg(struct skd_device *skdev)
4398{
4399 u32 i;
4400
4401 if (skdev->skmsg_table == NULL)
4402 return;
4403
4404 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4405 struct skd_fitmsg_context *skmsg;
4406
4407 skmsg = &skdev->skmsg_table[i];
4408
4409 if (skmsg->msg_buf != NULL) {
4410 skmsg->msg_buf += skmsg->offset;
4411 skmsg->mb_dma_address += skmsg->offset;
4412 pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES,
4413 skmsg->msg_buf,
4414 skmsg->mb_dma_address);
4415 }
4416 skmsg->msg_buf = NULL;
4417 skmsg->mb_dma_address = 0;
4418 }
4419
4420 kfree(skdev->skmsg_table);
4421 skdev->skmsg_table = NULL;
4422}
4423
542d7b00
BZ		 4424static void skd_free_sg_list(struct skd_device *skdev,
4425 struct fit_sg_descriptor *sg_list,
4426 u32 n_sg, dma_addr_t dma_addr)
4427{
4428 if (sg_list != NULL) {
4429 u32 nbytes;
4430
4431 nbytes = sizeof(*sg_list) * n_sg;
4432
4433 pci_free_consistent(skdev->pdev, nbytes, sg_list, dma_addr);
4434 }
4435}
4436
e67f86b3
AB		 4437static void skd_free_skreq(struct skd_device *skdev)
4438{
4439 u32 i;
4440
4441 if (skdev->skreq_table == NULL)
4442 return;
4443
4444 for (i = 0; i < skdev->num_req_context; i++) {
4445 struct skd_request_context *skreq;
4446
4447 skreq = &skdev->skreq_table[i];
4448
4449 skd_free_sg_list(skdev, skreq->sksg_list,
4450 skdev->sgs_per_request,
4451 skreq->sksg_dma_address);
4452
4453 skreq->sksg_list = NULL;
4454 skreq->sksg_dma_address = 0;
4455
4456 kfree(skreq->sg);
4457 }
4458
4459 kfree(skdev->skreq_table);
4460 skdev->skreq_table = NULL;
4461}
4462
4463static void skd_free_skspcl(struct skd_device *skdev)
4464{
4465 u32 i;
4466 u32 nbytes;
4467
4468 if (skdev->skspcl_table == NULL)
4469 return;
4470
4471 for (i = 0; i < skdev->n_special; i++) {
4472 struct skd_special_context *skspcl;
4473
4474 skspcl = &skdev->skspcl_table[i];
4475
4476 if (skspcl->msg_buf != NULL) {
4477 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4478 pci_free_consistent(skdev->pdev, nbytes,
4479 skspcl->msg_buf,
4480 skspcl->mb_dma_address);
4481 }
4482
4483 skspcl->msg_buf = NULL;
4484 skspcl->mb_dma_address = 0;
4485
4486 skd_free_sg_list(skdev, skspcl->req.sksg_list,
4487 SKD_N_SG_PER_SPECIAL,
4488 skspcl->req.sksg_dma_address);
4489
4490 skspcl->req.sksg_list = NULL;
4491 skspcl->req.sksg_dma_address = 0;
4492
4493 kfree(skspcl->req.sg);
4494 }
4495
4496 kfree(skdev->skspcl_table);
4497 skdev->skspcl_table = NULL;
4498}
4499
4500static void skd_free_sksb(struct skd_device *skdev)
4501{
4502 struct skd_special_context *skspcl;
4503 u32 nbytes;
4504
4505 skspcl = &skdev->internal_skspcl;
4506
4507 if (skspcl->data_buf != NULL) {
4508 nbytes = SKD_N_INTERNAL_BYTES;
4509
4510 pci_free_consistent(skdev->pdev, nbytes,
4511 skspcl->data_buf, skspcl->db_dma_address);
4512 }
4513
4514 skspcl->data_buf = NULL;
4515 skspcl->db_dma_address = 0;
4516
4517 if (skspcl->msg_buf != NULL) {
4518 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4519 pci_free_consistent(skdev->pdev, nbytes,
4520 skspcl->msg_buf, skspcl->mb_dma_address);
4521 }
4522
4523 skspcl->msg_buf = NULL;
4524 skspcl->mb_dma_address = 0;
4525
4526 skd_free_sg_list(skdev, skspcl->req.sksg_list, 1,
4527 skspcl->req.sksg_dma_address);
4528
4529 skspcl->req.sksg_list = NULL;
4530 skspcl->req.sksg_dma_address = 0;
4531}
4532
e67f86b3
AB		 4533static void skd_free_disk(struct skd_device *skdev)
4534{
4535 struct gendisk *disk = skdev->disk;
4536
4537 if (disk != NULL) {
4538 struct request_queue *q = disk->queue;
4539
4540 if (disk->flags & GENHD_FL_UP)
4541 del_gendisk(disk);
4542 if (q)
4543 blk_cleanup_queue(q);
4544 put_disk(disk);
4545 }
4546 skdev->disk = NULL;
4547}
4548
542d7b00
BZ		 4549static void skd_destruct(struct skd_device *skdev)
4550{
4551 if (skdev == NULL)
4552 return;
4553
4554
4555 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
4556 skd_free_disk(skdev);
4557
4558 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
4559 skd_free_sksb(skdev);
4560
4561 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
4562 skd_free_skspcl(skdev);
4563
4564 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
4565 skd_free_skreq(skdev);
4566
4567 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
4568 skd_free_skmsg(skdev);
e67f86b3 4569
542d7b00
BZ		 4570 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
4571 skd_free_skcomp(skdev);
4572
4573 pr_debug("%s:%s:%d skdev\n", skdev->name, __func__, __LINE__);
4574 kfree(skdev);
4575}
e67f86b3
AB		 4576
4577/*
4578 *****************************************************************************
4579 * BLOCK DEVICE (BDEV) GLUE
4580 *****************************************************************************
4581 */
4582
4583static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
4584{
4585 struct skd_device *skdev;
4586 u64 capacity;
4587
4588 skdev = bdev->bd_disk->private_data;
4589
2e44b427 4590 pr_debug("%s:%s:%d %s: CMD[%s] getgeo device\n",
4591 skdev->name, __func__, __LINE__,
4592 bdev->bd_disk->disk_name, current->comm);
e67f86b3
AB		 4593
4594 if (skdev->read_cap_is_valid) {
4595 capacity = get_capacity(skdev->disk);
4596 geo->heads = 64;
4597 geo->sectors = 255;
4598 geo->cylinders = (capacity) / (255 * 64);
4599
4600 return 0;
4601 }
4602 return -EIO;
4603}
4604
0d52c756 4605static int skd_bdev_attach(struct device *parent, struct skd_device *skdev)
e67f86b3 4606{
2e44b427 4607 pr_debug("%s:%s:%d add_disk\n", skdev->name, __func__, __LINE__);
0d52c756 4608 device_add_disk(parent, skdev->disk);
e67f86b3
AB		 4609 return 0;
4610}
4611
4612static const struct block_device_operations skd_blockdev_ops = {
4613 .owner = THIS_MODULE,
4614 .ioctl = skd_bdev_ioctl,
4615 .getgeo = skd_bdev_getgeo,
4616};
4617
4618
4619/*
4620 *****************************************************************************
4621 * PCIe DRIVER GLUE
4622 *****************************************************************************
4623 */
4624
9baa3c34 4625static const struct pci_device_id skd_pci_tbl[] = {
e67f86b3
AB		 4626 { PCI_VENDOR_ID_STEC, PCI_DEVICE_ID_S1120,
4627 PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
4628 { 0 } /* terminate list */
4629};
4630
4631MODULE_DEVICE_TABLE(pci, skd_pci_tbl);
4632
4633static char *skd_pci_info(struct skd_device *skdev, char *str)
4634{
4635 int pcie_reg;
4636
4637 strcpy(str, "PCIe (");
4638 pcie_reg = pci_find_capability(skdev->pdev, PCI_CAP_ID_EXP);
4639
4640 if (pcie_reg) {
4641
4642 char lwstr[6];
4643 uint16_t pcie_lstat, lspeed, lwidth;
4644
4645 pcie_reg += 0x12;
4646 pci_read_config_word(skdev->pdev, pcie_reg, &pcie_lstat);
4647 lspeed = pcie_lstat & (0xF);
4648 lwidth = (pcie_lstat & 0x3F0) >> 4;
4649
4650 if (lspeed == 1)
4651 strcat(str, "2.5GT/s ");
4652 else if (lspeed == 2)
4653 strcat(str, "5.0GT/s ");
4654 else
4655 strcat(str, "<unknown> ");
4656 snprintf(lwstr, sizeof(lwstr), "%dX)", lwidth);
4657 strcat(str, lwstr);
4658 }
4659 return str;
4660}
4661
4662static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4663{
4664 int i;
4665 int rc = 0;
4666 char pci_str[32];
4667 struct skd_device *skdev;
4668
4669 pr_info("STEC s1120 Driver(%s) version %s-b%s\n",
4670 DRV_NAME, DRV_VERSION, DRV_BUILD_ID);
4671 pr_info("(skd?:??:[%s]): vendor=%04X device=%04x\n",
4672 pci_name(pdev), pdev->vendor, pdev->device);
4673
4674 rc = pci_enable_device(pdev);
4675 if (rc)
4676 return rc;
4677 rc = pci_request_regions(pdev, DRV_NAME);
4678 if (rc)
4679 goto err_out;
4680 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
4681 if (!rc) {
4682 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
4683
4684 pr_err("(%s): consistent DMA mask error %d\n",
4685 pci_name(pdev), rc);
4686 }
4687 } else {
4688 (rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)));
4689 if (rc) {
4690
4691 pr_err("(%s): DMA mask error %d\n",
4692 pci_name(pdev), rc);
4693 goto err_out_regions;
4694 }
4695 }
4696
b8df6647
BZ		 4697 if (!skd_major) {
4698 rc = register_blkdev(0, DRV_NAME);
4699 if (rc < 0)
4700 goto err_out_regions;
4701 BUG_ON(!rc);
4702 skd_major = rc;
4703 }
4704
e67f86b3 4705 skdev = skd_construct(pdev);
1762b57f
WY		 4706 if (skdev == NULL) {
4707 rc = -ENOMEM;
e67f86b3 4708 goto err_out_regions;
1762b57f 4709 }
e67f86b3
AB		 4710
4711 skd_pci_info(skdev, pci_str);
4712 pr_info("(%s): %s 64bit\n", skd_name(skdev), pci_str);
4713
4714 pci_set_master(pdev);
4715 rc = pci_enable_pcie_error_reporting(pdev);
4716 if (rc) {
4717 pr_err(
4718 "(%s): bad enable of PCIe error reporting rc=%d\n",
4719 skd_name(skdev), rc);
4720 skdev->pcie_error_reporting_is_enabled = 0;
4721 } else
4722 skdev->pcie_error_reporting_is_enabled = 1;
4723
4724
4725 pci_set_drvdata(pdev, skdev);
ebedd16d 4726
e67f86b3
AB		 4727 for (i = 0; i < SKD_MAX_BARS; i++) {
4728 skdev->mem_phys[i] = pci_resource_start(pdev, i);
4729 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
4730 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
4731 skdev->mem_size[i]);
4732 if (!skdev->mem_map[i]) {
4733 pr_err("(%s): Unable to map adapter memory!\n",
4734 skd_name(skdev));
4735 rc = -ENODEV;
4736 goto err_out_iounmap;
4737 }
2e44b427 4738 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
4739 skdev->name, __func__, __LINE__,
4740 skdev->mem_map[i],
4741 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
e67f86b3
AB		 4742 }
4743
4744 rc = skd_acquire_irq(skdev);
4745 if (rc) {
4746 pr_err("(%s): interrupt resource error %d\n",
4747 skd_name(skdev), rc);
4748 goto err_out_iounmap;
4749 }
4750
4751 rc = skd_start_timer(skdev);
4752 if (rc)
4753 goto err_out_timer;
4754
4755 init_waitqueue_head(&skdev->waitq);
4756
4757 skd_start_device(skdev);
4758
4759 rc = wait_event_interruptible_timeout(skdev->waitq,
4760 (skdev->gendisk_on),
4761 (SKD_START_WAIT_SECONDS * HZ));
4762 if (skdev->gendisk_on > 0) {
4763 /* device came on-line after reset */
0d52c756 4764 skd_bdev_attach(&pdev->dev, skdev);
e67f86b3
AB		 4765 rc = 0;
4766 } else {
4767 /* we timed out, something is wrong with the device,
4768 don't add the disk structure */
4769 pr_err(
4770 "(%s): error: waiting for s1120 timed out %d!\n",
4771 skd_name(skdev), rc);
4772 /* in case of no error; we timeout with ENXIO */
4773 if (!rc)
4774 rc = -ENXIO;
4775 goto err_out_timer;
4776 }
4777
4778
4779#ifdef SKD_VMK_POLL_HANDLER
4780 if (skdev->irq_type == SKD_IRQ_MSIX) {
4781 /* MSIX completion handler is being used for coredump */
4782 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
4783 skdev->msix_entries[5].vector,
4784 skd_comp_q, skdev);
4785 } else {
4786 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
4787 skdev->pdev->irq, skd_isr,
4788 skdev);
4789 }
4790#endif /* SKD_VMK_POLL_HANDLER */
4791
4792 return rc;
4793
4794err_out_timer:
4795 skd_stop_device(skdev);
4796 skd_release_irq(skdev);
4797
4798err_out_iounmap:
4799 for (i = 0; i < SKD_MAX_BARS; i++)
4800 if (skdev->mem_map[i])
4801 iounmap(skdev->mem_map[i]);
4802
4803 if (skdev->pcie_error_reporting_is_enabled)
4804 pci_disable_pcie_error_reporting(pdev);
4805
4806 skd_destruct(skdev);
4807
4808err_out_regions:
4809 pci_release_regions(pdev);
4810
4811err_out:
4812 pci_disable_device(pdev);
4813 pci_set_drvdata(pdev, NULL);
4814 return rc;
4815}
4816
4817static void skd_pci_remove(struct pci_dev *pdev)
4818{
4819 int i;
4820 struct skd_device *skdev;
4821
4822 skdev = pci_get_drvdata(pdev);
4823 if (!skdev) {
4824 pr_err("%s: no device data for PCI\n", pci_name(pdev));
4825 return;
4826 }
4827 skd_stop_device(skdev);
4828 skd_release_irq(skdev);
4829
4830 for (i = 0; i < SKD_MAX_BARS; i++)
4831 if (skdev->mem_map[i])
4832 iounmap((u32 *)skdev->mem_map[i]);
4833
4834 if (skdev->pcie_error_reporting_is_enabled)
4835 pci_disable_pcie_error_reporting(pdev);
4836
4837 skd_destruct(skdev);
4838
4839 pci_release_regions(pdev);
4840 pci_disable_device(pdev);
4841 pci_set_drvdata(pdev, NULL);
4842
4843 return;
4844}
4845
4846static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
4847{
4848 int i;
4849 struct skd_device *skdev;
4850
4851 skdev = pci_get_drvdata(pdev);
4852 if (!skdev) {
4853 pr_err("%s: no device data for PCI\n", pci_name(pdev));
4854 return -EIO;
4855 }
4856
4857 skd_stop_device(skdev);
4858
4859 skd_release_irq(skdev);
4860
4861 for (i = 0; i < SKD_MAX_BARS; i++)
4862 if (skdev->mem_map[i])
4863 iounmap((u32 *)skdev->mem_map[i]);
4864
4865 if (skdev->pcie_error_reporting_is_enabled)
4866 pci_disable_pcie_error_reporting(pdev);
4867
4868 pci_release_regions(pdev);
4869 pci_save_state(pdev);
4870 pci_disable_device(pdev);
4871 pci_set_power_state(pdev, pci_choose_state(pdev, state));
4872 return 0;
4873}
4874
4875static int skd_pci_resume(struct pci_dev *pdev)
4876{
4877 int i;
4878 int rc = 0;
4879 struct skd_device *skdev;
4880
4881 skdev = pci_get_drvdata(pdev);
4882 if (!skdev) {
4883 pr_err("%s: no device data for PCI\n", pci_name(pdev));
4884 return -1;
4885 }
4886
4887 pci_set_power_state(pdev, PCI_D0);
4888 pci_enable_wake(pdev, PCI_D0, 0);
4889 pci_restore_state(pdev);
4890
4891 rc = pci_enable_device(pdev);
4892 if (rc)
4893 return rc;
4894 rc = pci_request_regions(pdev, DRV_NAME);
4895 if (rc)
4896 goto err_out;
4897 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
4898 if (!rc) {
4899 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
4900
4901 pr_err("(%s): consistent DMA mask error %d\n",
4902 pci_name(pdev), rc);
4903 }
4904 } else {
4905 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
4906 if (rc) {
4907
4908 pr_err("(%s): DMA mask error %d\n",
4909 pci_name(pdev), rc);
4910 goto err_out_regions;
4911 }
4912 }
4913
4914 pci_set_master(pdev);
4915 rc = pci_enable_pcie_error_reporting(pdev);
4916 if (rc) {
4917 pr_err("(%s): bad enable of PCIe error reporting rc=%d\n",
4918 skdev->name, rc);
4919 skdev->pcie_error_reporting_is_enabled = 0;
4920 } else
4921 skdev->pcie_error_reporting_is_enabled = 1;
4922
4923 for (i = 0; i < SKD_MAX_BARS; i++) {
4924
4925 skdev->mem_phys[i] = pci_resource_start(pdev, i);
4926 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
4927 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
4928 skdev->mem_size[i]);
4929 if (!skdev->mem_map[i]) {
4930 pr_err("(%s): Unable to map adapter memory!\n",
4931 skd_name(skdev));
4932 rc = -ENODEV;
4933 goto err_out_iounmap;
4934 }
2e44b427 4935 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
4936 skdev->name, __func__, __LINE__,
4937 skdev->mem_map[i],
4938 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
e67f86b3
AB		 4939 }
4940 rc = skd_acquire_irq(skdev);
4941 if (rc) {
4942
4943 pr_err("(%s): interrupt resource error %d\n",
4944 pci_name(pdev), rc);
4945 goto err_out_iounmap;
4946 }
4947
4948 rc = skd_start_timer(skdev);
4949 if (rc)
4950 goto err_out_timer;
4951
4952 init_waitqueue_head(&skdev->waitq);
4953
4954 skd_start_device(skdev);
4955
4956 return rc;
4957
4958err_out_timer:
4959 skd_stop_device(skdev);
4960 skd_release_irq(skdev);
4961
4962err_out_iounmap:
4963 for (i = 0; i < SKD_MAX_BARS; i++)
4964 if (skdev->mem_map[i])
4965 iounmap(skdev->mem_map[i]);
4966
4967 if (skdev->pcie_error_reporting_is_enabled)
4968 pci_disable_pcie_error_reporting(pdev);
4969
4970err_out_regions:
4971 pci_release_regions(pdev);
4972
4973err_out:
4974 pci_disable_device(pdev);
4975 return rc;
4976}
4977
4978static void skd_pci_shutdown(struct pci_dev *pdev)
4979{
4980 struct skd_device *skdev;
4981
4982 pr_err("skd_pci_shutdown called\n");
4983
4984 skdev = pci_get_drvdata(pdev);
4985 if (!skdev) {
4986 pr_err("%s: no device data for PCI\n", pci_name(pdev));
4987 return;
4988 }
4989
4990 pr_err("%s: calling stop\n", skd_name(skdev));
4991 skd_stop_device(skdev);
4992}
4993
4994static struct pci_driver skd_driver = {
4995 .name = DRV_NAME,
4996 .id_table = skd_pci_tbl,
4997 .probe = skd_pci_probe,
4998 .remove = skd_pci_remove,
4999 .suspend = skd_pci_suspend,
5000 .resume = skd_pci_resume,
5001 .shutdown = skd_pci_shutdown,
5002};
5003
5004/*
5005 *****************************************************************************
5006 * LOGGING SUPPORT
5007 *****************************************************************************
5008 */
5009
5010static const char *skd_name(struct skd_device *skdev)
5011{
5012 memset(skdev->id_str, 0, sizeof(skdev->id_str));
5013
5014 if (skdev->inquiry_is_valid)
5015 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:%s:[%s]",
5016 skdev->name, skdev->inq_serial_num,
5017 pci_name(skdev->pdev));
5018 else
5019 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:??:[%s]",
5020 skdev->name, pci_name(skdev->pdev));
5021
5022 return skdev->id_str;
5023}
5024
5025const char *skd_drive_state_to_str(int state)
5026{
5027 switch (state) {
5028 case FIT_SR_DRIVE_OFFLINE:
5029 return "OFFLINE";
5030 case FIT_SR_DRIVE_INIT:
5031 return "INIT";
5032 case FIT_SR_DRIVE_ONLINE:
5033 return "ONLINE";
5034 case FIT_SR_DRIVE_BUSY:
5035 return "BUSY";
5036 case FIT_SR_DRIVE_FAULT:
5037 return "FAULT";
5038 case FIT_SR_DRIVE_DEGRADED:
5039 return "DEGRADED";
5040 case FIT_SR_PCIE_LINK_DOWN:
5041 return "INK_DOWN";
5042 case FIT_SR_DRIVE_SOFT_RESET:
5043 return "SOFT_RESET";
5044 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
5045 return "NEED_FW";
5046 case FIT_SR_DRIVE_INIT_FAULT:
5047 return "INIT_FAULT";
5048 case FIT_SR_DRIVE_BUSY_SANITIZE:
5049 return "BUSY_SANITIZE";
5050 case FIT_SR_DRIVE_BUSY_ERASE:
5051 return "BUSY_ERASE";
5052 case FIT_SR_DRIVE_FW_BOOTING:
5053 return "FW_BOOTING";
5054 default:
5055 return "???";
5056 }
5057}
5058
5059const char *skd_skdev_state_to_str(enum skd_drvr_state state)
5060{
5061 switch (state) {
5062 case SKD_DRVR_STATE_LOAD:
5063 return "LOAD";
5064 case SKD_DRVR_STATE_IDLE:
5065 return "IDLE";
5066 case SKD_DRVR_STATE_BUSY:
5067 return "BUSY";
5068 case SKD_DRVR_STATE_STARTING:
5069 return "STARTING";
5070 case SKD_DRVR_STATE_ONLINE:
5071 return "ONLINE";
5072 case SKD_DRVR_STATE_PAUSING:
5073 return "PAUSING";
5074 case SKD_DRVR_STATE_PAUSED:
5075 return "PAUSED";
5076 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
5077 return "DRAINING_TIMEOUT";
5078 case SKD_DRVR_STATE_RESTARTING:
5079 return "RESTARTING";
5080 case SKD_DRVR_STATE_RESUMING:
5081 return "RESUMING";
5082 case SKD_DRVR_STATE_STOPPING:
5083 return "STOPPING";
5084 case SKD_DRVR_STATE_SYNCING:
5085 return "SYNCING";
5086 case SKD_DRVR_STATE_FAULT:
5087 return "FAULT";
5088 case SKD_DRVR_STATE_DISAPPEARED:
5089 return "DISAPPEARED";
5090 case SKD_DRVR_STATE_BUSY_ERASE:
5091 return "BUSY_ERASE";
5092 case SKD_DRVR_STATE_BUSY_SANITIZE:
5093 return "BUSY_SANITIZE";
5094 case SKD_DRVR_STATE_BUSY_IMMINENT:
5095 return "BUSY_IMMINENT";
5096 case SKD_DRVR_STATE_WAIT_BOOT:
5097 return "WAIT_BOOT";
5098
5099 default:
5100 return "???";
5101 }
5102}
5103
a26ba7fa 5104static const char *skd_skmsg_state_to_str(enum skd_fit_msg_state state)
e67f86b3
AB		 5105{
5106 switch (state) {
5107 case SKD_MSG_STATE_IDLE:
5108 return "IDLE";
5109 case SKD_MSG_STATE_BUSY:
5110 return "BUSY";
5111 default:
5112 return "???";
5113 }
5114}
5115
a26ba7fa 5116static const char *skd_skreq_state_to_str(enum skd_req_state state)
e67f86b3
AB		 5117{
5118 switch (state) {
5119 case SKD_REQ_STATE_IDLE:
5120 return "IDLE";
5121 case SKD_REQ_STATE_SETUP:
5122 return "SETUP";
5123 case SKD_REQ_STATE_BUSY:
5124 return "BUSY";
5125 case SKD_REQ_STATE_COMPLETED:
5126 return "COMPLETED";
5127 case SKD_REQ_STATE_TIMEOUT:
5128 return "TIMEOUT";
5129 case SKD_REQ_STATE_ABORTED:
5130 return "ABORTED";
5131 default:
5132 return "???";
5133 }
5134}
5135
5136static void skd_log_skdev(struct skd_device *skdev, const char *event)
5137{
2e44b427 5138 pr_debug("%s:%s:%d (%s) skdev=%p event='%s'\n",
5139 skdev->name, __func__, __LINE__, skdev->name, skdev, event);
5140 pr_debug("%s:%s:%d drive_state=%s(%d) driver_state=%s(%d)\n",
5141 skdev->name, __func__, __LINE__,
5142 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
5143 skd_skdev_state_to_str(skdev->state), skdev->state);
5144 pr_debug("%s:%s:%d busy=%d limit=%d dev=%d lowat=%d\n",
5145 skdev->name, __func__, __LINE__,
5146 skdev->in_flight, skdev->cur_max_queue_depth,
5147 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
5148 pr_debug("%s:%s:%d timestamp=0x%x cycle=%d cycle_ix=%d\n",
5149 skdev->name, __func__, __LINE__,
5150 skdev->timeout_stamp, skdev->skcomp_cycle, skdev->skcomp_ix);
e67f86b3
AB		 5151}
5152
5153static void skd_log_skmsg(struct skd_device *skdev,
5154 struct skd_fitmsg_context *skmsg, const char *event)
5155{
2e44b427 5156 pr_debug("%s:%s:%d (%s) skmsg=%p event='%s'\n",
5157 skdev->name, __func__, __LINE__, skdev->name, skmsg, event);
5158 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x length=%d\n",
5159 skdev->name, __func__, __LINE__,
5160 skd_skmsg_state_to_str(skmsg->state), skmsg->state,
5161 skmsg->id, skmsg->length);
e67f86b3
AB		 5162}
5163
5164static void skd_log_skreq(struct skd_device *skdev,
5165 struct skd_request_context *skreq, const char *event)
5166{
2e44b427 5167 pr_debug("%s:%s:%d (%s) skreq=%p event='%s'\n",
5168 skdev->name, __func__, __LINE__, skdev->name, skreq, event);
5169 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
5170 skdev->name, __func__, __LINE__,
5171 skd_skreq_state_to_str(skreq->state), skreq->state,
5172 skreq->id, skreq->fitmsg_id);
5173 pr_debug("%s:%s:%d timo=0x%x sg_dir=%d n_sg=%d\n",
5174 skdev->name, __func__, __LINE__,
5175 skreq->timeout_stamp, skreq->sg_data_dir, skreq->n_sg);
e67f86b3 5176
fcd37eb3
JA		 5177 if (skreq->req != NULL) {
5178 struct request *req = skreq->req;
5179 u32 lba = (u32)blk_rq_pos(req);
5180 u32 count = blk_rq_sectors(req);
e67f86b3 5181
fcd37eb3
JA		 5182 pr_debug("%s:%s:%d "
5183 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
5184 skdev->name, __func__, __LINE__,
5185 req, lba, lba, count, count,
5186 (int)rq_data_dir(req));
5187 } else
5188 pr_debug("%s:%s:%d req=NULL\n",
5189 skdev->name, __func__, __LINE__);
e67f86b3
AB		 5190}
5191
5192/*
5193 *****************************************************************************
5194 * MODULE GLUE
5195 *****************************************************************************
5196 */
5197
5198static int __init skd_init(void)
5199{
e67f86b3
AB		 5200 pr_info(PFX " v%s-b%s loaded\n", DRV_VERSION, DRV_BUILD_ID);
5201
5202 switch (skd_isr_type) {
5203 case SKD_IRQ_LEGACY:
5204 case SKD_IRQ_MSI:
5205 case SKD_IRQ_MSIX:
5206 break;
5207 default:
fbed149a 5208 pr_err(PFX "skd_isr_type %d invalid, re-set to %d\n",
e67f86b3
AB		 5209 skd_isr_type, SKD_IRQ_DEFAULT);
5210 skd_isr_type = SKD_IRQ_DEFAULT;
5211 }
5212
fbed149a
BZ		 5213 if (skd_max_queue_depth < 1 ||
5214 skd_max_queue_depth > SKD_MAX_QUEUE_DEPTH) {
5215 pr_err(PFX "skd_max_queue_depth %d invalid, re-set to %d\n",
e67f86b3
AB		 5216 skd_max_queue_depth, SKD_MAX_QUEUE_DEPTH_DEFAULT);
5217 skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
5218 }
5219
5220 if (skd_max_req_per_msg < 1 || skd_max_req_per_msg > 14) {
fbed149a 5221 pr_err(PFX "skd_max_req_per_msg %d invalid, re-set to %d\n",
e67f86b3
AB		 5222 skd_max_req_per_msg, SKD_MAX_REQ_PER_MSG_DEFAULT);
5223 skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
5224 }
5225
5226 if (skd_sgs_per_request < 1 || skd_sgs_per_request > 4096) {
fbed149a 5227 pr_err(PFX "skd_sg_per_request %d invalid, re-set to %d\n",
e67f86b3
AB		 5228 skd_sgs_per_request, SKD_N_SG_PER_REQ_DEFAULT);
5229 skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
5230 }
5231
5232 if (skd_dbg_level < 0 || skd_dbg_level > 2) {
fbed149a 5233 pr_err(PFX "skd_dbg_level %d invalid, re-set to %d\n",
e67f86b3
AB		 5234 skd_dbg_level, 0);
5235 skd_dbg_level = 0;
5236 }
5237
5238 if (skd_isr_comp_limit < 0) {
fbed149a 5239 pr_err(PFX "skd_isr_comp_limit %d invalid, set to %d\n",
e67f86b3
AB		 5240 skd_isr_comp_limit, 0);
5241 skd_isr_comp_limit = 0;
5242 }
5243
5244 if (skd_max_pass_thru < 1 || skd_max_pass_thru > 50) {
fbed149a 5245 pr_err(PFX "skd_max_pass_thru %d invalid, re-set to %d\n",
e67f86b3
AB		 5246 skd_max_pass_thru, SKD_N_SPECIAL_CONTEXT);
5247 skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
5248 }
5249
b8df6647 5250 return pci_register_driver(&skd_driver);
e67f86b3
AB		 5251}
5252
5253static void __exit skd_exit(void)
5254{
5255 pr_info(PFX " v%s-b%s unloading\n", DRV_VERSION, DRV_BUILD_ID);
5256
e67f86b3 5257 pci_unregister_driver(&skd_driver);
b8df6647
BZ		 5258
5259 if (skd_major)
5260 unregister_blkdev(skd_major, DRV_NAME);
e67f86b3
AB		 5261}
5262
e67f86b3
AB		 5263module_init(skd_init);
5264module_exit(skd_exit);
Ubuntu Artful kernel mirror