]>
Commit | Line | Data |
---|---|---|
bc50ad75 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
f11bb3e2 CH |
2 | /* |
3 | * Copyright (c) 2011-2014, Intel Corporation. | |
f11bb3e2 CH |
4 | */ |
5 | ||
6 | #ifndef _NVME_H | |
7 | #define _NVME_H | |
8 | ||
9 | #include <linux/nvme.h> | |
a6a5149b | 10 | #include <linux/cdev.h> |
f11bb3e2 CH |
11 | #include <linux/pci.h> |
12 | #include <linux/kref.h> | |
13 | #include <linux/blk-mq.h> | |
a98e58e5 | 14 | #include <linux/sed-opal.h> |
b9e03857 | 15 | #include <linux/fault-inject.h> |
978628ec | 16 | #include <linux/rcupdate.h> |
c1ac9a4b | 17 | #include <linux/wait.h> |
4d2ce688 | 18 | #include <linux/t10-pi.h> |
f11bb3e2 | 19 | |
35fe0d12 HR |
20 | #include <trace/events/block.h> |
21 | ||
8ae4e447 | 22 | extern unsigned int nvme_io_timeout; |
f11bb3e2 CH |
23 | #define NVME_IO_TIMEOUT (nvme_io_timeout * HZ) |
24 | ||
8ae4e447 | 25 | extern unsigned int admin_timeout; |
dc96f938 | 26 | #define NVME_ADMIN_TIMEOUT (admin_timeout * HZ) |
21d34711 | 27 | |
038bd4cb | 28 | #define NVME_DEFAULT_KATO 5 |
038bd4cb | 29 | |
38e18002 IR |
30 | #ifdef CONFIG_ARCH_NO_SG_CHAIN |
31 | #define NVME_INLINE_SG_CNT 0 | |
ba7ca2ae | 32 | #define NVME_INLINE_METADATA_SG_CNT 0 |
38e18002 IR |
33 | #else |
34 | #define NVME_INLINE_SG_CNT 2 | |
ba7ca2ae | 35 | #define NVME_INLINE_METADATA_SG_CNT 1 |
38e18002 IR |
36 | #endif |
37 | ||
6c3c05b0 CK |
38 | /* |
39 | * Default to a 4K page size, with the intention to update this | |
40 | * path in the future to accommodate architectures with differing | |
41 | * kernel and IO page sizes. | |
42 | */ | |
43 | #define NVME_CTRL_PAGE_SHIFT 12 | |
44 | #define NVME_CTRL_PAGE_SIZE (1 << NVME_CTRL_PAGE_SHIFT) | |
45 | ||
9a6327d2 | 46 | extern struct workqueue_struct *nvme_wq; |
b227c59b RS |
47 | extern struct workqueue_struct *nvme_reset_wq; |
48 | extern struct workqueue_struct *nvme_delete_wq; | |
9a6327d2 | 49 | |
f11bb3e2 | 50 | /* |
106198ed CH |
51 | * List of workarounds for devices that required behavior not specified in |
52 | * the standard. | |
f11bb3e2 | 53 | */ |
106198ed CH |
54 | enum nvme_quirks { |
55 | /* | |
56 | * Prefers I/O aligned to a stripe size specified in a vendor | |
57 | * specific Identify field. | |
58 | */ | |
59 | NVME_QUIRK_STRIPE_SIZE = (1 << 0), | |
540c801c KB |
60 | |
61 | /* | |
62 | * The controller doesn't handle Identify value others than 0 or 1 | |
63 | * correctly. | |
64 | */ | |
65 | NVME_QUIRK_IDENTIFY_CNS = (1 << 1), | |
08095e70 KB |
66 | |
67 | /* | |
e850fd16 CH |
68 | * The controller deterministically returns O's on reads to |
69 | * logical blocks that deallocate was called on. | |
08095e70 | 70 | */ |
e850fd16 | 71 | NVME_QUIRK_DEALLOCATE_ZEROES = (1 << 2), |
54adc010 GP |
72 | |
73 | /* | |
74 | * The controller needs a delay before starts checking the device | |
75 | * readiness, which is done by reading the NVME_CSTS_RDY bit. | |
76 | */ | |
77 | NVME_QUIRK_DELAY_BEFORE_CHK_RDY = (1 << 3), | |
c5552fde AL |
78 | |
79 | /* | |
80 | * APST should not be used. | |
81 | */ | |
82 | NVME_QUIRK_NO_APST = (1 << 4), | |
ff5350a8 AL |
83 | |
84 | /* | |
85 | * The deepest sleep state should not be used. | |
86 | */ | |
87 | NVME_QUIRK_NO_DEEPEST_PS = (1 << 5), | |
608cc4b1 | 88 | |
9abd68ef JA |
89 | /* |
90 | * Set MEDIUM priority on SQ creation | |
91 | */ | |
92 | NVME_QUIRK_MEDIUM_PRIO_SQ = (1 << 7), | |
6299358d JD |
93 | |
94 | /* | |
95 | * Ignore device provided subnqn. | |
96 | */ | |
97 | NVME_QUIRK_IGNORE_DEV_SUBNQN = (1 << 8), | |
7b210e4e CH |
98 | |
99 | /* | |
100 | * Broken Write Zeroes. | |
101 | */ | |
102 | NVME_QUIRK_DISABLE_WRITE_ZEROES = (1 << 9), | |
cb32de1b ML |
103 | |
104 | /* | |
105 | * Force simple suspend/resume path. | |
106 | */ | |
107 | NVME_QUIRK_SIMPLE_SUSPEND = (1 << 10), | |
7ad67ca5 | 108 | |
66341331 BH |
109 | /* |
110 | * Use only one interrupt vector for all queues | |
111 | */ | |
7ad67ca5 | 112 | NVME_QUIRK_SINGLE_VECTOR = (1 << 11), |
66341331 BH |
113 | |
114 | /* | |
115 | * Use non-standard 128 bytes SQEs. | |
116 | */ | |
7ad67ca5 | 117 | NVME_QUIRK_128_BYTES_SQES = (1 << 12), |
d38e9f04 BH |
118 | |
119 | /* | |
120 | * Prevent tag overlap between queues | |
121 | */ | |
7ad67ca5 | 122 | NVME_QUIRK_SHARED_TAGS = (1 << 13), |
6c6aa2f2 AM |
123 | |
124 | /* | |
125 | * Don't change the value of the temperature threshold feature | |
126 | */ | |
127 | NVME_QUIRK_NO_TEMP_THRESH_CHANGE = (1 << 14), | |
5bedd3af CH |
128 | |
129 | /* | |
130 | * The controller doesn't handle the Identify Namespace | |
131 | * Identification Descriptor list subcommand despite claiming | |
132 | * NVMe 1.3 compliance. | |
133 | */ | |
134 | NVME_QUIRK_NO_NS_DESC_LIST = (1 << 15), | |
4bdf2603 FS |
135 | |
136 | /* | |
137 | * The controller does not properly handle DMA addresses over | |
138 | * 48 bits. | |
139 | */ | |
140 | NVME_QUIRK_DMA_ADDRESS_BITS_48 = (1 << 16), | |
a2941f6a KB |
141 | |
142 | /* | |
143 | * The controller requires the command_id value be be limited, so skip | |
144 | * encoding the generation sequence number. | |
145 | */ | |
146 | NVME_QUIRK_SKIP_CID_GEN = (1 << 17), | |
106198ed CH |
147 | }; |
148 | ||
d49187e9 CH |
149 | /* |
150 | * Common request structure for NVMe passthrough. All drivers must have | |
151 | * this structure as the first member of their request-private data. | |
152 | */ | |
153 | struct nvme_request { | |
154 | struct nvme_command *cmd; | |
155 | union nvme_result result; | |
e7006de6 | 156 | u8 genctr; |
44e44b29 | 157 | u8 retries; |
27fa9bc5 CH |
158 | u8 flags; |
159 | u16 status; | |
59e29ce6 | 160 | struct nvme_ctrl *ctrl; |
27fa9bc5 CH |
161 | }; |
162 | ||
32acab31 CH |
163 | /* |
164 | * Mark a bio as coming in through the mpath node. | |
165 | */ | |
166 | #define REQ_NVME_MPATH REQ_DRV | |
167 | ||
27fa9bc5 CH |
168 | enum { |
169 | NVME_REQ_CANCELLED = (1 << 0), | |
bb06ec31 | 170 | NVME_REQ_USERCMD = (1 << 1), |
d49187e9 CH |
171 | }; |
172 | ||
173 | static inline struct nvme_request *nvme_req(struct request *req) | |
174 | { | |
175 | return blk_mq_rq_to_pdu(req); | |
176 | } | |
177 | ||
5d87eb94 KB |
178 | static inline u16 nvme_req_qid(struct request *req) |
179 | { | |
643c476d | 180 | if (!req->q->queuedata) |
5d87eb94 | 181 | return 0; |
84115d6d BW |
182 | |
183 | return req->mq_hctx->queue_num + 1; | |
5d87eb94 KB |
184 | } |
185 | ||
54adc010 GP |
186 | /* The below value is the specific amount of delay needed before checking |
187 | * readiness in case of the PCI_DEVICE(0x1c58, 0x0003), which needs the | |
188 | * NVME_QUIRK_DELAY_BEFORE_CHK_RDY quirk enabled. The value (in ms) was | |
189 | * found empirically. | |
190 | */ | |
8c97eecc | 191 | #define NVME_QUIRK_DELAY_AMOUNT 2300 |
54adc010 | 192 | |
4212f4e9 SG |
193 | /* |
194 | * enum nvme_ctrl_state: Controller state | |
195 | * | |
196 | * @NVME_CTRL_NEW: New controller just allocated, initial state | |
197 | * @NVME_CTRL_LIVE: Controller is connected and I/O capable | |
198 | * @NVME_CTRL_RESETTING: Controller is resetting (or scheduled reset) | |
199 | * @NVME_CTRL_CONNECTING: Controller is disconnected, now connecting the | |
200 | * transport | |
201 | * @NVME_CTRL_DELETING: Controller is deleting (or scheduled deletion) | |
ecca390e SG |
202 | * @NVME_CTRL_DELETING_NOIO: Controller is deleting and I/O is not |
203 | * disabled/failed immediately. This state comes | |
204 | * after all async event processing took place and | |
205 | * before ns removal and the controller deletion | |
206 | * progress | |
4212f4e9 SG |
207 | * @NVME_CTRL_DEAD: Controller is non-present/unresponsive during |
208 | * shutdown or removal. In this case we forcibly | |
209 | * kill all inflight I/O as they have no chance to | |
210 | * complete | |
211 | */ | |
bb8d261e CH |
212 | enum nvme_ctrl_state { |
213 | NVME_CTRL_NEW, | |
214 | NVME_CTRL_LIVE, | |
215 | NVME_CTRL_RESETTING, | |
ad6a0a52 | 216 | NVME_CTRL_CONNECTING, |
bb8d261e | 217 | NVME_CTRL_DELETING, |
ecca390e | 218 | NVME_CTRL_DELETING_NOIO, |
0ff9d4e1 | 219 | NVME_CTRL_DEAD, |
bb8d261e CH |
220 | }; |
221 | ||
a3646451 AM |
222 | struct nvme_fault_inject { |
223 | #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS | |
224 | struct fault_attr attr; | |
225 | struct dentry *parent; | |
226 | bool dont_retry; /* DNR, do not retry */ | |
227 | u16 status; /* status code */ | |
228 | #endif | |
229 | }; | |
230 | ||
1c63dc66 | 231 | struct nvme_ctrl { |
6e3ca03e | 232 | bool comp_seen; |
bb8d261e | 233 | enum nvme_ctrl_state state; |
bd4da3ab | 234 | bool identified; |
bb8d261e | 235 | spinlock_t lock; |
e7ad43c3 | 236 | struct mutex scan_lock; |
1c63dc66 | 237 | const struct nvme_ctrl_ops *ops; |
f11bb3e2 | 238 | struct request_queue *admin_q; |
07bfcd09 | 239 | struct request_queue *connect_q; |
e7832cb4 | 240 | struct request_queue *fabrics_q; |
f11bb3e2 | 241 | struct device *dev; |
f11bb3e2 | 242 | int instance; |
103e515e | 243 | int numa_node; |
5bae7f73 | 244 | struct blk_mq_tag_set *tagset; |
34b6c231 | 245 | struct blk_mq_tag_set *admin_tagset; |
f11bb3e2 | 246 | struct list_head namespaces; |
765cc031 | 247 | struct rw_semaphore namespaces_rwsem; |
d22524a4 | 248 | struct device ctrl_device; |
5bae7f73 | 249 | struct device *device; /* char device */ |
ed7770f6 HR |
250 | #ifdef CONFIG_NVME_HWMON |
251 | struct device *hwmon_device; | |
252 | #endif | |
a6a5149b | 253 | struct cdev cdev; |
d86c4d8e | 254 | struct work_struct reset_work; |
c5017e85 | 255 | struct work_struct delete_work; |
c1ac9a4b | 256 | wait_queue_head_t state_wq; |
1c63dc66 | 257 | |
ab9e00cc CH |
258 | struct nvme_subsystem *subsys; |
259 | struct list_head subsys_entry; | |
260 | ||
4f1244c8 | 261 | struct opal_dev *opal_dev; |
a98e58e5 | 262 | |
f11bb3e2 | 263 | char name[12]; |
76e3914a | 264 | u16 cntlid; |
5fd4ce1b CH |
265 | |
266 | u32 ctrl_config; | |
b6dccf7f | 267 | u16 mtfa; |
d858e5f0 | 268 | u32 queue_count; |
5fd4ce1b | 269 | |
20d0dfe6 | 270 | u64 cap; |
f11bb3e2 | 271 | u32 max_hw_sectors; |
943e942e | 272 | u32 max_segments; |
95093350 | 273 | u32 max_integrity_segments; |
5befc7c2 KB |
274 | u32 max_discard_sectors; |
275 | u32 max_discard_segments; | |
276 | u32 max_zeroes_sectors; | |
240e6ee2 KB |
277 | #ifdef CONFIG_BLK_DEV_ZONED |
278 | u32 max_zone_append; | |
279 | #endif | |
49cd84b6 | 280 | u16 crdt[3]; |
f11bb3e2 | 281 | u16 oncs; |
8a9ae523 | 282 | u16 oacs; |
f5d11840 JA |
283 | u16 nssa; |
284 | u16 nr_streams; | |
f968688f | 285 | u16 sqsize; |
0d0b660f | 286 | u32 max_namespaces; |
6bf25d16 | 287 | atomic_t abort_limit; |
f11bb3e2 | 288 | u8 vwc; |
f3ca80fc | 289 | u32 vs; |
07bfcd09 | 290 | u32 sgls; |
038bd4cb | 291 | u16 kas; |
c5552fde AL |
292 | u8 npss; |
293 | u8 apsta; | |
400b6a7b GR |
294 | u16 wctemp; |
295 | u16 cctemp; | |
c0561f82 | 296 | u32 oaes; |
e3d7874d | 297 | u32 aen_result; |
3e53ba38 | 298 | u32 ctratt; |
07fbd32a | 299 | unsigned int shutdown_timeout; |
038bd4cb | 300 | unsigned int kato; |
f3ca80fc | 301 | bool subsystem; |
106198ed | 302 | unsigned long quirks; |
c5552fde | 303 | struct nvme_id_power_state psd[32]; |
84fef62d | 304 | struct nvme_effects_log *effects; |
1cf7a12e | 305 | struct xarray cels; |
5955be21 | 306 | struct work_struct scan_work; |
f866fc42 | 307 | struct work_struct async_event_work; |
038bd4cb | 308 | struct delayed_work ka_work; |
8c4dfea9 | 309 | struct delayed_work failfast_work; |
0a34e466 | 310 | struct nvme_command ka_cmd; |
b6dccf7f | 311 | struct work_struct fw_act_work; |
30d90964 | 312 | unsigned long events; |
07bfcd09 | 313 | |
0d0b660f CH |
314 | #ifdef CONFIG_NVME_MULTIPATH |
315 | /* asymmetric namespace access: */ | |
316 | u8 anacap; | |
317 | u8 anatt; | |
318 | u32 anagrpmax; | |
319 | u32 nanagrpid; | |
320 | struct mutex ana_lock; | |
321 | struct nvme_ana_rsp_hdr *ana_log_buf; | |
322 | size_t ana_log_size; | |
323 | struct timer_list anatt_timer; | |
324 | struct work_struct ana_work; | |
325 | #endif | |
326 | ||
c5552fde AL |
327 | /* Power saving configuration */ |
328 | u64 ps_max_latency_us; | |
76a5af84 | 329 | bool apst_enabled; |
c5552fde | 330 | |
044a9df1 | 331 | /* PCIe only: */ |
fe6d53c9 CH |
332 | u32 hmpre; |
333 | u32 hmmin; | |
044a9df1 CH |
334 | u32 hmminds; |
335 | u16 hmmaxd; | |
fe6d53c9 | 336 | |
07bfcd09 | 337 | /* Fabrics only */ |
07bfcd09 CH |
338 | u32 ioccsz; |
339 | u32 iorcsz; | |
340 | u16 icdoff; | |
341 | u16 maxcmd; | |
fdf9dfa8 | 342 | int nr_reconnects; |
8c4dfea9 VG |
343 | unsigned long flags; |
344 | #define NVME_CTRL_FAILFAST_EXPIRED 0 | |
07bfcd09 | 345 | struct nvmf_ctrl_options *opts; |
cb5b7262 JA |
346 | |
347 | struct page *discard_page; | |
348 | unsigned long discard_page_busy; | |
f79d5fda AM |
349 | |
350 | struct nvme_fault_inject fault_inject; | |
f11bb3e2 CH |
351 | }; |
352 | ||
75c10e73 HR |
353 | enum nvme_iopolicy { |
354 | NVME_IOPOLICY_NUMA, | |
355 | NVME_IOPOLICY_RR, | |
356 | }; | |
357 | ||
ab9e00cc CH |
358 | struct nvme_subsystem { |
359 | int instance; | |
360 | struct device dev; | |
361 | /* | |
362 | * Because we unregister the device on the last put we need | |
363 | * a separate refcount. | |
364 | */ | |
365 | struct kref ref; | |
366 | struct list_head entry; | |
367 | struct mutex lock; | |
368 | struct list_head ctrls; | |
ed754e5d | 369 | struct list_head nsheads; |
ab9e00cc CH |
370 | char subnqn[NVMF_NQN_SIZE]; |
371 | char serial[20]; | |
372 | char model[40]; | |
373 | char firmware_rev[8]; | |
374 | u8 cmic; | |
375 | u16 vendor_id; | |
81adb863 | 376 | u16 awupf; /* 0's based awupf value. */ |
ed754e5d | 377 | struct ida ns_ida; |
75c10e73 HR |
378 | #ifdef CONFIG_NVME_MULTIPATH |
379 | enum nvme_iopolicy iopolicy; | |
380 | #endif | |
ab9e00cc CH |
381 | }; |
382 | ||
002fab04 CH |
383 | /* |
384 | * Container structure for uniqueue namespace identifiers. | |
385 | */ | |
386 | struct nvme_ns_ids { | |
387 | u8 eui64[8]; | |
388 | u8 nguid[16]; | |
389 | uuid_t uuid; | |
71010c30 | 390 | u8 csi; |
002fab04 CH |
391 | }; |
392 | ||
ed754e5d CH |
393 | /* |
394 | * Anchor structure for namespaces. There is one for each namespace in a | |
395 | * NVMe subsystem that any of our controllers can see, and the namespace | |
396 | * structure for each controller is chained of it. For private namespaces | |
397 | * there is a 1:1 relation to our namespace structures, that is ->list | |
398 | * only ever has a single entry for private namespaces. | |
399 | */ | |
400 | struct nvme_ns_head { | |
401 | struct list_head list; | |
402 | struct srcu_struct srcu; | |
403 | struct nvme_subsystem *subsys; | |
404 | unsigned ns_id; | |
405 | struct nvme_ns_ids ids; | |
406 | struct list_head entry; | |
407 | struct kref ref; | |
0c284db7 | 408 | bool shared; |
ed754e5d | 409 | int instance; |
be93e87e | 410 | struct nvme_effects_log *effects; |
2637baed MI |
411 | |
412 | struct cdev cdev; | |
413 | struct device cdev_device; | |
414 | ||
f3334447 | 415 | struct gendisk *disk; |
30897388 | 416 | #ifdef CONFIG_NVME_MULTIPATH |
f3334447 CH |
417 | struct bio_list requeue_list; |
418 | spinlock_t requeue_lock; | |
419 | struct work_struct requeue_work; | |
420 | struct mutex lock; | |
d8a22f85 AE |
421 | unsigned long flags; |
422 | #define NVME_NSHEAD_DISK_LIVE 0 | |
f3334447 CH |
423 | struct nvme_ns __rcu *current_path[]; |
424 | #endif | |
ed754e5d CH |
425 | }; |
426 | ||
30897388 MI |
427 | static inline bool nvme_ns_head_multipath(struct nvme_ns_head *head) |
428 | { | |
429 | return IS_ENABLED(CONFIG_NVME_MULTIPATH) && head->disk; | |
430 | } | |
431 | ||
ffc89b1d MG |
432 | enum nvme_ns_features { |
433 | NVME_NS_EXT_LBAS = 1 << 0, /* support extended LBA format */ | |
b29f8485 | 434 | NVME_NS_METADATA_SUPPORTED = 1 << 1, /* support getting generated md */ |
ffc89b1d MG |
435 | }; |
436 | ||
f11bb3e2 CH |
437 | struct nvme_ns { |
438 | struct list_head list; | |
439 | ||
1c63dc66 | 440 | struct nvme_ctrl *ctrl; |
f11bb3e2 CH |
441 | struct request_queue *queue; |
442 | struct gendisk *disk; | |
0d0b660f CH |
443 | #ifdef CONFIG_NVME_MULTIPATH |
444 | enum nvme_ana_state ana_state; | |
445 | u32 ana_grpid; | |
446 | #endif | |
ed754e5d | 447 | struct list_head siblings; |
f11bb3e2 | 448 | struct kref kref; |
ed754e5d | 449 | struct nvme_ns_head *head; |
f11bb3e2 | 450 | |
f11bb3e2 CH |
451 | int lba_shift; |
452 | u16 ms; | |
f5d11840 JA |
453 | u16 sgs; |
454 | u32 sws; | |
f11bb3e2 | 455 | u8 pi_type; |
240e6ee2 KB |
456 | #ifdef CONFIG_BLK_DEV_ZONED |
457 | u64 zsze; | |
458 | #endif | |
ffc89b1d | 459 | unsigned long features; |
646017a6 | 460 | unsigned long flags; |
0d0b660f CH |
461 | #define NVME_NS_REMOVING 0 |
462 | #define NVME_NS_DEAD 1 | |
463 | #define NVME_NS_ANA_PENDING 2 | |
2f4c9ba2 | 464 | #define NVME_NS_FORCE_RO 3 |
e7d65803 | 465 | #define NVME_NS_READY 4 |
b9e03857 | 466 | |
2637baed MI |
467 | struct cdev cdev; |
468 | struct device cdev_device; | |
469 | ||
b9e03857 | 470 | struct nvme_fault_inject fault_inject; |
b9e03857 | 471 | |
f11bb3e2 CH |
472 | }; |
473 | ||
4d2ce688 JS |
474 | /* NVMe ns supports metadata actions by the controller (generate/strip) */ |
475 | static inline bool nvme_ns_has_pi(struct nvme_ns *ns) | |
476 | { | |
477 | return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple); | |
478 | } | |
479 | ||
1c63dc66 | 480 | struct nvme_ctrl_ops { |
1a353d85 | 481 | const char *name; |
e439bb12 | 482 | struct module *module; |
d3d5b87d CH |
483 | unsigned int flags; |
484 | #define NVME_F_FABRICS (1 << 0) | |
c81bfba9 | 485 | #define NVME_F_METADATA_SUPPORTED (1 << 1) |
e0596ab2 | 486 | #define NVME_F_PCI_P2PDMA (1 << 2) |
1c63dc66 | 487 | int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val); |
5fd4ce1b | 488 | int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val); |
7fd8930f | 489 | int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val); |
1673f1f0 | 490 | void (*free_ctrl)(struct nvme_ctrl *ctrl); |
ad22c355 | 491 | void (*submit_async_event)(struct nvme_ctrl *ctrl); |
c5017e85 | 492 | void (*delete_ctrl)(struct nvme_ctrl *ctrl); |
1a353d85 | 493 | int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size); |
f11bb3e2 CH |
494 | }; |
495 | ||
e7006de6 SG |
496 | /* |
497 | * nvme command_id is constructed as such: | |
498 | * | xxxx | xxxxxxxxxxxx | | |
499 | * gen request tag | |
500 | */ | |
501 | #define nvme_genctr_mask(gen) (gen & 0xf) | |
502 | #define nvme_cid_install_genctr(gen) (nvme_genctr_mask(gen) << 12) | |
503 | #define nvme_genctr_from_cid(cid) ((cid & 0xf000) >> 12) | |
504 | #define nvme_tag_from_cid(cid) (cid & 0xfff) | |
505 | ||
506 | static inline u16 nvme_cid(struct request *rq) | |
507 | { | |
508 | return nvme_cid_install_genctr(nvme_req(rq)->genctr) | rq->tag; | |
509 | } | |
510 | ||
511 | static inline struct request *nvme_find_rq(struct blk_mq_tags *tags, | |
512 | u16 command_id) | |
513 | { | |
514 | u8 genctr = nvme_genctr_from_cid(command_id); | |
515 | u16 tag = nvme_tag_from_cid(command_id); | |
516 | struct request *rq; | |
517 | ||
518 | rq = blk_mq_tag_to_rq(tags, tag); | |
519 | if (unlikely(!rq)) { | |
520 | pr_err("could not locate request for tag %#x\n", | |
521 | tag); | |
522 | return NULL; | |
523 | } | |
524 | if (unlikely(nvme_genctr_mask(nvme_req(rq)->genctr) != genctr)) { | |
525 | dev_err(nvme_req(rq)->ctrl->device, | |
526 | "request %#x genctr mismatch (got %#x expected %#x)\n", | |
527 | tag, genctr, nvme_genctr_mask(nvme_req(rq)->genctr)); | |
528 | return NULL; | |
529 | } | |
530 | return rq; | |
531 | } | |
532 | ||
533 | static inline struct request *nvme_cid_to_rq(struct blk_mq_tags *tags, | |
534 | u16 command_id) | |
535 | { | |
536 | return blk_mq_tag_to_rq(tags, nvme_tag_from_cid(command_id)); | |
537 | } | |
538 | ||
b9e03857 | 539 | #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS |
a3646451 AM |
540 | void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj, |
541 | const char *dev_name); | |
542 | void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inject); | |
b9e03857 TT |
543 | void nvme_should_fail(struct request *req); |
544 | #else | |
a3646451 AM |
545 | static inline void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj, |
546 | const char *dev_name) | |
547 | { | |
548 | } | |
549 | static inline void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inj) | |
550 | { | |
551 | } | |
b9e03857 TT |
552 | static inline void nvme_should_fail(struct request *req) {} |
553 | #endif | |
554 | ||
f3ca80fc CH |
555 | static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl) |
556 | { | |
557 | if (!ctrl->subsystem) | |
558 | return -ENOTTY; | |
559 | return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65); | |
560 | } | |
561 | ||
314d48dd DLM |
562 | /* |
563 | * Convert a 512B sector number to a device logical block number. | |
564 | */ | |
565 | static inline u64 nvme_sect_to_lba(struct nvme_ns *ns, sector_t sector) | |
f11bb3e2 | 566 | { |
314d48dd | 567 | return sector >> (ns->lba_shift - SECTOR_SHIFT); |
f11bb3e2 CH |
568 | } |
569 | ||
e08f2ae8 DLM |
570 | /* |
571 | * Convert a device logical block number to a 512B sector number. | |
572 | */ | |
573 | static inline sector_t nvme_lba_to_sect(struct nvme_ns *ns, u64 lba) | |
f11bb3e2 | 574 | { |
e08f2ae8 | 575 | return lba << (ns->lba_shift - SECTOR_SHIFT); |
f11bb3e2 CH |
576 | } |
577 | ||
71fb90eb KB |
578 | /* |
579 | * Convert byte length to nvme's 0-based num dwords | |
580 | */ | |
581 | static inline u32 nvme_bytes_to_numd(size_t len) | |
582 | { | |
583 | return (len >> 2) - 1; | |
584 | } | |
585 | ||
5ddaabe8 CH |
586 | static inline bool nvme_is_ana_error(u16 status) |
587 | { | |
588 | switch (status & 0x7ff) { | |
589 | case NVME_SC_ANA_TRANSITION: | |
590 | case NVME_SC_ANA_INACCESSIBLE: | |
591 | case NVME_SC_ANA_PERSISTENT_LOSS: | |
592 | return true; | |
593 | default: | |
594 | return false; | |
595 | } | |
596 | } | |
597 | ||
598 | static inline bool nvme_is_path_error(u16 status) | |
599 | { | |
1e41f3bd CH |
600 | /* check for a status code type of 'path related status' */ |
601 | return (status & 0x700) == 0x300; | |
5ddaabe8 CH |
602 | } |
603 | ||
2eb81a33 CH |
604 | /* |
605 | * Fill in the status and result information from the CQE, and then figure out | |
606 | * if blk-mq will need to use IPI magic to complete the request, and if yes do | |
607 | * so. If not let the caller complete the request without an indirect function | |
608 | * call. | |
609 | */ | |
610 | static inline bool nvme_try_complete_req(struct request *req, __le16 status, | |
27fa9bc5 | 611 | union nvme_result result) |
15a190f7 | 612 | { |
27fa9bc5 | 613 | struct nvme_request *rq = nvme_req(req); |
15a190f7 | 614 | |
27fa9bc5 CH |
615 | rq->status = le16_to_cpu(status) >> 1; |
616 | rq->result = result; | |
b9e03857 TT |
617 | /* inject error when permitted by fault injection framework */ |
618 | nvme_should_fail(req); | |
ff029451 CH |
619 | if (unlikely(blk_should_fake_timeout(req->q))) |
620 | return true; | |
621 | return blk_mq_complete_request_remote(req); | |
7688faa6 CH |
622 | } |
623 | ||
d22524a4 CH |
624 | static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl) |
625 | { | |
626 | get_device(ctrl->device); | |
627 | } | |
628 | ||
629 | static inline void nvme_put_ctrl(struct nvme_ctrl *ctrl) | |
630 | { | |
631 | put_device(ctrl->device); | |
632 | } | |
633 | ||
58a8df67 IR |
634 | static inline bool nvme_is_aen_req(u16 qid, __u16 command_id) |
635 | { | |
e7006de6 SG |
636 | return !qid && |
637 | nvme_tag_from_cid(command_id) >= NVME_AQ_BLK_MQ_DEPTH; | |
58a8df67 IR |
638 | } |
639 | ||
77f02a7a | 640 | void nvme_complete_rq(struct request *req); |
dda3248e | 641 | blk_status_t nvme_host_path_error(struct request *req); |
7baa8572 | 642 | bool nvme_cancel_request(struct request *req, void *data, bool reserved); |
25479069 CL |
643 | void nvme_cancel_tagset(struct nvme_ctrl *ctrl); |
644 | void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl); | |
bb8d261e CH |
645 | bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, |
646 | enum nvme_ctrl_state new_state); | |
c1ac9a4b | 647 | bool nvme_wait_reset(struct nvme_ctrl *ctrl); |
b5b05048 | 648 | int nvme_disable_ctrl(struct nvme_ctrl *ctrl); |
c0f2f45b | 649 | int nvme_enable_ctrl(struct nvme_ctrl *ctrl); |
5fd4ce1b | 650 | int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl); |
f3ca80fc CH |
651 | int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, |
652 | const struct nvme_ctrl_ops *ops, unsigned long quirks); | |
53029b04 | 653 | void nvme_uninit_ctrl(struct nvme_ctrl *ctrl); |
d09f2b45 SG |
654 | void nvme_start_ctrl(struct nvme_ctrl *ctrl); |
655 | void nvme_stop_ctrl(struct nvme_ctrl *ctrl); | |
f21c4769 | 656 | int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl); |
5bae7f73 | 657 | |
5bae7f73 | 658 | void nvme_remove_namespaces(struct nvme_ctrl *ctrl); |
1673f1f0 | 659 | |
4f1244c8 CH |
660 | int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len, |
661 | bool send); | |
a98e58e5 | 662 | |
7bf58533 | 663 | void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status, |
287a63eb | 664 | volatile union nvme_result *res); |
f866fc42 | 665 | |
25646264 KB |
666 | void nvme_stop_queues(struct nvme_ctrl *ctrl); |
667 | void nvme_start_queues(struct nvme_ctrl *ctrl); | |
69d9a99c | 668 | void nvme_kill_queues(struct nvme_ctrl *ctrl); |
d6135c3a | 669 | void nvme_sync_queues(struct nvme_ctrl *ctrl); |
04800fbf | 670 | void nvme_sync_io_queues(struct nvme_ctrl *ctrl); |
302ad8cc KB |
671 | void nvme_unfreeze(struct nvme_ctrl *ctrl); |
672 | void nvme_wait_freeze(struct nvme_ctrl *ctrl); | |
7cf0d7c0 | 673 | int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout); |
302ad8cc | 674 | void nvme_start_freeze(struct nvme_ctrl *ctrl); |
363c9aac | 675 | |
eb71f435 | 676 | #define NVME_QID_ANY -1 |
4160982e | 677 | struct request *nvme_alloc_request(struct request_queue *q, |
39dfe844 | 678 | struct nvme_command *cmd, blk_mq_req_flags_t flags); |
f7f1fc36 | 679 | void nvme_cleanup_cmd(struct request *req); |
f4b9e6c9 | 680 | blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req); |
a9715744 TC |
681 | blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl, |
682 | struct request *req); | |
683 | bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq, | |
684 | bool queue_live); | |
685 | ||
686 | static inline bool nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq, | |
687 | bool queue_live) | |
688 | { | |
689 | if (likely(ctrl->state == NVME_CTRL_LIVE)) | |
690 | return true; | |
691 | if (ctrl->ops->flags & NVME_F_FABRICS && | |
692 | ctrl->state == NVME_CTRL_DELETING) | |
693 | return true; | |
694 | return __nvme_check_ready(ctrl, rq, queue_live); | |
695 | } | |
f11bb3e2 CH |
696 | int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, |
697 | void *buf, unsigned bufflen); | |
698 | int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, | |
d49187e9 | 699 | union nvme_result *result, void *buffer, unsigned bufflen, |
9a95e4ef | 700 | unsigned timeout, int qid, int at_head, |
be42a33b | 701 | blk_mq_req_flags_t flags); |
1a87ee65 KB |
702 | int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid, |
703 | unsigned int dword11, void *buffer, size_t buflen, | |
704 | u32 *result); | |
705 | int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid, | |
706 | unsigned int dword11, void *buffer, size_t buflen, | |
707 | u32 *result); | |
9a0be7ab | 708 | int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count); |
038bd4cb | 709 | void nvme_stop_keep_alive(struct nvme_ctrl *ctrl); |
d86c4d8e | 710 | int nvme_reset_ctrl(struct nvme_ctrl *ctrl); |
2405252a | 711 | int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl); |
c1ac9a4b | 712 | int nvme_try_sched_reset(struct nvme_ctrl *ctrl); |
c5017e85 | 713 | int nvme_delete_ctrl(struct nvme_ctrl *ctrl); |
2405252a | 714 | void nvme_queue_scan(struct nvme_ctrl *ctrl); |
be93e87e | 715 | int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, |
0e98719b | 716 | void *log, size_t size, u64 offset); |
1496bd49 CH |
717 | bool nvme_tryget_ns_head(struct nvme_ns_head *head); |
718 | void nvme_put_ns_head(struct nvme_ns_head *head); | |
2637baed MI |
719 | int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device, |
720 | const struct file_operations *fops, struct module *owner); | |
721 | void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device); | |
2405252a CH |
722 | int nvme_ioctl(struct block_device *bdev, fmode_t mode, |
723 | unsigned int cmd, unsigned long arg); | |
2637baed | 724 | long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg); |
2405252a CH |
725 | int nvme_ns_head_ioctl(struct block_device *bdev, fmode_t mode, |
726 | unsigned int cmd, unsigned long arg); | |
2637baed MI |
727 | long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, |
728 | unsigned long arg); | |
2405252a CH |
729 | long nvme_dev_ioctl(struct file *file, unsigned int cmd, |
730 | unsigned long arg); | |
1496bd49 | 731 | int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo); |
d558fb51 | 732 | |
33b14f67 | 733 | extern const struct attribute_group *nvme_ns_id_attr_groups[]; |
1496bd49 | 734 | extern const struct pr_ops nvme_pr_ops; |
32acab31 CH |
735 | extern const struct block_device_operations nvme_ns_head_ops; |
736 | ||
f1cf35e1 | 737 | struct nvme_ns *nvme_find_path(struct nvme_ns_head *head); |
32acab31 | 738 | #ifdef CONFIG_NVME_MULTIPATH |
66b20ac0 MR |
739 | static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl) |
740 | { | |
741 | return ctrl->ana_log_buf != NULL; | |
742 | } | |
743 | ||
b9156dae SG |
744 | void nvme_mpath_unfreeze(struct nvme_subsystem *subsys); |
745 | void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys); | |
746 | void nvme_mpath_start_freeze(struct nvme_subsystem *subsys); | |
9953ab0c | 747 | bool nvme_mpath_set_disk_name(struct nvme_ns *ns, char *disk_name, int *flags); |
5ddaabe8 | 748 | void nvme_failover_req(struct request *req); |
32acab31 CH |
749 | void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl); |
750 | int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head); | |
0d0b660f | 751 | void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id); |
32acab31 | 752 | void nvme_mpath_remove_disk(struct nvme_ns_head *head); |
5e1f6899 CH |
753 | int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id); |
754 | void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl); | |
0d0b660f CH |
755 | void nvme_mpath_uninit(struct nvme_ctrl *ctrl); |
756 | void nvme_mpath_stop(struct nvme_ctrl *ctrl); | |
0157ec8d | 757 | bool nvme_mpath_clear_current_path(struct nvme_ns *ns); |
e7d65803 | 758 | void nvme_mpath_revalidate_paths(struct nvme_ns *ns); |
0157ec8d | 759 | void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl); |
5396fdac | 760 | void nvme_mpath_shutdown_disk(struct nvme_ns_head *head); |
479a322f | 761 | |
2b59787a | 762 | static inline void nvme_trace_bio_complete(struct request *req) |
35fe0d12 HR |
763 | { |
764 | struct nvme_ns *ns = req->q->queuedata; | |
765 | ||
766 | if (req->cmd_flags & REQ_NVME_MPATH) | |
d24de76a | 767 | trace_block_bio_complete(ns->head->disk->queue, req->bio); |
35fe0d12 HR |
768 | } |
769 | ||
0d0b660f CH |
770 | extern struct device_attribute dev_attr_ana_grpid; |
771 | extern struct device_attribute dev_attr_ana_state; | |
75c10e73 | 772 | extern struct device_attribute subsys_attr_iopolicy; |
0d0b660f | 773 | |
32acab31 | 774 | #else |
0d0b660f CH |
775 | static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl) |
776 | { | |
777 | return false; | |
778 | } | |
9953ab0c CH |
779 | static inline bool nvme_mpath_set_disk_name(struct nvme_ns *ns, char *disk_name, |
780 | int *flags) | |
a785dbcc | 781 | { |
9953ab0c | 782 | return false; |
a785dbcc | 783 | } |
5ddaabe8 | 784 | static inline void nvme_failover_req(struct request *req) |
32acab31 CH |
785 | { |
786 | } | |
32acab31 CH |
787 | static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) |
788 | { | |
789 | } | |
790 | static inline int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, | |
791 | struct nvme_ns_head *head) | |
792 | { | |
793 | return 0; | |
794 | } | |
0d0b660f CH |
795 | static inline void nvme_mpath_add_disk(struct nvme_ns *ns, |
796 | struct nvme_id_ns *id) | |
32acab31 CH |
797 | { |
798 | } | |
799 | static inline void nvme_mpath_remove_disk(struct nvme_ns_head *head) | |
800 | { | |
801 | } | |
0157ec8d SG |
802 | static inline bool nvme_mpath_clear_current_path(struct nvme_ns *ns) |
803 | { | |
804 | return false; | |
805 | } | |
e7d65803 HR |
806 | static inline void nvme_mpath_revalidate_paths(struct nvme_ns *ns) |
807 | { | |
808 | } | |
0157ec8d | 809 | static inline void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) |
479a322f SG |
810 | { |
811 | } | |
5396fdac | 812 | static inline void nvme_mpath_shutdown_disk(struct nvme_ns_head *head) |
32acab31 CH |
813 | { |
814 | } | |
2b59787a | 815 | static inline void nvme_trace_bio_complete(struct request *req) |
35fe0d12 HR |
816 | { |
817 | } | |
5e1f6899 CH |
818 | static inline void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl) |
819 | { | |
820 | } | |
821 | static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, | |
0d0b660f CH |
822 | struct nvme_id_ctrl *id) |
823 | { | |
2bd64307 | 824 | if (ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA) |
14a1336e CH |
825 | dev_warn(ctrl->device, |
826 | "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n"); | |
0d0b660f CH |
827 | return 0; |
828 | } | |
829 | static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl) | |
830 | { | |
831 | } | |
832 | static inline void nvme_mpath_stop(struct nvme_ctrl *ctrl) | |
833 | { | |
834 | } | |
b9156dae SG |
835 | static inline void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) |
836 | { | |
837 | } | |
838 | static inline void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) | |
839 | { | |
840 | } | |
841 | static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) | |
842 | { | |
843 | } | |
32acab31 CH |
844 | #endif /* CONFIG_NVME_MULTIPATH */ |
845 | ||
7fad20dd | 846 | int nvme_revalidate_zones(struct nvme_ns *ns); |
8b4fb0f9 CH |
847 | int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, |
848 | unsigned int nr_zones, report_zones_cb cb, void *data); | |
240e6ee2 | 849 | #ifdef CONFIG_BLK_DEV_ZONED |
d525c3c0 | 850 | int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf); |
240e6ee2 KB |
851 | blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns, struct request *req, |
852 | struct nvme_command *cmnd, | |
853 | enum nvme_zone_mgmt_action action); | |
854 | #else | |
240e6ee2 KB |
855 | static inline blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns, |
856 | struct request *req, struct nvme_command *cmnd, | |
857 | enum nvme_zone_mgmt_action action) | |
858 | { | |
859 | return BLK_STS_NOTSUPP; | |
860 | } | |
861 | ||
d525c3c0 | 862 | static inline int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf) |
240e6ee2 KB |
863 | { |
864 | dev_warn(ns->ctrl->device, | |
865 | "Please enable CONFIG_BLK_DEV_ZONED to support ZNS devices\n"); | |
866 | return -EPROTONOSUPPORT; | |
867 | } | |
868 | #endif | |
869 | ||
40267efd SL |
870 | static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev) |
871 | { | |
872 | return dev_to_disk(dev)->private_data; | |
873 | } | |
ca064085 | 874 | |
400b6a7b | 875 | #ifdef CONFIG_NVME_HWMON |
59e330f8 | 876 | int nvme_hwmon_init(struct nvme_ctrl *ctrl); |
ed7770f6 | 877 | void nvme_hwmon_exit(struct nvme_ctrl *ctrl); |
400b6a7b | 878 | #else |
59e330f8 KB |
879 | static inline int nvme_hwmon_init(struct nvme_ctrl *ctrl) |
880 | { | |
881 | return 0; | |
882 | } | |
ed7770f6 HR |
883 | |
884 | static inline void nvme_hwmon_exit(struct nvme_ctrl *ctrl) | |
885 | { | |
886 | } | |
400b6a7b GR |
887 | #endif |
888 | ||
73eefc27 CK |
889 | static inline bool nvme_ctrl_sgl_supported(struct nvme_ctrl *ctrl) |
890 | { | |
891 | return ctrl->sgls & ((1 << 0) | (1 << 1)); | |
892 | } | |
893 | ||
df21b6b1 LG |
894 | u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, |
895 | u8 opcode); | |
ae5e6886 | 896 | int nvme_execute_passthru_rq(struct request *rq); |
b2702aaa | 897 | struct nvme_ctrl *nvme_ctrl_from_file(struct file *file); |
24493b8b LG |
898 | struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid); |
899 | void nvme_put_ns(struct nvme_ns *ns); | |
df21b6b1 | 900 | |
43dc9878 AM |
901 | static inline bool nvme_multi_css(struct nvme_ctrl *ctrl) |
902 | { | |
903 | return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI; | |
904 | } | |
905 | ||
f11bb3e2 | 906 | #endif /* _NVME_H */ |