2 * CXL Flash Device Driver
4 * Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation
5 * Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
7 * Copyright (C) 2015 IBM Corporation
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation; either version
12 * 2 of the License, or (at your option) any later version.
15 #include <linux/syscalls.h>
17 #include <asm/unaligned.h>
18 #include <asm/bitsperlong.h>
20 #include <scsi/scsi_cmnd.h>
21 #include <scsi/scsi_host.h>
22 #include <uapi/scsi/cxlflash_ioctl.h>
27 #include "superpipe.h"
30 * marshal_virt_to_resize() - translate uvirtual to resize structure
31 * @virt: Source structure from which to translate/copy.
32 * @resize: Destination structure for the translate/copy.
34 static void marshal_virt_to_resize(struct dk_cxlflash_uvirtual
*virt
,
35 struct dk_cxlflash_resize
*resize
)
37 resize
->hdr
= virt
->hdr
;
38 resize
->context_id
= virt
->context_id
;
39 resize
->rsrc_handle
= virt
->rsrc_handle
;
40 resize
->req_size
= virt
->lun_size
;
41 resize
->last_lba
= virt
->last_lba
;
45 * marshal_clone_to_rele() - translate clone to release structure
46 * @clone: Source structure from which to translate/copy.
47 * @rele: Destination structure for the translate/copy.
49 static void marshal_clone_to_rele(struct dk_cxlflash_clone
*clone
,
50 struct dk_cxlflash_release
*release
)
52 release
->hdr
= clone
->hdr
;
53 release
->context_id
= clone
->context_id_dst
;
57 * ba_init() - initializes a block allocator
58 * @ba_lun: Block allocator to initialize.
60 * Return: 0 on success, -errno on failure
62 static int ba_init(struct ba_lun
*ba_lun
)
64 struct ba_lun_info
*bali
= NULL
;
65 int lun_size_au
= 0, i
= 0;
66 int last_word_underflow
= 0;
69 pr_debug("%s: Initializing LUN: lun_id = %llX, "
70 "ba_lun->lsize = %lX, ba_lun->au_size = %lX\n",
71 __func__
, ba_lun
->lun_id
, ba_lun
->lsize
, ba_lun
->au_size
);
73 /* Calculate bit map size */
74 lun_size_au
= ba_lun
->lsize
/ ba_lun
->au_size
;
75 if (lun_size_au
== 0) {
76 pr_debug("%s: Requested LUN size of 0!\n", __func__
);
80 /* Allocate lun information container */
81 bali
= kzalloc(sizeof(struct ba_lun_info
), GFP_KERNEL
);
82 if (unlikely(!bali
)) {
83 pr_err("%s: Failed to allocate lun_info for lun_id %llX\n",
84 __func__
, ba_lun
->lun_id
);
88 bali
->total_aus
= lun_size_au
;
89 bali
->lun_bmap_size
= lun_size_au
/ BITS_PER_LONG
;
91 if (lun_size_au
% BITS_PER_LONG
)
92 bali
->lun_bmap_size
++;
94 /* Allocate bitmap space */
95 bali
->lun_alloc_map
= kzalloc((bali
->lun_bmap_size
* sizeof(u64
)),
97 if (unlikely(!bali
->lun_alloc_map
)) {
98 pr_err("%s: Failed to allocate lun allocation map: "
99 "lun_id = %llX\n", __func__
, ba_lun
->lun_id
);
104 /* Initialize the bit map size and set all bits to '1' */
105 bali
->free_aun_cnt
= lun_size_au
;
107 for (i
= 0; i
< bali
->lun_bmap_size
; i
++)
108 bali
->lun_alloc_map
[i
] = 0xFFFFFFFFFFFFFFFFULL
;
110 /* If the last word not fully utilized, mark extra bits as allocated */
111 last_word_underflow
= (bali
->lun_bmap_size
* BITS_PER_LONG
);
112 last_word_underflow
-= bali
->free_aun_cnt
;
113 if (last_word_underflow
> 0) {
114 lam
= &bali
->lun_alloc_map
[bali
->lun_bmap_size
- 1];
115 for (i
= (HIBIT
- last_word_underflow
+ 1);
118 clear_bit(i
, (ulong
*)lam
);
121 /* Initialize high elevator index, low/curr already at 0 from kzalloc */
122 bali
->free_high_idx
= bali
->lun_bmap_size
;
124 /* Allocate clone map */
125 bali
->aun_clone_map
= kzalloc((bali
->total_aus
* sizeof(u8
)),
127 if (unlikely(!bali
->aun_clone_map
)) {
128 pr_err("%s: Failed to allocate clone map: lun_id = %llX\n",
129 __func__
, ba_lun
->lun_id
);
130 kfree(bali
->lun_alloc_map
);
135 /* Pass the allocated lun info as a handle to the user */
136 ba_lun
->ba_lun_handle
= bali
;
138 pr_debug("%s: Successfully initialized the LUN: "
139 "lun_id = %llX, bitmap size = %X, free_aun_cnt = %llX\n",
140 __func__
, ba_lun
->lun_id
, bali
->lun_bmap_size
,
146 * find_free_range() - locates a free bit within the block allocator
147 * @low: First word in block allocator to start search.
148 * @high: Last word in block allocator to search.
149 * @bali: LUN information structure owning the block allocator to search.
150 * @bit_word: Passes back the word in the block allocator owning the free bit.
152 * Return: The bit position within the passed back word, -1 on failure
154 static int find_free_range(u32 low
,
156 struct ba_lun_info
*bali
, int *bit_word
)
160 ulong
*lam
, num_bits
;
162 for (i
= low
; i
< high
; i
++)
163 if (bali
->lun_alloc_map
[i
] != 0) {
164 lam
= (ulong
*)&bali
->lun_alloc_map
[i
];
165 num_bits
= (sizeof(*lam
) * BITS_PER_BYTE
);
166 bit_pos
= find_first_bit(lam
, num_bits
);
168 pr_devel("%s: Found free bit %llX in lun "
169 "map entry %llX at bitmap index = %X\n",
170 __func__
, bit_pos
, bali
->lun_alloc_map
[i
],
174 bali
->free_aun_cnt
--;
175 clear_bit(bit_pos
, lam
);
183 * ba_alloc() - allocates a block from the block allocator
184 * @ba_lun: Block allocator from which to allocate a block.
186 * Return: The allocated block, -1 on failure
188 static u64
ba_alloc(struct ba_lun
*ba_lun
)
192 struct ba_lun_info
*bali
= NULL
;
194 bali
= ba_lun
->ba_lun_handle
;
196 pr_debug("%s: Received block allocation request: "
197 "lun_id = %llX, free_aun_cnt = %llX\n",
198 __func__
, ba_lun
->lun_id
, bali
->free_aun_cnt
);
200 if (bali
->free_aun_cnt
== 0) {
201 pr_debug("%s: No space left on LUN: lun_id = %llX\n",
202 __func__
, ba_lun
->lun_id
);
206 /* Search to find a free entry, curr->high then low->curr */
207 bit_pos
= find_free_range(bali
->free_curr_idx
,
208 bali
->free_high_idx
, bali
, &bit_word
);
210 bit_pos
= find_free_range(bali
->free_low_idx
,
214 pr_debug("%s: Could not find an allocation unit on LUN:"
215 " lun_id = %llX\n", __func__
, ba_lun
->lun_id
);
220 /* Update the free_curr_idx */
221 if (bit_pos
== HIBIT
)
222 bali
->free_curr_idx
= bit_word
+ 1;
224 bali
->free_curr_idx
= bit_word
;
226 pr_debug("%s: Allocating AU number %llX, on lun_id %llX, "
227 "free_aun_cnt = %llX\n", __func__
,
228 ((bit_word
* BITS_PER_LONG
) + bit_pos
), ba_lun
->lun_id
,
231 return (u64
) ((bit_word
* BITS_PER_LONG
) + bit_pos
);
235 * validate_alloc() - validates the specified block has been allocated
236 * @ba_lun_info: LUN info owning the block allocator.
237 * @aun: Block to validate.
239 * Return: 0 on success, -1 on failure
241 static int validate_alloc(struct ba_lun_info
*bali
, u64 aun
)
243 int idx
= 0, bit_pos
= 0;
245 idx
= aun
/ BITS_PER_LONG
;
246 bit_pos
= aun
% BITS_PER_LONG
;
248 if (test_bit(bit_pos
, (ulong
*)&bali
->lun_alloc_map
[idx
]))
255 * ba_free() - frees a block from the block allocator
256 * @ba_lun: Block allocator from which to allocate a block.
257 * @to_free: Block to free.
259 * Return: 0 on success, -1 on failure
261 static int ba_free(struct ba_lun
*ba_lun
, u64 to_free
)
263 int idx
= 0, bit_pos
= 0;
264 struct ba_lun_info
*bali
= NULL
;
266 bali
= ba_lun
->ba_lun_handle
;
268 if (validate_alloc(bali
, to_free
)) {
269 pr_debug("%s: The AUN %llX is not allocated on lun_id %llX\n",
270 __func__
, to_free
, ba_lun
->lun_id
);
274 pr_debug("%s: Received a request to free AU %llX on lun_id %llX, "
275 "free_aun_cnt = %llX\n", __func__
, to_free
, ba_lun
->lun_id
,
278 if (bali
->aun_clone_map
[to_free
] > 0) {
279 pr_debug("%s: AUN %llX on lun_id %llX has been cloned. Clone "
280 "count = %X\n", __func__
, to_free
, ba_lun
->lun_id
,
281 bali
->aun_clone_map
[to_free
]);
282 bali
->aun_clone_map
[to_free
]--;
286 idx
= to_free
/ BITS_PER_LONG
;
287 bit_pos
= to_free
% BITS_PER_LONG
;
289 set_bit(bit_pos
, (ulong
*)&bali
->lun_alloc_map
[idx
]);
290 bali
->free_aun_cnt
++;
292 if (idx
< bali
->free_low_idx
)
293 bali
->free_low_idx
= idx
;
294 else if (idx
> bali
->free_high_idx
)
295 bali
->free_high_idx
= idx
;
297 pr_debug("%s: Successfully freed AU at bit_pos %X, bit map index %X on "
298 "lun_id %llX, free_aun_cnt = %llX\n", __func__
, bit_pos
, idx
,
299 ba_lun
->lun_id
, bali
->free_aun_cnt
);
305 * ba_clone() - Clone a chunk of the block allocation table
306 * @ba_lun: Block allocator from which to allocate a block.
307 * @to_free: Block to free.
309 * Return: 0 on success, -1 on failure
311 static int ba_clone(struct ba_lun
*ba_lun
, u64 to_clone
)
313 struct ba_lun_info
*bali
= ba_lun
->ba_lun_handle
;
315 if (validate_alloc(bali
, to_clone
)) {
316 pr_debug("%s: AUN %llX is not allocated on lun_id %llX\n",
317 __func__
, to_clone
, ba_lun
->lun_id
);
321 pr_debug("%s: Received a request to clone AUN %llX on lun_id %llX\n",
322 __func__
, to_clone
, ba_lun
->lun_id
);
324 if (bali
->aun_clone_map
[to_clone
] == MAX_AUN_CLONE_CNT
) {
325 pr_debug("%s: AUN %llX on lun_id %llX hit max clones already\n",
326 __func__
, to_clone
, ba_lun
->lun_id
);
330 bali
->aun_clone_map
[to_clone
]++;
336 * ba_space() - returns the amount of free space left in the block allocator
337 * @ba_lun: Block allocator.
339 * Return: Amount of free space in block allocator
341 static u64
ba_space(struct ba_lun
*ba_lun
)
343 struct ba_lun_info
*bali
= ba_lun
->ba_lun_handle
;
345 return bali
->free_aun_cnt
;
349 * cxlflash_ba_terminate() - frees resources associated with the block allocator
350 * @ba_lun: Block allocator.
352 * Safe to call in a partially allocated state.
354 void cxlflash_ba_terminate(struct ba_lun
*ba_lun
)
356 struct ba_lun_info
*bali
= ba_lun
->ba_lun_handle
;
359 kfree(bali
->aun_clone_map
);
360 kfree(bali
->lun_alloc_map
);
362 ba_lun
->ba_lun_handle
= NULL
;
367 * init_vlun() - initializes a LUN for virtual use
368 * @lun_info: LUN information structure that owns the block allocator.
370 * Return: 0 on success, -errno on failure
372 static int init_vlun(struct llun_info
*lli
)
375 struct glun_info
*gli
= lli
->parent
;
376 struct blka
*blka
= &gli
->blka
;
378 memset(blka
, 0, sizeof(*blka
));
379 mutex_init(&blka
->mutex
);
381 /* LUN IDs are unique per port, save the index instead */
382 blka
->ba_lun
.lun_id
= lli
->lun_index
;
383 blka
->ba_lun
.lsize
= gli
->max_lba
+ 1;
384 blka
->ba_lun
.lba_size
= gli
->blk_len
;
386 blka
->ba_lun
.au_size
= MC_CHUNK_SIZE
;
387 blka
->nchunk
= blka
->ba_lun
.lsize
/ MC_CHUNK_SIZE
;
389 rc
= ba_init(&blka
->ba_lun
);
391 pr_debug("%s: cannot init block_alloc, rc=%d\n", __func__
, rc
);
393 pr_debug("%s: returning rc=%d lli=%p\n", __func__
, rc
, lli
);
398 * write_same16() - sends a SCSI WRITE_SAME16 (0) command to specified LUN
399 * @sdev: SCSI device associated with LUN.
400 * @lba: Logical block address to start write same.
401 * @nblks: Number of logical blocks to write same.
403 * Return: 0 on success, -errno on failure
405 static int write_same16(struct scsi_device
*sdev
,
411 u8
*sense_buf
= NULL
;
414 int ws_limit
= SISLITE_MAX_WS_BLOCKS
;
417 u32 to
= sdev
->request_queue
->rq_timeout
;
418 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
419 struct device
*dev
= &cfg
->dev
->dev
;
421 cmd_buf
= kzalloc(CMD_BUFSIZE
, GFP_KERNEL
);
422 scsi_cmd
= kzalloc(MAX_COMMAND_SIZE
, GFP_KERNEL
);
423 sense_buf
= kzalloc(SCSI_SENSE_BUFFERSIZE
, GFP_KERNEL
);
424 if (unlikely(!cmd_buf
|| !scsi_cmd
|| !sense_buf
)) {
431 scsi_cmd
[0] = WRITE_SAME_16
;
432 put_unaligned_be64(offset
, &scsi_cmd
[2]);
433 put_unaligned_be32(ws_limit
< left
? ws_limit
: left
,
436 result
= scsi_execute(sdev
, scsi_cmd
, DMA_TO_DEVICE
, cmd_buf
,
437 CMD_BUFSIZE
, sense_buf
, to
, CMD_RETRIES
,
440 dev_err_ratelimited(dev
, "%s: command failed for "
441 "offset %lld result=0x%x\n",
442 __func__
, offset
, result
);
454 pr_debug("%s: returning rc=%d\n", __func__
, rc
);
459 * grow_lxt() - expands the translation table associated with the specified RHTE
460 * @afu: AFU associated with the host.
461 * @sdev: SCSI device associated with LUN.
462 * @ctxid: Context ID of context owning the RHTE.
463 * @rhndl: Resource handle associated with the RHTE.
464 * @rhte: Resource handle entry (RHTE).
465 * @new_size: Number of translation entries associated with RHTE.
467 * By design, this routine employs a 'best attempt' allocation and will
468 * truncate the requested size down if there is not sufficient space in
469 * the block allocator to satisfy the request but there does exist some
470 * amount of space. The user is made aware of this by returning the size
473 * Return: 0 on success, -errno on failure
475 static int grow_lxt(struct afu
*afu
,
476 struct scsi_device
*sdev
,
479 struct sisl_rht_entry
*rhte
,
482 struct sisl_lxt_entry
*lxt
= NULL
, *lxt_old
= NULL
;
483 struct llun_info
*lli
= sdev
->hostdata
;
484 struct glun_info
*gli
= lli
->parent
;
485 struct blka
*blka
= &gli
->blka
;
487 u32 ngrps
, ngrps_old
;
488 u64 aun
; /* chunk# allocated by block allocator */
489 u64 delta
= *new_size
- rhte
->lxt_cnt
;
494 * Check what is available in the block allocator before re-allocating
495 * LXT array. This is done up front under the mutex which must not be
496 * released until after allocation is complete.
498 mutex_lock(&blka
->mutex
);
499 av_size
= ba_space(&blka
->ba_lun
);
500 if (unlikely(av_size
<= 0)) {
501 pr_debug("%s: ba_space error: av_size %d\n", __func__
, av_size
);
502 mutex_unlock(&blka
->mutex
);
510 lxt_old
= rhte
->lxt_start
;
511 ngrps_old
= LXT_NUM_GROUPS(rhte
->lxt_cnt
);
512 ngrps
= LXT_NUM_GROUPS(rhte
->lxt_cnt
+ delta
);
514 if (ngrps
!= ngrps_old
) {
515 /* reallocate to fit new size */
516 lxt
= kzalloc((sizeof(*lxt
) * LXT_GROUP_SIZE
* ngrps
),
518 if (unlikely(!lxt
)) {
519 mutex_unlock(&blka
->mutex
);
524 /* copy over all old entries */
525 memcpy(lxt
, lxt_old
, (sizeof(*lxt
) * rhte
->lxt_cnt
));
529 /* nothing can fail from now on */
530 my_new_size
= rhte
->lxt_cnt
+ delta
;
532 /* add new entries to the end */
533 for (i
= rhte
->lxt_cnt
; i
< my_new_size
; i
++) {
535 * Due to the earlier check of available space, ba_alloc
536 * cannot fail here. If it did due to internal error,
537 * leave a rlba_base of -1u which will likely be a
538 * invalid LUN (too large).
540 aun
= ba_alloc(&blka
->ba_lun
);
541 if ((aun
== -1ULL) || (aun
>= blka
->nchunk
))
542 pr_debug("%s: ba_alloc error: allocated chunk# %llX, "
543 "max %llX\n", __func__
, aun
, blka
->nchunk
- 1);
545 /* select both ports, use r/w perms from RHT */
546 lxt
[i
].rlba_base
= ((aun
<< MC_CHUNK_SHIFT
) |
547 (lli
->lun_index
<< LXT_LUNIDX_SHIFT
) |
548 (RHT_PERM_RW
<< LXT_PERM_SHIFT
|
552 mutex_unlock(&blka
->mutex
);
555 * The following sequence is prescribed in the SISlite spec
556 * for syncing up with the AFU when adding LXT entries.
558 dma_wmb(); /* Make LXT updates are visible */
560 rhte
->lxt_start
= lxt
;
561 dma_wmb(); /* Make RHT entry's LXT table update visible */
563 rhte
->lxt_cnt
= my_new_size
;
564 dma_wmb(); /* Make RHT entry's LXT table size update visible */
566 cxlflash_afu_sync(afu
, ctxid
, rhndl
, AFU_LW_SYNC
);
568 /* free old lxt if reallocated */
571 *new_size
= my_new_size
;
573 pr_debug("%s: returning rc=%d\n", __func__
, rc
);
578 * shrink_lxt() - reduces translation table associated with the specified RHTE
579 * @afu: AFU associated with the host.
580 * @sdev: SCSI device associated with LUN.
581 * @rhndl: Resource handle associated with the RHTE.
582 * @rhte: Resource handle entry (RHTE).
583 * @ctxi: Context owning resources.
584 * @new_size: Number of translation entries associated with RHTE.
586 * Return: 0 on success, -errno on failure
588 static int shrink_lxt(struct afu
*afu
,
589 struct scsi_device
*sdev
,
591 struct sisl_rht_entry
*rhte
,
592 struct ctx_info
*ctxi
,
595 struct sisl_lxt_entry
*lxt
, *lxt_old
;
596 struct llun_info
*lli
= sdev
->hostdata
;
597 struct glun_info
*gli
= lli
->parent
;
598 struct blka
*blka
= &gli
->blka
;
599 ctx_hndl_t ctxid
= DECODE_CTXID(ctxi
->ctxid
);
600 bool needs_ws
= ctxi
->rht_needs_ws
[rhndl
];
601 bool needs_sync
= !ctxi
->err_recovery_active
;
602 u32 ngrps
, ngrps_old
;
603 u64 aun
; /* chunk# allocated by block allocator */
604 u64 delta
= rhte
->lxt_cnt
- *new_size
;
608 lxt_old
= rhte
->lxt_start
;
609 ngrps_old
= LXT_NUM_GROUPS(rhte
->lxt_cnt
);
610 ngrps
= LXT_NUM_GROUPS(rhte
->lxt_cnt
- delta
);
612 if (ngrps
!= ngrps_old
) {
613 /* Reallocate to fit new size unless new size is 0 */
615 lxt
= kzalloc((sizeof(*lxt
) * LXT_GROUP_SIZE
* ngrps
),
617 if (unlikely(!lxt
)) {
622 /* Copy over old entries that will remain */
624 (sizeof(*lxt
) * (rhte
->lxt_cnt
- delta
)));
630 /* Nothing can fail from now on */
631 my_new_size
= rhte
->lxt_cnt
- delta
;
634 * The following sequence is prescribed in the SISlite spec
635 * for syncing up with the AFU when removing LXT entries.
637 rhte
->lxt_cnt
= my_new_size
;
638 dma_wmb(); /* Make RHT entry's LXT table size update visible */
640 rhte
->lxt_start
= lxt
;
641 dma_wmb(); /* Make RHT entry's LXT table update visible */
644 cxlflash_afu_sync(afu
, ctxid
, rhndl
, AFU_HW_SYNC
);
648 * Mark the context as unavailable, so that we can release
651 ctxi
->unavail
= true;
652 mutex_unlock(&ctxi
->mutex
);
655 /* Free LBAs allocated to freed chunks */
656 mutex_lock(&blka
->mutex
);
657 for (i
= delta
- 1; i
>= 0; i
--) {
658 /* Mask the higher 48 bits before shifting, even though
661 aun
= (lxt_old
[my_new_size
+ i
].rlba_base
& SISL_ASTATUS_MASK
);
662 aun
= (aun
>> MC_CHUNK_SHIFT
);
664 write_same16(sdev
, aun
, MC_CHUNK_SIZE
);
665 ba_free(&blka
->ba_lun
, aun
);
667 mutex_unlock(&blka
->mutex
);
670 /* Make the context visible again */
671 mutex_lock(&ctxi
->mutex
);
672 ctxi
->unavail
= false;
675 /* Free old lxt if reallocated */
678 *new_size
= my_new_size
;
680 pr_debug("%s: returning rc=%d\n", __func__
, rc
);
685 * _cxlflash_vlun_resize() - changes the size of a virtual lun
686 * @sdev: SCSI device associated with LUN owning virtual LUN.
687 * @ctxi: Context owning resources.
688 * @resize: Resize ioctl data structure.
690 * On successful return, the user is informed of the new size (in blocks)
691 * of the virtual lun in last LBA format. When the size of the virtual
692 * lun is zero, the last LBA is reflected as -1. See comment in the
693 * prologue for _cxlflash_disk_release() regarding AFU syncs and contexts
694 * on the error recovery list.
696 * Return: 0 on success, -errno on failure
698 int _cxlflash_vlun_resize(struct scsi_device
*sdev
,
699 struct ctx_info
*ctxi
,
700 struct dk_cxlflash_resize
*resize
)
702 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
703 struct llun_info
*lli
= sdev
->hostdata
;
704 struct glun_info
*gli
= lli
->parent
;
705 struct afu
*afu
= cfg
->afu
;
706 bool put_ctx
= false;
708 res_hndl_t rhndl
= resize
->rsrc_handle
;
711 u64 ctxid
= DECODE_CTXID(resize
->context_id
),
712 rctxid
= resize
->context_id
;
714 struct sisl_rht_entry
*rhte
;
719 * The requested size (req_size) is always assumed to be in 4k blocks,
720 * so we have to convert it here from 4k to chunk size.
722 nsectors
= (resize
->req_size
* CXLFLASH_BLOCK_SIZE
) / gli
->blk_len
;
723 new_size
= DIV_ROUND_UP(nsectors
, MC_CHUNK_SIZE
);
725 pr_debug("%s: ctxid=%llu rhndl=0x%llx, req_size=0x%llx,"
726 "new_size=%llx\n", __func__
, ctxid
, resize
->rsrc_handle
,
727 resize
->req_size
, new_size
);
729 if (unlikely(gli
->mode
!= MODE_VIRTUAL
)) {
730 pr_debug("%s: LUN mode does not support resize! (%d)\n",
731 __func__
, gli
->mode
);
738 ctxi
= get_context(cfg
, rctxid
, lli
, CTX_CTRL_ERR_FALLBACK
);
739 if (unlikely(!ctxi
)) {
740 pr_debug("%s: Bad context! (%llu)\n", __func__
, ctxid
);
748 rhte
= get_rhte(ctxi
, rhndl
, lli
);
749 if (unlikely(!rhte
)) {
750 pr_debug("%s: Bad resource handle! (%u)\n", __func__
, rhndl
);
755 if (new_size
> rhte
->lxt_cnt
)
756 rc
= grow_lxt(afu
, sdev
, ctxid
, rhndl
, rhte
, &new_size
);
757 else if (new_size
< rhte
->lxt_cnt
)
758 rc
= shrink_lxt(afu
, sdev
, rhndl
, rhte
, ctxi
, &new_size
);
760 resize
->hdr
.return_flags
= 0;
761 resize
->last_lba
= (new_size
* MC_CHUNK_SIZE
* gli
->blk_len
);
762 resize
->last_lba
/= CXLFLASH_BLOCK_SIZE
;
768 pr_debug("%s: resized to %lld returning rc=%d\n",
769 __func__
, resize
->last_lba
, rc
);
773 int cxlflash_vlun_resize(struct scsi_device
*sdev
,
774 struct dk_cxlflash_resize
*resize
)
776 return _cxlflash_vlun_resize(sdev
, NULL
, resize
);
780 * cxlflash_restore_luntable() - Restore LUN table to prior state
781 * @cfg: Internal structure associated with the host.
783 void cxlflash_restore_luntable(struct cxlflash_cfg
*cfg
)
785 struct llun_info
*lli
, *temp
;
788 struct afu
*afu
= cfg
->afu
;
789 struct sisl_global_map
*agm
= &afu
->afu_map
->global
;
791 mutex_lock(&global
.mutex
);
793 list_for_each_entry_safe(lli
, temp
, &cfg
->lluns
, list
) {
797 lind
= lli
->lun_index
;
799 if (lli
->port_sel
== BOTH_PORTS
) {
800 writeq_be(lli
->lun_id
[0], &agm
->fc_port
[0][lind
]);
801 writeq_be(lli
->lun_id
[1], &agm
->fc_port
[1][lind
]);
802 pr_debug("%s: Virtual LUN on slot %d id0=%llx, "
803 "id1=%llx\n", __func__
, lind
,
804 lli
->lun_id
[0], lli
->lun_id
[1]);
806 chan
= PORT2CHAN(lli
->port_sel
);
807 writeq_be(lli
->lun_id
[chan
], &agm
->fc_port
[chan
][lind
]);
808 pr_debug("%s: Virtual LUN on slot %d chan=%d, "
809 "id=%llx\n", __func__
, lind
, chan
,
814 mutex_unlock(&global
.mutex
);
818 * init_luntable() - write an entry in the LUN table
819 * @cfg: Internal structure associated with the host.
820 * @lli: Per adapter LUN information structure.
822 * On successful return, a LUN table entry is created.
823 * At the top for LUNs visible on both ports.
824 * At the bottom for LUNs visible only on one port.
826 * Return: 0 on success, -errno on failure
828 static int init_luntable(struct cxlflash_cfg
*cfg
, struct llun_info
*lli
)
833 struct afu
*afu
= cfg
->afu
;
834 struct sisl_global_map
*agm
= &afu
->afu_map
->global
;
836 mutex_lock(&global
.mutex
);
841 if (lli
->port_sel
== BOTH_PORTS
) {
843 * If this LUN is visible from both ports, we will put
844 * it in the top half of the LUN table.
846 if ((cfg
->promote_lun_index
== cfg
->last_lun_index
[0]) ||
847 (cfg
->promote_lun_index
== cfg
->last_lun_index
[1])) {
852 lind
= lli
->lun_index
= cfg
->promote_lun_index
;
853 writeq_be(lli
->lun_id
[0], &agm
->fc_port
[0][lind
]);
854 writeq_be(lli
->lun_id
[1], &agm
->fc_port
[1][lind
]);
855 cfg
->promote_lun_index
++;
856 pr_debug("%s: Virtual LUN on slot %d id0=%llx, id1=%llx\n",
857 __func__
, lind
, lli
->lun_id
[0], lli
->lun_id
[1]);
860 * If this LUN is visible only from one port, we will put
861 * it in the bottom half of the LUN table.
863 chan
= PORT2CHAN(lli
->port_sel
);
864 if (cfg
->promote_lun_index
== cfg
->last_lun_index
[chan
]) {
869 lind
= lli
->lun_index
= cfg
->last_lun_index
[chan
];
870 writeq_be(lli
->lun_id
[chan
], &agm
->fc_port
[chan
][lind
]);
871 cfg
->last_lun_index
[chan
]--;
872 pr_debug("%s: Virtual LUN on slot %d chan=%d, id=%llx\n",
873 __func__
, lind
, chan
, lli
->lun_id
[chan
]);
876 lli
->in_table
= true;
878 mutex_unlock(&global
.mutex
);
879 pr_debug("%s: returning rc=%d\n", __func__
, rc
);
884 * cxlflash_disk_virtual_open() - open a virtual disk of specified size
885 * @sdev: SCSI device associated with LUN owning virtual LUN.
886 * @arg: UVirtual ioctl data structure.
888 * On successful return, the user is informed of the resource handle
889 * to be used to identify the virtual lun and the size (in blocks) of
890 * the virtual lun in last LBA format. When the size of the virtual lun
891 * is zero, the last LBA is reflected as -1.
893 * Return: 0 on success, -errno on failure
895 int cxlflash_disk_virtual_open(struct scsi_device
*sdev
, void *arg
)
897 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
898 struct device
*dev
= &cfg
->dev
->dev
;
899 struct llun_info
*lli
= sdev
->hostdata
;
900 struct glun_info
*gli
= lli
->parent
;
902 struct dk_cxlflash_uvirtual
*virt
= (struct dk_cxlflash_uvirtual
*)arg
;
903 struct dk_cxlflash_resize resize
;
905 u64 ctxid
= DECODE_CTXID(virt
->context_id
),
906 rctxid
= virt
->context_id
;
907 u64 lun_size
= virt
->lun_size
;
909 u64 rsrc_handle
= -1;
913 struct ctx_info
*ctxi
= NULL
;
914 struct sisl_rht_entry
*rhte
= NULL
;
916 pr_debug("%s: ctxid=%llu ls=0x%llx\n", __func__
, ctxid
, lun_size
);
918 /* Setup the LUNs block allocator on first call */
919 mutex_lock(&gli
->mutex
);
920 if (gli
->mode
== MODE_NONE
) {
923 dev_err(dev
, "%s: call to init_vlun failed rc=%d!\n",
930 rc
= cxlflash_lun_attach(gli
, MODE_VIRTUAL
, true);
932 dev_err(dev
, "%s: Failed to attach to LUN! (VIRTUAL)\n",
936 mutex_unlock(&gli
->mutex
);
938 rc
= init_luntable(cfg
, lli
);
940 dev_err(dev
, "%s: call to init_luntable failed rc=%d!\n",
945 ctxi
= get_context(cfg
, rctxid
, lli
, 0);
946 if (unlikely(!ctxi
)) {
947 dev_err(dev
, "%s: Bad context! (%llu)\n", __func__
, ctxid
);
952 rhte
= rhte_checkout(ctxi
, lli
);
953 if (unlikely(!rhte
)) {
954 dev_err(dev
, "%s: too many opens for this context\n", __func__
);
955 rc
= -EMFILE
; /* too many opens */
959 rsrc_handle
= (rhte
- ctxi
->rht_start
);
961 /* Populate RHT format 0 */
962 rhte
->nmask
= MC_RHT_NMASK
;
963 rhte
->fp
= SISL_RHT_FP(0U, ctxi
->rht_perms
);
965 /* Resize even if requested size is 0 */
966 marshal_virt_to_resize(virt
, &resize
);
967 resize
.rsrc_handle
= rsrc_handle
;
968 rc
= _cxlflash_vlun_resize(sdev
, ctxi
, &resize
);
970 dev_err(dev
, "%s: resize failed rc %d\n", __func__
, rc
);
973 last_lba
= resize
.last_lba
;
975 if (virt
->hdr
.flags
& DK_CXLFLASH_UVIRTUAL_NEED_WRITE_SAME
)
976 ctxi
->rht_needs_ws
[rsrc_handle
] = true;
978 virt
->hdr
.return_flags
= 0;
979 virt
->last_lba
= last_lba
;
980 virt
->rsrc_handle
= rsrc_handle
;
985 pr_debug("%s: returning handle 0x%llx rc=%d llba %lld\n",
986 __func__
, rsrc_handle
, rc
, last_lba
);
990 rhte_checkin(ctxi
, rhte
);
992 cxlflash_lun_detach(gli
);
995 /* Special common cleanup prior to successful LUN attach */
996 cxlflash_ba_terminate(&gli
->blka
.ba_lun
);
997 mutex_unlock(&gli
->mutex
);
1002 * clone_lxt() - copies translation tables from source to destination RHTE
1003 * @afu: AFU associated with the host.
1004 * @blka: Block allocator associated with LUN.
1005 * @ctxid: Context ID of context owning the RHTE.
1006 * @rhndl: Resource handle associated with the RHTE.
1007 * @rhte: Destination resource handle entry (RHTE).
1008 * @rhte_src: Source resource handle entry (RHTE).
1010 * Return: 0 on success, -errno on failure
1012 static int clone_lxt(struct afu
*afu
,
1016 struct sisl_rht_entry
*rhte
,
1017 struct sisl_rht_entry
*rhte_src
)
1019 struct sisl_lxt_entry
*lxt
;
1021 u64 aun
; /* chunk# allocated by block allocator */
1024 ngrps
= LXT_NUM_GROUPS(rhte_src
->lxt_cnt
);
1027 /* allocate new LXTs for clone */
1028 lxt
= kzalloc((sizeof(*lxt
) * LXT_GROUP_SIZE
* ngrps
),
1034 memcpy(lxt
, rhte_src
->lxt_start
,
1035 (sizeof(*lxt
) * rhte_src
->lxt_cnt
));
1037 /* clone the LBAs in block allocator via ref_cnt */
1038 mutex_lock(&blka
->mutex
);
1039 for (i
= 0; i
< rhte_src
->lxt_cnt
; i
++) {
1040 aun
= (lxt
[i
].rlba_base
>> MC_CHUNK_SHIFT
);
1041 if (ba_clone(&blka
->ba_lun
, aun
) == -1ULL) {
1042 /* free the clones already made */
1043 for (j
= 0; j
< i
; j
++) {
1044 aun
= (lxt
[j
].rlba_base
>>
1046 ba_free(&blka
->ba_lun
, aun
);
1049 mutex_unlock(&blka
->mutex
);
1054 mutex_unlock(&blka
->mutex
);
1060 * The following sequence is prescribed in the SISlite spec
1061 * for syncing up with the AFU when adding LXT entries.
1063 dma_wmb(); /* Make LXT updates are visible */
1065 rhte
->lxt_start
= lxt
;
1066 dma_wmb(); /* Make RHT entry's LXT table update visible */
1068 rhte
->lxt_cnt
= rhte_src
->lxt_cnt
;
1069 dma_wmb(); /* Make RHT entry's LXT table size update visible */
1071 cxlflash_afu_sync(afu
, ctxid
, rhndl
, AFU_LW_SYNC
);
1073 pr_debug("%s: returning\n", __func__
);
1078 * cxlflash_disk_clone() - clone a context by making snapshot of another
1079 * @sdev: SCSI device associated with LUN owning virtual LUN.
1080 * @clone: Clone ioctl data structure.
1082 * This routine effectively performs cxlflash_disk_open operation for each
1083 * in-use virtual resource in the source context. Note that the destination
1084 * context must be in pristine state and cannot have any resource handles
1085 * open at the time of the clone.
1087 * Return: 0 on success, -errno on failure
1089 int cxlflash_disk_clone(struct scsi_device
*sdev
,
1090 struct dk_cxlflash_clone
*clone
)
1092 struct cxlflash_cfg
*cfg
= (struct cxlflash_cfg
*)sdev
->host
->hostdata
;
1093 struct llun_info
*lli
= sdev
->hostdata
;
1094 struct glun_info
*gli
= lli
->parent
;
1095 struct blka
*blka
= &gli
->blka
;
1096 struct afu
*afu
= cfg
->afu
;
1097 struct dk_cxlflash_release release
= { { 0 }, 0 };
1099 struct ctx_info
*ctxi_src
= NULL
,
1101 struct lun_access
*lun_access_src
, *lun_access_dst
;
1103 u64 ctxid_src
= DECODE_CTXID(clone
->context_id_src
),
1104 ctxid_dst
= DECODE_CTXID(clone
->context_id_dst
),
1105 rctxid_src
= clone
->context_id_src
,
1106 rctxid_dst
= clone
->context_id_dst
;
1107 int adap_fd_src
= clone
->adap_fd_src
;
1113 pr_debug("%s: ctxid_src=%llu ctxid_dst=%llu adap_fd_src=%d\n",
1114 __func__
, ctxid_src
, ctxid_dst
, adap_fd_src
);
1116 /* Do not clone yourself */
1117 if (unlikely(rctxid_src
== rctxid_dst
)) {
1122 if (unlikely(gli
->mode
!= MODE_VIRTUAL
)) {
1124 pr_debug("%s: Clone not supported on physical LUNs! (%d)\n",
1125 __func__
, gli
->mode
);
1129 ctxi_src
= get_context(cfg
, rctxid_src
, lli
, CTX_CTRL_CLONE
);
1130 ctxi_dst
= get_context(cfg
, rctxid_dst
, lli
, 0);
1131 if (unlikely(!ctxi_src
|| !ctxi_dst
)) {
1132 pr_debug("%s: Bad context! (%llu,%llu)\n", __func__
,
1133 ctxid_src
, ctxid_dst
);
1138 if (unlikely(adap_fd_src
!= ctxi_src
->lfd
)) {
1139 pr_debug("%s: Invalid source adapter fd! (%d)\n",
1140 __func__
, adap_fd_src
);
1145 /* Verify there is no open resource handle in the destination context */
1146 for (i
= 0; i
< MAX_RHT_PER_CONTEXT
; i
++)
1147 if (ctxi_dst
->rht_start
[i
].nmask
!= 0) {
1152 /* Clone LUN access list */
1153 list_for_each_entry(lun_access_src
, &ctxi_src
->luns
, list
) {
1155 list_for_each_entry(lun_access_dst
, &ctxi_dst
->luns
, list
)
1156 if (lun_access_dst
->sdev
== lun_access_src
->sdev
) {
1162 lun_access_dst
= kzalloc(sizeof(*lun_access_dst
),
1164 if (unlikely(!lun_access_dst
)) {
1165 pr_err("%s: Unable to allocate lun_access!\n",
1171 *lun_access_dst
= *lun_access_src
;
1172 list_add(&lun_access_dst
->list
, &sidecar
);
1176 if (unlikely(!ctxi_src
->rht_out
)) {
1177 pr_debug("%s: Nothing to clone!\n", __func__
);
1181 /* User specified permission on attach */
1182 perms
= ctxi_dst
->rht_perms
;
1185 * Copy over checked-out RHT (and their associated LXT) entries by
1186 * hand, stopping after we've copied all outstanding entries and
1187 * cleaning up if the clone fails.
1189 * Note: This loop is equivalent to performing cxlflash_disk_open and
1190 * cxlflash_vlun_resize. As such, LUN accounting needs to be taken into
1191 * account by attaching after each successful RHT entry clone. In the
1192 * event that a clone failure is experienced, the LUN detach is handled
1193 * via the cleanup performed by _cxlflash_disk_release.
1195 for (i
= 0; i
< MAX_RHT_PER_CONTEXT
; i
++) {
1196 if (ctxi_src
->rht_out
== ctxi_dst
->rht_out
)
1198 if (ctxi_src
->rht_start
[i
].nmask
== 0)
1201 /* Consume a destination RHT entry */
1202 ctxi_dst
->rht_out
++;
1203 ctxi_dst
->rht_start
[i
].nmask
= ctxi_src
->rht_start
[i
].nmask
;
1204 ctxi_dst
->rht_start
[i
].fp
=
1205 SISL_RHT_FP_CLONE(ctxi_src
->rht_start
[i
].fp
, perms
);
1206 ctxi_dst
->rht_lun
[i
] = ctxi_src
->rht_lun
[i
];
1208 rc
= clone_lxt(afu
, blka
, ctxid_dst
, i
,
1209 &ctxi_dst
->rht_start
[i
],
1210 &ctxi_src
->rht_start
[i
]);
1212 marshal_clone_to_rele(clone
, &release
);
1213 for (j
= 0; j
< i
; j
++) {
1214 release
.rsrc_handle
= j
;
1215 _cxlflash_disk_release(sdev
, ctxi_dst
,
1219 /* Put back the one we failed on */
1220 rhte_checkin(ctxi_dst
, &ctxi_dst
->rht_start
[i
]);
1224 cxlflash_lun_attach(gli
, gli
->mode
, false);
1228 list_splice(&sidecar
, &ctxi_dst
->luns
);
1229 sys_close(adap_fd_src
);
1234 put_context(ctxi_src
);
1236 put_context(ctxi_dst
);
1237 pr_debug("%s: returning rc=%d\n", __func__
, rc
);
1241 list_for_each_entry_safe(lun_access_src
, lun_access_dst
, &sidecar
, list
)
1242 kfree(lun_access_src
);