2 * IBM Accelerator Family 'GenWQE'
4 * (C) Copyright IBM Corp. 2013
6 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
7 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
8 * Author: Michael Jung <mijung@de.ibm.com>
9 * Author: Michael Ruettger <michael@ibmra.de>
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License (version 2 only)
13 * as published by the Free Software Foundation.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
22 * Character device representation of the GenWQE device. This allows
23 * user-space applications to communicate with the card.
26 #include <linux/kernel.h>
27 #include <linux/types.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/string.h>
32 #include <linux/sched.h>
33 #include <linux/wait.h>
34 #include <linux/delay.h>
35 #include <linux/atomic.h>
37 #include "card_base.h"
38 #include "card_ddcb.h"
40 static int genwqe_open_files(struct genwqe_dev
*cd
)
45 spin_lock_irqsave(&cd
->file_lock
, flags
);
46 rc
= list_empty(&cd
->file_list
);
47 spin_unlock_irqrestore(&cd
->file_lock
, flags
);
51 static void genwqe_add_file(struct genwqe_dev
*cd
, struct genwqe_file
*cfile
)
55 cfile
->owner
= current
;
56 spin_lock_irqsave(&cd
->file_lock
, flags
);
57 list_add(&cfile
->list
, &cd
->file_list
);
58 spin_unlock_irqrestore(&cd
->file_lock
, flags
);
61 static int genwqe_del_file(struct genwqe_dev
*cd
, struct genwqe_file
*cfile
)
65 spin_lock_irqsave(&cd
->file_lock
, flags
);
66 list_del(&cfile
->list
);
67 spin_unlock_irqrestore(&cd
->file_lock
, flags
);
72 static void genwqe_add_pin(struct genwqe_file
*cfile
, struct dma_mapping
*m
)
76 spin_lock_irqsave(&cfile
->pin_lock
, flags
);
77 list_add(&m
->pin_list
, &cfile
->pin_list
);
78 spin_unlock_irqrestore(&cfile
->pin_lock
, flags
);
81 static int genwqe_del_pin(struct genwqe_file
*cfile
, struct dma_mapping
*m
)
85 spin_lock_irqsave(&cfile
->pin_lock
, flags
);
86 list_del(&m
->pin_list
);
87 spin_unlock_irqrestore(&cfile
->pin_lock
, flags
);
93 * genwqe_search_pin() - Search for the mapping for a userspace address
94 * @cfile: Descriptor of opened file
95 * @u_addr: User virtual address
96 * @size: Size of buffer
97 * @dma_addr: DMA address to be updated
99 * Return: Pointer to the corresponding mapping NULL if not found
101 static struct dma_mapping
*genwqe_search_pin(struct genwqe_file
*cfile
,
102 unsigned long u_addr
,
107 struct dma_mapping
*m
;
109 spin_lock_irqsave(&cfile
->pin_lock
, flags
);
111 list_for_each_entry(m
, &cfile
->pin_list
, pin_list
) {
112 if ((((u64
)m
->u_vaddr
) <= (u_addr
)) &&
113 (((u64
)m
->u_vaddr
+ m
->size
) >= (u_addr
+ size
))) {
116 *virt_addr
= m
->k_vaddr
+
117 (u_addr
- (u64
)m
->u_vaddr
);
119 spin_unlock_irqrestore(&cfile
->pin_lock
, flags
);
123 spin_unlock_irqrestore(&cfile
->pin_lock
, flags
);
127 static void __genwqe_add_mapping(struct genwqe_file
*cfile
,
128 struct dma_mapping
*dma_map
)
132 spin_lock_irqsave(&cfile
->map_lock
, flags
);
133 list_add(&dma_map
->card_list
, &cfile
->map_list
);
134 spin_unlock_irqrestore(&cfile
->map_lock
, flags
);
137 static void __genwqe_del_mapping(struct genwqe_file
*cfile
,
138 struct dma_mapping
*dma_map
)
142 spin_lock_irqsave(&cfile
->map_lock
, flags
);
143 list_del(&dma_map
->card_list
);
144 spin_unlock_irqrestore(&cfile
->map_lock
, flags
);
149 * __genwqe_search_mapping() - Search for the mapping for a userspace address
150 * @cfile: descriptor of opened file
151 * @u_addr: user virtual address
152 * @size: size of buffer
153 * @dma_addr: DMA address to be updated
154 * Return: Pointer to the corresponding mapping NULL if not found
156 static struct dma_mapping
*__genwqe_search_mapping(struct genwqe_file
*cfile
,
157 unsigned long u_addr
,
159 dma_addr_t
*dma_addr
,
163 struct dma_mapping
*m
;
164 struct pci_dev
*pci_dev
= cfile
->cd
->pci_dev
;
166 spin_lock_irqsave(&cfile
->map_lock
, flags
);
167 list_for_each_entry(m
, &cfile
->map_list
, card_list
) {
169 if ((((u64
)m
->u_vaddr
) <= (u_addr
)) &&
170 (((u64
)m
->u_vaddr
+ m
->size
) >= (u_addr
+ size
))) {
172 /* match found: current is as expected and
175 *dma_addr
= m
->dma_addr
+
176 (u_addr
- (u64
)m
->u_vaddr
);
179 *virt_addr
= m
->k_vaddr
+
180 (u_addr
- (u64
)m
->u_vaddr
);
182 spin_unlock_irqrestore(&cfile
->map_lock
, flags
);
186 spin_unlock_irqrestore(&cfile
->map_lock
, flags
);
188 dev_err(&pci_dev
->dev
,
189 "[%s] Entry not found: u_addr=%lx, size=%x\n",
190 __func__
, u_addr
, size
);
195 static void genwqe_remove_mappings(struct genwqe_file
*cfile
)
198 struct list_head
*node
, *next
;
199 struct dma_mapping
*dma_map
;
200 struct genwqe_dev
*cd
= cfile
->cd
;
201 struct pci_dev
*pci_dev
= cfile
->cd
->pci_dev
;
203 list_for_each_safe(node
, next
, &cfile
->map_list
) {
204 dma_map
= list_entry(node
, struct dma_mapping
, card_list
);
206 list_del_init(&dma_map
->card_list
);
209 * This is really a bug, because those things should
210 * have been already tidied up.
212 * GENWQE_MAPPING_RAW should have been removed via mmunmap().
213 * GENWQE_MAPPING_SGL_TEMP should be removed by tidy up code.
215 dev_err(&pci_dev
->dev
,
216 "[%s] %d. cleanup mapping: u_vaddr=%p "
217 "u_kaddr=%016lx dma_addr=%lx\n", __func__
, i
++,
218 dma_map
->u_vaddr
, (unsigned long)dma_map
->k_vaddr
,
219 (unsigned long)dma_map
->dma_addr
);
221 if (dma_map
->type
== GENWQE_MAPPING_RAW
) {
222 /* we allocated this dynamically */
223 __genwqe_free_consistent(cd
, dma_map
->size
,
227 } else if (dma_map
->type
== GENWQE_MAPPING_SGL_TEMP
) {
228 /* we use dma_map statically from the request */
229 genwqe_user_vunmap(cd
, dma_map
, NULL
);
234 static void genwqe_remove_pinnings(struct genwqe_file
*cfile
)
236 struct list_head
*node
, *next
;
237 struct dma_mapping
*dma_map
;
238 struct genwqe_dev
*cd
= cfile
->cd
;
240 list_for_each_safe(node
, next
, &cfile
->pin_list
) {
241 dma_map
= list_entry(node
, struct dma_mapping
, pin_list
);
244 * This is not a bug, because a killed processed might
245 * not call the unpin ioctl, which is supposed to free
248 * Pinnings are dymically allocated and need to be
251 list_del_init(&dma_map
->pin_list
);
252 genwqe_user_vunmap(cd
, dma_map
, NULL
);
258 * genwqe_kill_fasync() - Send signal to all processes with open GenWQE files
260 * E.g. genwqe_send_signal(cd, SIGIO);
262 static int genwqe_kill_fasync(struct genwqe_dev
*cd
, int sig
)
264 unsigned int files
= 0;
266 struct genwqe_file
*cfile
;
268 spin_lock_irqsave(&cd
->file_lock
, flags
);
269 list_for_each_entry(cfile
, &cd
->file_list
, list
) {
270 if (cfile
->async_queue
)
271 kill_fasync(&cfile
->async_queue
, sig
, POLL_HUP
);
274 spin_unlock_irqrestore(&cd
->file_lock
, flags
);
278 static int genwqe_force_sig(struct genwqe_dev
*cd
, int sig
)
280 unsigned int files
= 0;
282 struct genwqe_file
*cfile
;
284 spin_lock_irqsave(&cd
->file_lock
, flags
);
285 list_for_each_entry(cfile
, &cd
->file_list
, list
) {
286 force_sig(sig
, cfile
->owner
);
289 spin_unlock_irqrestore(&cd
->file_lock
, flags
);
294 * genwqe_open() - file open
295 * @inode: file system information
298 * This function is executed whenever an application calls
299 * open("/dev/genwqe",..).
301 * Return: 0 if successful or <0 if errors
303 static int genwqe_open(struct inode
*inode
, struct file
*filp
)
305 struct genwqe_dev
*cd
;
306 struct genwqe_file
*cfile
;
307 struct pci_dev
*pci_dev
;
309 cfile
= kzalloc(sizeof(*cfile
), GFP_KERNEL
);
313 cd
= container_of(inode
->i_cdev
, struct genwqe_dev
, cdev_genwqe
);
314 pci_dev
= cd
->pci_dev
;
317 cfile
->client
= NULL
;
319 spin_lock_init(&cfile
->map_lock
); /* list of raw memory allocations */
320 INIT_LIST_HEAD(&cfile
->map_list
);
322 spin_lock_init(&cfile
->pin_lock
); /* list of user pinned memory */
323 INIT_LIST_HEAD(&cfile
->pin_list
);
325 filp
->private_data
= cfile
;
327 genwqe_add_file(cd
, cfile
);
332 * genwqe_fasync() - Setup process to receive SIGIO.
333 * @fd: file descriptor
337 * Sending a signal is working as following:
339 * if (cdev->async_queue)
340 * kill_fasync(&cdev->async_queue, SIGIO, POLL_IN);
342 * Some devices also implement asynchronous notification to indicate
343 * when the device can be written; in this case, of course,
344 * kill_fasync must be called with a mode of POLL_OUT.
346 static int genwqe_fasync(int fd
, struct file
*filp
, int mode
)
348 struct genwqe_file
*cdev
= (struct genwqe_file
*)filp
->private_data
;
349 return fasync_helper(fd
, filp
, mode
, &cdev
->async_queue
);
354 * genwqe_release() - file close
355 * @inode: file system information
358 * This function is executed whenever an application calls 'close(fd_genwqe)'
362 static int genwqe_release(struct inode
*inode
, struct file
*filp
)
364 struct genwqe_file
*cfile
= (struct genwqe_file
*)filp
->private_data
;
365 struct genwqe_dev
*cd
= cfile
->cd
;
367 /* there must be no entries in these lists! */
368 genwqe_remove_mappings(cfile
);
369 genwqe_remove_pinnings(cfile
);
371 /* remove this filp from the asynchronously notified filp's */
372 genwqe_fasync(-1, filp
, 0);
375 * For this to work we must not release cd when this cfile is
376 * not yet released, otherwise the list entry is invalid,
377 * because the list itself gets reinstantiated!
379 genwqe_del_file(cd
, cfile
);
384 static void genwqe_vma_open(struct vm_area_struct
*vma
)
390 * genwqe_vma_close() - Called each time when vma is unmapped
392 * Free memory which got allocated by GenWQE mmap().
394 static void genwqe_vma_close(struct vm_area_struct
*vma
)
396 unsigned long vsize
= vma
->vm_end
- vma
->vm_start
;
397 struct inode
*inode
= vma
->vm_file
->f_dentry
->d_inode
;
398 struct dma_mapping
*dma_map
;
399 struct genwqe_dev
*cd
= container_of(inode
->i_cdev
, struct genwqe_dev
,
401 struct pci_dev
*pci_dev
= cd
->pci_dev
;
402 dma_addr_t d_addr
= 0;
403 struct genwqe_file
*cfile
= vma
->vm_private_data
;
405 dma_map
= __genwqe_search_mapping(cfile
, vma
->vm_start
, vsize
,
407 if (dma_map
== NULL
) {
408 dev_err(&pci_dev
->dev
,
409 " [%s] err: mapping not found: v=%lx, p=%lx s=%lx\n",
410 __func__
, vma
->vm_start
, vma
->vm_pgoff
<< PAGE_SHIFT
,
414 __genwqe_del_mapping(cfile
, dma_map
);
415 __genwqe_free_consistent(cd
, dma_map
->size
, dma_map
->k_vaddr
,
420 static struct vm_operations_struct genwqe_vma_ops
= {
421 .open
= genwqe_vma_open
,
422 .close
= genwqe_vma_close
,
426 * genwqe_mmap() - Provide contignous buffers to userspace
428 * We use mmap() to allocate contignous buffers used for DMA
429 * transfers. After the buffer is allocated we remap it to user-space
430 * and remember a reference to our dma_mapping data structure, where
431 * we store the associated DMA address and allocated size.
433 * When we receive a DDCB execution request with the ATS bits set to
434 * plain buffer, we lookup our dma_mapping list to find the
435 * corresponding DMA address for the associated user-space address.
437 static int genwqe_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
440 unsigned long pfn
, vsize
= vma
->vm_end
- vma
->vm_start
;
441 struct genwqe_file
*cfile
= (struct genwqe_file
*)filp
->private_data
;
442 struct genwqe_dev
*cd
= cfile
->cd
;
443 struct dma_mapping
*dma_map
;
448 if (get_order(vsize
) > MAX_ORDER
)
451 dma_map
= kzalloc(sizeof(struct dma_mapping
), GFP_ATOMIC
);
455 genwqe_mapping_init(dma_map
, GENWQE_MAPPING_RAW
);
456 dma_map
->u_vaddr
= (void *)vma
->vm_start
;
457 dma_map
->size
= vsize
;
458 dma_map
->nr_pages
= DIV_ROUND_UP(vsize
, PAGE_SIZE
);
459 dma_map
->k_vaddr
= __genwqe_alloc_consistent(cd
, vsize
,
461 if (dma_map
->k_vaddr
== NULL
) {
466 if (capable(CAP_SYS_ADMIN
) && (vsize
> sizeof(dma_addr_t
)))
467 *(dma_addr_t
*)dma_map
->k_vaddr
= dma_map
->dma_addr
;
469 pfn
= virt_to_phys(dma_map
->k_vaddr
) >> PAGE_SHIFT
;
470 rc
= remap_pfn_range(vma
,
480 vma
->vm_private_data
= cfile
;
481 vma
->vm_ops
= &genwqe_vma_ops
;
482 __genwqe_add_mapping(cfile
, dma_map
);
487 __genwqe_free_consistent(cd
, dma_map
->size
,
496 * do_flash_update() - Excute flash update (write image or CVPD)
498 * @load: details about image load
500 * Return: 0 if successful
503 #define FLASH_BLOCK 0x40000 /* we use 256k blocks */
505 static int do_flash_update(struct genwqe_file
*cfile
,
506 struct genwqe_bitstream
*load
)
517 struct genwqe_dev
*cd
= cfile
->cd
;
518 struct pci_dev
*pci_dev
= cd
->pci_dev
;
520 if ((load
->size
& 0x3) != 0)
523 if (((unsigned long)(load
->data_addr
) & ~PAGE_MASK
) != 0)
526 /* FIXME Bits have changed for new service layer! */
527 switch ((char)load
->partition
) {
530 break; /* download/erase_first/part_0 */
533 break; /* download/erase_first/part_1 */
536 break; /* download/erase_first/vpd */
541 buf
= (u8 __user
*)load
->data_addr
;
542 xbuf
= __genwqe_alloc_consistent(cd
, FLASH_BLOCK
, &dma_addr
);
546 blocks_to_flash
= load
->size
/ FLASH_BLOCK
;
548 struct genwqe_ddcb_cmd
*req
;
551 * We must be 4 byte aligned. Buffer must be 0 appened
552 * to have defined values when calculating CRC.
554 tocopy
= min_t(size_t, load
->size
, FLASH_BLOCK
);
556 rc
= copy_from_user(xbuf
, buf
, tocopy
);
561 crc
= genwqe_crc32(xbuf
, tocopy
, 0xffffffff);
563 dev_dbg(&pci_dev
->dev
,
564 "[%s] DMA: %lx CRC: %08x SZ: %ld %d\n",
565 __func__
, (unsigned long)dma_addr
, crc
, tocopy
,
568 /* prepare DDCB for SLU process */
569 req
= ddcb_requ_alloc();
575 req
->cmd
= SLCMD_MOVE_FLASH
;
576 req
->cmdopts
= cmdopts
;
578 /* prepare invariant values */
579 if (genwqe_get_slu_id(cd
) <= 0x2) {
580 *(__be64
*)&req
->__asiv
[0] = cpu_to_be64(dma_addr
);
581 *(__be64
*)&req
->__asiv
[8] = cpu_to_be64(tocopy
);
582 *(__be64
*)&req
->__asiv
[16] = cpu_to_be64(flash
);
583 *(__be32
*)&req
->__asiv
[24] = cpu_to_be32(0);
584 req
->__asiv
[24] = load
->uid
;
585 *(__be32
*)&req
->__asiv
[28] = cpu_to_be32(crc
);
587 /* for simulation only */
588 *(__be64
*)&req
->__asiv
[88] = cpu_to_be64(load
->slu_id
);
589 *(__be64
*)&req
->__asiv
[96] = cpu_to_be64(load
->app_id
);
590 req
->asiv_length
= 32; /* bytes included in crc calc */
591 } else { /* setup DDCB for ATS architecture */
592 *(__be64
*)&req
->asiv
[0] = cpu_to_be64(dma_addr
);
593 *(__be32
*)&req
->asiv
[8] = cpu_to_be32(tocopy
);
594 *(__be32
*)&req
->asiv
[12] = cpu_to_be32(0); /* resvd */
595 *(__be64
*)&req
->asiv
[16] = cpu_to_be64(flash
);
596 *(__be32
*)&req
->asiv
[24] = cpu_to_be32(load
->uid
<<24);
597 *(__be32
*)&req
->asiv
[28] = cpu_to_be32(crc
);
599 /* for simulation only */
600 *(__be64
*)&req
->asiv
[80] = cpu_to_be64(load
->slu_id
);
601 *(__be64
*)&req
->asiv
[88] = cpu_to_be64(load
->app_id
);
604 req
->ats
= 0x4ULL
<< 44;
605 req
->asiv_length
= 40; /* bytes included in crc calc */
609 /* For Genwqe5 we get back the calculated CRC */
610 *(u64
*)&req
->asv
[0] = 0ULL; /* 0x80 */
612 rc
= __genwqe_execute_raw_ddcb(cd
, req
);
614 load
->retc
= req
->retc
;
615 load
->attn
= req
->attn
;
616 load
->progress
= req
->progress
;
623 if (req
->retc
!= DDCB_RETC_COMPLETE
) {
629 load
->size
-= tocopy
;
637 __genwqe_free_consistent(cd
, FLASH_BLOCK
, xbuf
, dma_addr
);
641 static int do_flash_read(struct genwqe_file
*cfile
,
642 struct genwqe_bitstream
*load
)
644 int rc
, blocks_to_flash
;
651 struct genwqe_dev
*cd
= cfile
->cd
;
652 struct pci_dev
*pci_dev
= cd
->pci_dev
;
653 struct genwqe_ddcb_cmd
*cmd
;
655 if ((load
->size
& 0x3) != 0)
658 if (((unsigned long)(load
->data_addr
) & ~PAGE_MASK
) != 0)
661 /* FIXME Bits have changed for new service layer! */
662 switch ((char)load
->partition
) {
665 break; /* upload/part_0 */
668 break; /* upload/part_1 */
671 break; /* upload/vpd */
676 buf
= (u8 __user
*)load
->data_addr
;
677 xbuf
= __genwqe_alloc_consistent(cd
, FLASH_BLOCK
, &dma_addr
);
681 blocks_to_flash
= load
->size
/ FLASH_BLOCK
;
684 * We must be 4 byte aligned. Buffer must be 0 appened
685 * to have defined values when calculating CRC.
687 tocopy
= min_t(size_t, load
->size
, FLASH_BLOCK
);
689 dev_dbg(&pci_dev
->dev
,
690 "[%s] DMA: %lx SZ: %ld %d\n",
691 __func__
, (unsigned long)dma_addr
, tocopy
,
694 /* prepare DDCB for SLU process */
695 cmd
= ddcb_requ_alloc();
700 cmd
->cmd
= SLCMD_MOVE_FLASH
;
701 cmd
->cmdopts
= cmdopts
;
703 /* prepare invariant values */
704 if (genwqe_get_slu_id(cd
) <= 0x2) {
705 *(__be64
*)&cmd
->__asiv
[0] = cpu_to_be64(dma_addr
);
706 *(__be64
*)&cmd
->__asiv
[8] = cpu_to_be64(tocopy
);
707 *(__be64
*)&cmd
->__asiv
[16] = cpu_to_be64(flash
);
708 *(__be32
*)&cmd
->__asiv
[24] = cpu_to_be32(0);
709 cmd
->__asiv
[24] = load
->uid
;
710 *(__be32
*)&cmd
->__asiv
[28] = cpu_to_be32(0) /* CRC */;
711 cmd
->asiv_length
= 32; /* bytes included in crc calc */
712 } else { /* setup DDCB for ATS architecture */
713 *(__be64
*)&cmd
->asiv
[0] = cpu_to_be64(dma_addr
);
714 *(__be32
*)&cmd
->asiv
[8] = cpu_to_be32(tocopy
);
715 *(__be32
*)&cmd
->asiv
[12] = cpu_to_be32(0); /* resvd */
716 *(__be64
*)&cmd
->asiv
[16] = cpu_to_be64(flash
);
717 *(__be32
*)&cmd
->asiv
[24] = cpu_to_be32(load
->uid
<<24);
718 *(__be32
*)&cmd
->asiv
[28] = cpu_to_be32(0); /* CRC */
721 cmd
->ats
= 0x5ULL
<< 44;
722 cmd
->asiv_length
= 40; /* bytes included in crc calc */
726 /* we only get back the calculated CRC */
727 *(u64
*)&cmd
->asv
[0] = 0ULL; /* 0x80 */
729 rc
= __genwqe_execute_raw_ddcb(cd
, cmd
);
731 load
->retc
= cmd
->retc
;
732 load
->attn
= cmd
->attn
;
733 load
->progress
= cmd
->progress
;
735 if ((rc
< 0) && (rc
!= -EBADMSG
)) {
740 rc
= copy_to_user(buf
, xbuf
, tocopy
);
747 /* We know that we can get retc 0x104 with CRC err */
748 if (((cmd
->retc
== DDCB_RETC_FAULT
) &&
749 (cmd
->attn
!= 0x02)) || /* Normally ignore CRC error */
750 ((cmd
->retc
== DDCB_RETC_COMPLETE
) &&
751 (cmd
->attn
!= 0x00))) { /* Everything was fine */
757 load
->size
-= tocopy
;
766 __genwqe_free_consistent(cd
, FLASH_BLOCK
, xbuf
, dma_addr
);
770 static int genwqe_pin_mem(struct genwqe_file
*cfile
, struct genwqe_mem
*m
)
773 struct genwqe_dev
*cd
= cfile
->cd
;
774 struct pci_dev
*pci_dev
= cfile
->cd
->pci_dev
;
775 struct dma_mapping
*dma_map
;
776 unsigned long map_addr
;
777 unsigned long map_size
;
779 if ((m
->addr
== 0x0) || (m
->size
== 0))
782 map_addr
= (m
->addr
& PAGE_MASK
);
783 map_size
= round_up(m
->size
+ (m
->addr
& ~PAGE_MASK
), PAGE_SIZE
);
785 dma_map
= kzalloc(sizeof(struct dma_mapping
), GFP_ATOMIC
);
789 genwqe_mapping_init(dma_map
, GENWQE_MAPPING_SGL_PINNED
);
790 rc
= genwqe_user_vmap(cd
, dma_map
, (void *)map_addr
, map_size
, NULL
);
792 dev_err(&pci_dev
->dev
,
793 "[%s] genwqe_user_vmap rc=%d\n", __func__
, rc
);
798 genwqe_add_pin(cfile
, dma_map
);
802 static int genwqe_unpin_mem(struct genwqe_file
*cfile
, struct genwqe_mem
*m
)
804 struct genwqe_dev
*cd
= cfile
->cd
;
805 struct dma_mapping
*dma_map
;
806 unsigned long map_addr
;
807 unsigned long map_size
;
812 map_addr
= (m
->addr
& PAGE_MASK
);
813 map_size
= round_up(m
->size
+ (m
->addr
& ~PAGE_MASK
), PAGE_SIZE
);
815 dma_map
= genwqe_search_pin(cfile
, map_addr
, map_size
, NULL
);
819 genwqe_del_pin(cfile
, dma_map
);
820 genwqe_user_vunmap(cd
, dma_map
, NULL
);
826 * ddcb_cmd_cleanup() - Remove dynamically created fixup entries
828 * Only if there are any. Pinnings are not removed.
830 static int ddcb_cmd_cleanup(struct genwqe_file
*cfile
, struct ddcb_requ
*req
)
833 struct dma_mapping
*dma_map
;
834 struct genwqe_dev
*cd
= cfile
->cd
;
836 for (i
= 0; i
< DDCB_FIXUPS
; i
++) {
837 dma_map
= &req
->dma_mappings
[i
];
839 if (dma_mapping_used(dma_map
)) {
840 __genwqe_del_mapping(cfile
, dma_map
);
841 genwqe_user_vunmap(cd
, dma_map
, req
);
843 if (req
->sgl
[i
] != NULL
) {
844 genwqe_free_sgl(cd
, req
->sgl
[i
],
845 req
->sgl_dma_addr
[i
],
848 req
->sgl_dma_addr
[i
] = 0x0;
849 req
->sgl_size
[i
] = 0;
857 * ddcb_cmd_fixups() - Establish DMA fixups/sglists for user memory references
859 * Before the DDCB gets executed we need to handle the fixups. We
860 * replace the user-space addresses with DMA addresses or do
861 * additional setup work e.g. generating a scatter-gather list which
862 * is used to describe the memory referred to in the fixup.
864 static int ddcb_cmd_fixups(struct genwqe_file
*cfile
, struct ddcb_requ
*req
)
867 unsigned int asiv_offs
, i
;
868 struct genwqe_dev
*cd
= cfile
->cd
;
869 struct genwqe_ddcb_cmd
*cmd
= &req
->cmd
;
870 struct dma_mapping
*m
;
871 const char *type
= "UNKNOWN";
873 for (i
= 0, asiv_offs
= 0x00; asiv_offs
<= 0x58;
874 i
++, asiv_offs
+= 0x08) {
881 ats_flags
= ATS_GET_FLAGS(cmd
->ats
, asiv_offs
);
886 break; /* nothing to do here */
888 case ATS_TYPE_FLAT_RDWR
:
889 case ATS_TYPE_FLAT_RD
: {
890 u_addr
= be64_to_cpu(*((__be64
*)&cmd
->
892 u_size
= be32_to_cpu(*((__be32
*)&cmd
->
893 asiv
[asiv_offs
+ 0x08]));
896 * No data available. Ignore u_addr in this
897 * case and set addr to 0. Hardware must not
901 *((__be64
*)&cmd
->asiv
[asiv_offs
]) =
906 m
= __genwqe_search_mapping(cfile
, u_addr
, u_size
,
913 *((__be64
*)&cmd
->asiv
[asiv_offs
]) =
918 case ATS_TYPE_SGL_RDWR
:
919 case ATS_TYPE_SGL_RD
: {
920 int page_offs
, nr_pages
, offs
;
922 u_addr
= be64_to_cpu(*((__be64
*)
923 &cmd
->asiv
[asiv_offs
]));
924 u_size
= be32_to_cpu(*((__be32
*)
925 &cmd
->asiv
[asiv_offs
+ 0x08]));
928 * No data available. Ignore u_addr in this
929 * case and set addr to 0. Hardware must not
930 * fetch the empty sgl.
933 *((__be64
*)&cmd
->asiv
[asiv_offs
]) =
938 m
= genwqe_search_pin(cfile
, u_addr
, u_size
, NULL
);
941 page_offs
= (u_addr
-
942 (u64
)m
->u_vaddr
)/PAGE_SIZE
;
945 m
= &req
->dma_mappings
[i
];
947 genwqe_mapping_init(m
,
948 GENWQE_MAPPING_SGL_TEMP
);
949 rc
= genwqe_user_vmap(cd
, m
, (void *)u_addr
,
954 __genwqe_add_mapping(cfile
, m
);
958 offs
= offset_in_page(u_addr
);
959 nr_pages
= DIV_ROUND_UP(offs
+ u_size
, PAGE_SIZE
);
961 /* create genwqe style scatter gather list */
962 req
->sgl
[i
] = genwqe_alloc_sgl(cd
, m
->nr_pages
,
963 &req
->sgl_dma_addr
[i
],
965 if (req
->sgl
[i
] == NULL
) {
969 genwqe_setup_sgl(cd
, offs
, u_size
,
971 req
->sgl_dma_addr
[i
],
977 *((__be64
*)&cmd
->asiv
[asiv_offs
]) =
978 cpu_to_be64(req
->sgl_dma_addr
[i
]);
990 ddcb_cmd_cleanup(cfile
, req
);
995 * genwqe_execute_ddcb() - Execute DDCB using userspace address fixups
997 * The code will build up the translation tables or lookup the
998 * contignous memory allocation table to find the right translations
1001 static int genwqe_execute_ddcb(struct genwqe_file
*cfile
,
1002 struct genwqe_ddcb_cmd
*cmd
)
1005 struct genwqe_dev
*cd
= cfile
->cd
;
1006 struct ddcb_requ
*req
= container_of(cmd
, struct ddcb_requ
, cmd
);
1008 rc
= ddcb_cmd_fixups(cfile
, req
);
1012 rc
= __genwqe_execute_raw_ddcb(cd
, cmd
);
1013 ddcb_cmd_cleanup(cfile
, req
);
1017 static int do_execute_ddcb(struct genwqe_file
*cfile
,
1018 unsigned long arg
, int raw
)
1021 struct genwqe_ddcb_cmd
*cmd
;
1022 struct ddcb_requ
*req
;
1023 struct genwqe_dev
*cd
= cfile
->cd
;
1025 cmd
= ddcb_requ_alloc();
1029 req
= container_of(cmd
, struct ddcb_requ
, cmd
);
1031 if (copy_from_user(cmd
, (void __user
*)arg
, sizeof(*cmd
))) {
1032 ddcb_requ_free(cmd
);
1037 rc
= genwqe_execute_ddcb(cfile
, cmd
);
1039 rc
= __genwqe_execute_raw_ddcb(cd
, cmd
);
1041 /* Copy back only the modifed fields. Do not copy ASIV
1042 back since the copy got modified by the driver. */
1043 if (copy_to_user((void __user
*)arg
, cmd
,
1044 sizeof(*cmd
) - DDCB_ASIV_LENGTH
)) {
1045 ddcb_requ_free(cmd
);
1049 ddcb_requ_free(cmd
);
1054 * genwqe_ioctl() - IO control
1055 * @filp: file handle
1056 * @cmd: command identifier (passed from user)
1057 * @arg: argument (passed from user)
1061 static long genwqe_ioctl(struct file
*filp
, unsigned int cmd
,
1065 struct genwqe_file
*cfile
= (struct genwqe_file
*)filp
->private_data
;
1066 struct genwqe_dev
*cd
= cfile
->cd
;
1067 struct genwqe_reg_io __user
*io
;
1071 if (_IOC_TYPE(cmd
) != GENWQE_IOC_CODE
)
1076 case GENWQE_GET_CARD_STATE
:
1077 put_user(cd
->card_state
, (enum genwqe_card_state __user
*)arg
);
1080 /* Register access */
1081 case GENWQE_READ_REG64
: {
1082 io
= (struct genwqe_reg_io __user
*)arg
;
1084 if (get_user(reg_offs
, &io
->num
))
1087 if ((reg_offs
>= cd
->mmio_len
) || (reg_offs
& 0x7))
1090 val
= __genwqe_readq(cd
, reg_offs
);
1091 put_user(val
, &io
->val64
);
1095 case GENWQE_WRITE_REG64
: {
1096 io
= (struct genwqe_reg_io __user
*)arg
;
1098 if (!capable(CAP_SYS_ADMIN
))
1101 if ((filp
->f_flags
& O_ACCMODE
) == O_RDONLY
)
1104 if (get_user(reg_offs
, &io
->num
))
1107 if ((reg_offs
>= cd
->mmio_len
) || (reg_offs
& 0x7))
1110 if (get_user(val
, &io
->val64
))
1113 __genwqe_writeq(cd
, reg_offs
, val
);
1117 case GENWQE_READ_REG32
: {
1118 io
= (struct genwqe_reg_io __user
*)arg
;
1120 if (get_user(reg_offs
, &io
->num
))
1123 if ((reg_offs
>= cd
->mmio_len
) || (reg_offs
& 0x3))
1126 val
= __genwqe_readl(cd
, reg_offs
);
1127 put_user(val
, &io
->val64
);
1131 case GENWQE_WRITE_REG32
: {
1132 io
= (struct genwqe_reg_io __user
*)arg
;
1134 if (!capable(CAP_SYS_ADMIN
))
1137 if ((filp
->f_flags
& O_ACCMODE
) == O_RDONLY
)
1140 if (get_user(reg_offs
, &io
->num
))
1143 if ((reg_offs
>= cd
->mmio_len
) || (reg_offs
& 0x3))
1146 if (get_user(val
, &io
->val64
))
1149 __genwqe_writel(cd
, reg_offs
, val
);
1153 /* Flash update/reading */
1154 case GENWQE_SLU_UPDATE
: {
1155 struct genwqe_bitstream load
;
1157 if (!genwqe_is_privileged(cd
))
1160 if ((filp
->f_flags
& O_ACCMODE
) == O_RDONLY
)
1163 if (copy_from_user(&load
, (void __user
*)arg
,
1167 rc
= do_flash_update(cfile
, &load
);
1169 if (copy_to_user((void __user
*)arg
, &load
, sizeof(load
)))
1175 case GENWQE_SLU_READ
: {
1176 struct genwqe_bitstream load
;
1178 if (!genwqe_is_privileged(cd
))
1181 if (genwqe_flash_readback_fails(cd
))
1182 return -ENOSPC
; /* known to fail for old versions */
1184 if (copy_from_user(&load
, (void __user
*)arg
, sizeof(load
)))
1187 rc
= do_flash_read(cfile
, &load
);
1189 if (copy_to_user((void __user
*)arg
, &load
, sizeof(load
)))
1195 /* memory pinning and unpinning */
1196 case GENWQE_PIN_MEM
: {
1197 struct genwqe_mem m
;
1199 if (copy_from_user(&m
, (void __user
*)arg
, sizeof(m
)))
1202 return genwqe_pin_mem(cfile
, &m
);
1205 case GENWQE_UNPIN_MEM
: {
1206 struct genwqe_mem m
;
1208 if (copy_from_user(&m
, (void __user
*)arg
, sizeof(m
)))
1211 return genwqe_unpin_mem(cfile
, &m
);
1214 /* launch an DDCB and wait for completion */
1215 case GENWQE_EXECUTE_DDCB
:
1216 return do_execute_ddcb(cfile
, arg
, 0);
1218 case GENWQE_EXECUTE_RAW_DDCB
: {
1220 if (!capable(CAP_SYS_ADMIN
))
1223 return do_execute_ddcb(cfile
, arg
, 1);
1233 #if defined(CONFIG_COMPAT)
1235 * genwqe_compat_ioctl() - Compatibility ioctl
1237 * Called whenever a 32-bit process running under a 64-bit kernel
1238 * performs an ioctl on /dev/genwqe<n>_card.
1240 * @filp: file pointer.
1242 * @arg: user argument.
1243 * Return: zero on success or negative number on failure.
1245 static long genwqe_compat_ioctl(struct file
*filp
, unsigned int cmd
,
1248 return genwqe_ioctl(filp
, cmd
, arg
);
1250 #endif /* defined(CONFIG_COMPAT) */
1252 static const struct file_operations genwqe_fops
= {
1253 .owner
= THIS_MODULE
,
1254 .open
= genwqe_open
,
1255 .fasync
= genwqe_fasync
,
1256 .mmap
= genwqe_mmap
,
1257 .unlocked_ioctl
= genwqe_ioctl
,
1258 #if defined(CONFIG_COMPAT)
1259 .compat_ioctl
= genwqe_compat_ioctl
,
1261 .release
= genwqe_release
,
1264 static int genwqe_device_initialized(struct genwqe_dev
*cd
)
1266 return cd
->dev
!= NULL
;
1270 * genwqe_device_create() - Create and configure genwqe char device
1271 * @cd: genwqe device descriptor
1273 * This function must be called before we create any more genwqe
1274 * character devices, because it is allocating the major and minor
1275 * number which are supposed to be used by the client drivers.
1277 int genwqe_device_create(struct genwqe_dev
*cd
)
1280 struct pci_dev
*pci_dev
= cd
->pci_dev
;
1283 * Here starts the individual setup per client. It must
1284 * initialize its own cdev data structure with its own fops.
1285 * The appropriate devnum needs to be created. The ranges must
1288 rc
= alloc_chrdev_region(&cd
->devnum_genwqe
, 0,
1289 GENWQE_MAX_MINOR
, GENWQE_DEVNAME
);
1291 dev_err(&pci_dev
->dev
, "err: alloc_chrdev_region failed\n");
1295 cdev_init(&cd
->cdev_genwqe
, &genwqe_fops
);
1296 cd
->cdev_genwqe
.owner
= THIS_MODULE
;
1298 rc
= cdev_add(&cd
->cdev_genwqe
, cd
->devnum_genwqe
, 1);
1300 dev_err(&pci_dev
->dev
, "err: cdev_add failed\n");
1305 * Finally the device in /dev/... must be created. The rule is
1306 * to use card%d_clientname for each created device.
1308 cd
->dev
= device_create_with_groups(cd
->class_genwqe
,
1310 cd
->devnum_genwqe
, cd
,
1311 genwqe_attribute_groups
,
1312 GENWQE_DEVNAME
"%u_card",
1314 if (IS_ERR(cd
->dev
)) {
1315 rc
= PTR_ERR(cd
->dev
);
1319 rc
= genwqe_init_debugfs(cd
);
1326 device_destroy(cd
->class_genwqe
, cd
->devnum_genwqe
);
1328 cdev_del(&cd
->cdev_genwqe
);
1330 unregister_chrdev_region(cd
->devnum_genwqe
, GENWQE_MAX_MINOR
);
1336 static int genwqe_inform_and_stop_processes(struct genwqe_dev
*cd
)
1340 struct pci_dev
*pci_dev
= cd
->pci_dev
;
1342 if (!genwqe_open_files(cd
))
1345 dev_warn(&pci_dev
->dev
, "[%s] send SIGIO and wait ...\n", __func__
);
1347 rc
= genwqe_kill_fasync(cd
, SIGIO
);
1349 /* give kill_timeout seconds to close file descriptors ... */
1350 for (i
= 0; (i
< genwqe_kill_timeout
) &&
1351 genwqe_open_files(cd
); i
++) {
1352 dev_info(&pci_dev
->dev
, " %d sec ...", i
);
1358 /* if no open files we can safely continue, else ... */
1359 if (!genwqe_open_files(cd
))
1362 dev_warn(&pci_dev
->dev
,
1363 "[%s] send SIGKILL and wait ...\n", __func__
);
1365 rc
= genwqe_force_sig(cd
, SIGKILL
); /* force terminate */
1367 /* Give kill_timout more seconds to end processes */
1368 for (i
= 0; (i
< genwqe_kill_timeout
) &&
1369 genwqe_open_files(cd
); i
++) {
1370 dev_warn(&pci_dev
->dev
, " %d sec ...", i
);
1381 * genwqe_device_remove() - Remove genwqe's char device
1383 * This function must be called after the client devices are removed
1384 * because it will free the major/minor number range for the genwqe
1387 * This function must be robust enough to be called twice.
1389 int genwqe_device_remove(struct genwqe_dev
*cd
)
1392 struct pci_dev
*pci_dev
= cd
->pci_dev
;
1394 if (!genwqe_device_initialized(cd
))
1397 genwqe_inform_and_stop_processes(cd
);
1400 * We currently do wait until all filedescriptors are
1401 * closed. This leads to a problem when we abort the
1402 * application which will decrease this reference from
1403 * 1/unused to 0/illegal and not from 2/used 1/empty.
1405 rc
= atomic_read(&cd
->cdev_genwqe
.kobj
.kref
.refcount
);
1407 dev_err(&pci_dev
->dev
,
1408 "[%s] err: cdev_genwqe...refcount=%d\n", __func__
, rc
);
1409 panic("Fatal err: cannot free resources with pending references!");
1412 genqwe_exit_debugfs(cd
);
1413 device_destroy(cd
->class_genwqe
, cd
->devnum_genwqe
);
1414 cdev_del(&cd
->cdev_genwqe
);
1415 unregister_chrdev_region(cd
->devnum_genwqe
, GENWQE_MAX_MINOR
);