1 // SPDX-License-Identifier: GPL-2.0
3 * ccw based virtio transport
5 * Copyright IBM Corp. 2012, 2014
7 * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
10 #include <linux/kernel_stat.h>
11 #include <linux/init.h>
12 #include <linux/memblock.h>
13 #include <linux/err.h>
14 #include <linux/virtio.h>
15 #include <linux/virtio_config.h>
16 #include <linux/slab.h>
17 #include <linux/interrupt.h>
18 #include <linux/virtio_ring.h>
19 #include <linux/pfn.h>
20 #include <linux/async.h>
21 #include <linux/wait.h>
22 #include <linux/list.h>
23 #include <linux/bitops.h>
24 #include <linux/moduleparam.h>
26 #include <linux/kvm_para.h>
27 #include <linux/notifier.h>
29 #include <asm/setup.h>
32 #include <asm/ccwdev.h>
33 #include <asm/virtio-ccw.h>
38 * virtio related functions
41 struct vq_config_block
{
46 #define VIRTIO_CCW_CONFIG_SIZE 0x100
47 /* same as PCI config space size, should be enough for all drivers */
49 struct virtio_ccw_device
{
50 struct virtio_device vdev
;
52 __u8 config
[VIRTIO_CCW_CONFIG_SIZE
];
53 struct ccw_device
*cdev
;
56 unsigned int revision
; /* Transport revision */
57 wait_queue_head_t wait_q
;
59 struct mutex io_lock
; /* Serializes I/O requests */
60 struct list_head virtqueues
;
61 unsigned long indicators
;
62 unsigned long indicators2
;
63 struct vq_config_block
*config_block
;
67 unsigned int config_ready
;
71 struct vq_info_block_legacy
{
78 struct vq_info_block
{
87 struct virtio_feature_desc
{
92 struct virtio_thinint_area
{
93 unsigned long summary_indicator
;
94 unsigned long indicator
;
99 struct virtio_rev_info
{
105 /* the highest virtio-ccw revision we support */
106 #define VIRTIO_CCW_REV_MAX 1
108 struct virtio_ccw_vq_info
{
109 struct virtqueue
*vq
;
113 struct vq_info_block s
;
114 struct vq_info_block_legacy l
;
117 struct list_head node
;
121 #define VIRTIO_AIRQ_ISC IO_SCH_ISC /* inherit from subchannel */
123 #define VIRTIO_IV_BITS (L1_CACHE_BYTES * 8)
124 #define MAX_AIRQ_AREAS 20
126 static int virtio_ccw_use_airq
= 1;
130 u8 summary_indicator
;
131 struct airq_struct airq
;
134 static struct airq_info
*airq_areas
[MAX_AIRQ_AREAS
];
136 #define CCW_CMD_SET_VQ 0x13
137 #define CCW_CMD_VDEV_RESET 0x33
138 #define CCW_CMD_SET_IND 0x43
139 #define CCW_CMD_SET_CONF_IND 0x53
140 #define CCW_CMD_READ_FEAT 0x12
141 #define CCW_CMD_WRITE_FEAT 0x11
142 #define CCW_CMD_READ_CONF 0x22
143 #define CCW_CMD_WRITE_CONF 0x21
144 #define CCW_CMD_WRITE_STATUS 0x31
145 #define CCW_CMD_READ_VQ_CONF 0x32
146 #define CCW_CMD_READ_STATUS 0x72
147 #define CCW_CMD_SET_IND_ADAPTER 0x73
148 #define CCW_CMD_SET_VIRTIO_REV 0x83
150 #define VIRTIO_CCW_DOING_SET_VQ 0x00010000
151 #define VIRTIO_CCW_DOING_RESET 0x00040000
152 #define VIRTIO_CCW_DOING_READ_FEAT 0x00080000
153 #define VIRTIO_CCW_DOING_WRITE_FEAT 0x00100000
154 #define VIRTIO_CCW_DOING_READ_CONFIG 0x00200000
155 #define VIRTIO_CCW_DOING_WRITE_CONFIG 0x00400000
156 #define VIRTIO_CCW_DOING_WRITE_STATUS 0x00800000
157 #define VIRTIO_CCW_DOING_SET_IND 0x01000000
158 #define VIRTIO_CCW_DOING_READ_VQ_CONF 0x02000000
159 #define VIRTIO_CCW_DOING_SET_CONF_IND 0x04000000
160 #define VIRTIO_CCW_DOING_SET_IND_ADAPTER 0x08000000
161 #define VIRTIO_CCW_DOING_SET_VIRTIO_REV 0x10000000
162 #define VIRTIO_CCW_DOING_READ_STATUS 0x20000000
163 #define VIRTIO_CCW_INTPARM_MASK 0xffff0000
165 static struct virtio_ccw_device
*to_vc_device(struct virtio_device
*vdev
)
167 return container_of(vdev
, struct virtio_ccw_device
, vdev
);
170 static void drop_airq_indicator(struct virtqueue
*vq
, struct airq_info
*info
)
172 unsigned long i
, flags
;
174 write_lock_irqsave(&info
->lock
, flags
);
175 for (i
= 0; i
< airq_iv_end(info
->aiv
); i
++) {
176 if (vq
== (void *)airq_iv_get_ptr(info
->aiv
, i
)) {
177 airq_iv_free_bit(info
->aiv
, i
);
178 airq_iv_set_ptr(info
->aiv
, i
, 0);
182 write_unlock_irqrestore(&info
->lock
, flags
);
185 static void virtio_airq_handler(struct airq_struct
*airq
)
187 struct airq_info
*info
= container_of(airq
, struct airq_info
, airq
);
190 inc_irq_stat(IRQIO_VAI
);
191 read_lock(&info
->lock
);
192 /* Walk through indicators field, summary indicator active. */
194 ai
= airq_iv_scan(info
->aiv
, ai
, airq_iv_end(info
->aiv
));
197 vring_interrupt(0, (void *)airq_iv_get_ptr(info
->aiv
, ai
));
199 info
->summary_indicator
= 0;
201 /* Walk through indicators field, summary indicator not active. */
203 ai
= airq_iv_scan(info
->aiv
, ai
, airq_iv_end(info
->aiv
));
206 vring_interrupt(0, (void *)airq_iv_get_ptr(info
->aiv
, ai
));
208 read_unlock(&info
->lock
);
211 static struct airq_info
*new_airq_info(void)
213 struct airq_info
*info
;
216 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
219 rwlock_init(&info
->lock
);
220 info
->aiv
= airq_iv_create(VIRTIO_IV_BITS
, AIRQ_IV_ALLOC
| AIRQ_IV_PTR
);
225 info
->airq
.handler
= virtio_airq_handler
;
226 info
->airq
.lsi_ptr
= &info
->summary_indicator
;
227 info
->airq
.lsi_mask
= 0xff;
228 info
->airq
.isc
= VIRTIO_AIRQ_ISC
;
229 rc
= register_adapter_interrupt(&info
->airq
);
231 airq_iv_release(info
->aiv
);
238 static unsigned long get_airq_indicator(struct virtqueue
*vqs
[], int nvqs
,
239 u64
*first
, void **airq_info
)
242 struct airq_info
*info
;
243 unsigned long indicator_addr
= 0;
244 unsigned long bit
, flags
;
246 for (i
= 0; i
< MAX_AIRQ_AREAS
&& !indicator_addr
; i
++) {
248 airq_areas
[i
] = new_airq_info();
249 info
= airq_areas
[i
];
252 write_lock_irqsave(&info
->lock
, flags
);
253 bit
= airq_iv_alloc(info
->aiv
, nvqs
);
255 /* Not enough vacancies. */
256 write_unlock_irqrestore(&info
->lock
, flags
);
261 indicator_addr
= (unsigned long)info
->aiv
->vector
;
262 for (j
= 0; j
< nvqs
; j
++) {
263 airq_iv_set_ptr(info
->aiv
, bit
+ j
,
264 (unsigned long)vqs
[j
]);
266 write_unlock_irqrestore(&info
->lock
, flags
);
268 return indicator_addr
;
271 static void virtio_ccw_drop_indicators(struct virtio_ccw_device
*vcdev
)
273 struct virtio_ccw_vq_info
*info
;
275 list_for_each_entry(info
, &vcdev
->virtqueues
, node
)
276 drop_airq_indicator(info
->vq
, vcdev
->airq_info
);
279 static int doing_io(struct virtio_ccw_device
*vcdev
, __u32 flag
)
284 spin_lock_irqsave(get_ccwdev_lock(vcdev
->cdev
), flags
);
288 ret
= vcdev
->curr_io
& flag
;
289 spin_unlock_irqrestore(get_ccwdev_lock(vcdev
->cdev
), flags
);
293 static int ccw_io_helper(struct virtio_ccw_device
*vcdev
,
294 struct ccw1
*ccw
, __u32 intparm
)
298 int flag
= intparm
& VIRTIO_CCW_INTPARM_MASK
;
300 mutex_lock(&vcdev
->io_lock
);
302 spin_lock_irqsave(get_ccwdev_lock(vcdev
->cdev
), flags
);
303 ret
= ccw_device_start(vcdev
->cdev
, ccw
, intparm
, 0, 0);
307 vcdev
->curr_io
|= flag
;
309 spin_unlock_irqrestore(get_ccwdev_lock(vcdev
->cdev
), flags
);
311 } while (ret
== -EBUSY
);
312 wait_event(vcdev
->wait_q
, doing_io(vcdev
, flag
) == 0);
313 ret
= ret
? ret
: vcdev
->err
;
314 mutex_unlock(&vcdev
->io_lock
);
318 static void virtio_ccw_drop_indicator(struct virtio_ccw_device
*vcdev
,
322 unsigned long *indicatorp
= NULL
;
323 struct virtio_thinint_area
*thinint_area
= NULL
;
324 struct airq_info
*airq_info
= vcdev
->airq_info
;
326 if (vcdev
->is_thinint
) {
327 thinint_area
= kzalloc(sizeof(*thinint_area
),
328 GFP_DMA
| GFP_KERNEL
);
331 thinint_area
->summary_indicator
=
332 (unsigned long) &airq_info
->summary_indicator
;
333 thinint_area
->isc
= VIRTIO_AIRQ_ISC
;
334 ccw
->cmd_code
= CCW_CMD_SET_IND_ADAPTER
;
335 ccw
->count
= sizeof(*thinint_area
);
336 ccw
->cda
= (__u32
)(unsigned long) thinint_area
;
338 /* payload is the address of the indicators */
339 indicatorp
= kmalloc(sizeof(&vcdev
->indicators
),
340 GFP_DMA
| GFP_KERNEL
);
344 ccw
->cmd_code
= CCW_CMD_SET_IND
;
345 ccw
->count
= sizeof(&vcdev
->indicators
);
346 ccw
->cda
= (__u32
)(unsigned long) indicatorp
;
348 /* Deregister indicators from host. */
349 vcdev
->indicators
= 0;
351 ret
= ccw_io_helper(vcdev
, ccw
,
353 VIRTIO_CCW_DOING_SET_IND_ADAPTER
:
354 VIRTIO_CCW_DOING_SET_IND
);
355 if (ret
&& (ret
!= -ENODEV
))
356 dev_info(&vcdev
->cdev
->dev
,
357 "Failed to deregister indicators (%d)\n", ret
);
358 else if (vcdev
->is_thinint
)
359 virtio_ccw_drop_indicators(vcdev
);
364 static inline long __do_kvm_notify(struct subchannel_id schid
,
365 unsigned long queue_index
,
368 register unsigned long __nr
asm("1") = KVM_S390_VIRTIO_CCW_NOTIFY
;
369 register struct subchannel_id __schid
asm("2") = schid
;
370 register unsigned long __index
asm("3") = queue_index
;
371 register long __rc
asm("2");
372 register long __cookie
asm("4") = cookie
;
374 asm volatile ("diag 2,4,0x500\n"
375 : "=d" (__rc
) : "d" (__nr
), "d" (__schid
), "d" (__index
),
381 static inline long do_kvm_notify(struct subchannel_id schid
,
382 unsigned long queue_index
,
385 diag_stat_inc(DIAG_STAT_X500
);
386 return __do_kvm_notify(schid
, queue_index
, cookie
);
389 static bool virtio_ccw_kvm_notify(struct virtqueue
*vq
)
391 struct virtio_ccw_vq_info
*info
= vq
->priv
;
392 struct virtio_ccw_device
*vcdev
;
393 struct subchannel_id schid
;
395 vcdev
= to_vc_device(info
->vq
->vdev
);
396 ccw_device_get_schid(vcdev
->cdev
, &schid
);
397 info
->cookie
= do_kvm_notify(schid
, vq
->index
, info
->cookie
);
398 if (info
->cookie
< 0)
403 static int virtio_ccw_read_vq_conf(struct virtio_ccw_device
*vcdev
,
404 struct ccw1
*ccw
, int index
)
408 vcdev
->config_block
->index
= index
;
409 ccw
->cmd_code
= CCW_CMD_READ_VQ_CONF
;
411 ccw
->count
= sizeof(struct vq_config_block
);
412 ccw
->cda
= (__u32
)(unsigned long)(vcdev
->config_block
);
413 ret
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_READ_VQ_CONF
);
416 return vcdev
->config_block
->num
;
419 static void virtio_ccw_del_vq(struct virtqueue
*vq
, struct ccw1
*ccw
)
421 struct virtio_ccw_device
*vcdev
= to_vc_device(vq
->vdev
);
422 struct virtio_ccw_vq_info
*info
= vq
->priv
;
426 unsigned int index
= vq
->index
;
428 /* Remove from our list. */
429 spin_lock_irqsave(&vcdev
->lock
, flags
);
430 list_del(&info
->node
);
431 spin_unlock_irqrestore(&vcdev
->lock
, flags
);
433 /* Release from host. */
434 if (vcdev
->revision
== 0) {
435 info
->info_block
->l
.queue
= 0;
436 info
->info_block
->l
.align
= 0;
437 info
->info_block
->l
.index
= index
;
438 info
->info_block
->l
.num
= 0;
439 ccw
->count
= sizeof(info
->info_block
->l
);
441 info
->info_block
->s
.desc
= 0;
442 info
->info_block
->s
.index
= index
;
443 info
->info_block
->s
.num
= 0;
444 info
->info_block
->s
.avail
= 0;
445 info
->info_block
->s
.used
= 0;
446 ccw
->count
= sizeof(info
->info_block
->s
);
448 ccw
->cmd_code
= CCW_CMD_SET_VQ
;
450 ccw
->cda
= (__u32
)(unsigned long)(info
->info_block
);
451 ret
= ccw_io_helper(vcdev
, ccw
,
452 VIRTIO_CCW_DOING_SET_VQ
| index
);
454 * -ENODEV isn't considered an error: The device is gone anyway.
455 * This may happen on device detach.
457 if (ret
&& (ret
!= -ENODEV
))
458 dev_warn(&vq
->vdev
->dev
, "Error %d while deleting queue %d\n",
461 vring_del_virtqueue(vq
);
462 size
= PAGE_ALIGN(vring_size(info
->num
, KVM_VIRTIO_CCW_RING_ALIGN
));
463 free_pages_exact(info
->queue
, size
);
464 kfree(info
->info_block
);
468 static void virtio_ccw_del_vqs(struct virtio_device
*vdev
)
470 struct virtqueue
*vq
, *n
;
472 struct virtio_ccw_device
*vcdev
= to_vc_device(vdev
);
474 ccw
= kzalloc(sizeof(*ccw
), GFP_DMA
| GFP_KERNEL
);
478 virtio_ccw_drop_indicator(vcdev
, ccw
);
480 list_for_each_entry_safe(vq
, n
, &vdev
->vqs
, list
)
481 virtio_ccw_del_vq(vq
, ccw
);
486 static struct virtqueue
*virtio_ccw_setup_vq(struct virtio_device
*vdev
,
487 int i
, vq_callback_t
*callback
,
488 const char *name
, bool ctx
,
491 struct virtio_ccw_device
*vcdev
= to_vc_device(vdev
);
493 struct virtqueue
*vq
= NULL
;
494 struct virtio_ccw_vq_info
*info
;
495 unsigned long size
= 0; /* silence the compiler */
498 /* Allocate queue. */
499 info
= kzalloc(sizeof(struct virtio_ccw_vq_info
), GFP_KERNEL
);
501 dev_warn(&vcdev
->cdev
->dev
, "no info\n");
505 info
->info_block
= kzalloc(sizeof(*info
->info_block
),
506 GFP_DMA
| GFP_KERNEL
);
507 if (!info
->info_block
) {
508 dev_warn(&vcdev
->cdev
->dev
, "no info block\n");
512 info
->num
= virtio_ccw_read_vq_conf(vcdev
, ccw
, i
);
517 size
= PAGE_ALIGN(vring_size(info
->num
, KVM_VIRTIO_CCW_RING_ALIGN
));
518 info
->queue
= alloc_pages_exact(size
, GFP_KERNEL
| __GFP_ZERO
);
519 if (info
->queue
== NULL
) {
520 dev_warn(&vcdev
->cdev
->dev
, "no queue\n");
525 vq
= vring_new_virtqueue(i
, info
->num
, KVM_VIRTIO_CCW_RING_ALIGN
, vdev
,
526 true, ctx
, info
->queue
, virtio_ccw_kvm_notify
,
529 /* For now, we fail if we can't get the requested size. */
530 dev_warn(&vcdev
->cdev
->dev
, "no vq\n");
535 /* Register it with the host. */
536 if (vcdev
->revision
== 0) {
537 info
->info_block
->l
.queue
= (__u64
)info
->queue
;
538 info
->info_block
->l
.align
= KVM_VIRTIO_CCW_RING_ALIGN
;
539 info
->info_block
->l
.index
= i
;
540 info
->info_block
->l
.num
= info
->num
;
541 ccw
->count
= sizeof(info
->info_block
->l
);
543 info
->info_block
->s
.desc
= (__u64
)info
->queue
;
544 info
->info_block
->s
.index
= i
;
545 info
->info_block
->s
.num
= info
->num
;
546 info
->info_block
->s
.avail
= (__u64
)virtqueue_get_avail(vq
);
547 info
->info_block
->s
.used
= (__u64
)virtqueue_get_used(vq
);
548 ccw
->count
= sizeof(info
->info_block
->s
);
550 ccw
->cmd_code
= CCW_CMD_SET_VQ
;
552 ccw
->cda
= (__u32
)(unsigned long)(info
->info_block
);
553 err
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_SET_VQ
| i
);
555 dev_warn(&vcdev
->cdev
->dev
, "SET_VQ failed\n");
562 /* Save it to our list. */
563 spin_lock_irqsave(&vcdev
->lock
, flags
);
564 list_add(&info
->node
, &vcdev
->virtqueues
);
565 spin_unlock_irqrestore(&vcdev
->lock
, flags
);
571 vring_del_virtqueue(vq
);
574 free_pages_exact(info
->queue
, size
);
575 kfree(info
->info_block
);
581 static int virtio_ccw_register_adapter_ind(struct virtio_ccw_device
*vcdev
,
582 struct virtqueue
*vqs
[], int nvqs
,
586 struct virtio_thinint_area
*thinint_area
= NULL
;
587 struct airq_info
*info
;
589 thinint_area
= kzalloc(sizeof(*thinint_area
), GFP_DMA
| GFP_KERNEL
);
594 /* Try to get an indicator. */
595 thinint_area
->indicator
= get_airq_indicator(vqs
, nvqs
,
596 &thinint_area
->bit_nr
,
598 if (!thinint_area
->indicator
) {
602 info
= vcdev
->airq_info
;
603 thinint_area
->summary_indicator
=
604 (unsigned long) &info
->summary_indicator
;
605 thinint_area
->isc
= VIRTIO_AIRQ_ISC
;
606 ccw
->cmd_code
= CCW_CMD_SET_IND_ADAPTER
;
607 ccw
->flags
= CCW_FLAG_SLI
;
608 ccw
->count
= sizeof(*thinint_area
);
609 ccw
->cda
= (__u32
)(unsigned long)thinint_area
;
610 ret
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_SET_IND_ADAPTER
);
612 if (ret
== -EOPNOTSUPP
) {
614 * The host does not support adapter interrupts
615 * for virtio-ccw, stop trying.
617 virtio_ccw_use_airq
= 0;
618 pr_info("Adapter interrupts unsupported on host\n");
620 dev_warn(&vcdev
->cdev
->dev
,
621 "enabling adapter interrupts = %d\n", ret
);
622 virtio_ccw_drop_indicators(vcdev
);
629 static int virtio_ccw_find_vqs(struct virtio_device
*vdev
, unsigned nvqs
,
630 struct virtqueue
*vqs
[],
631 vq_callback_t
*callbacks
[],
632 const char * const names
[],
634 struct irq_affinity
*desc
)
636 struct virtio_ccw_device
*vcdev
= to_vc_device(vdev
);
637 unsigned long *indicatorp
= NULL
;
638 int ret
, i
, queue_idx
= 0;
641 ccw
= kzalloc(sizeof(*ccw
), GFP_DMA
| GFP_KERNEL
);
645 for (i
= 0; i
< nvqs
; ++i
) {
651 vqs
[i
] = virtio_ccw_setup_vq(vdev
, queue_idx
++, callbacks
[i
],
652 names
[i
], ctx
? ctx
[i
] : false,
654 if (IS_ERR(vqs
[i
])) {
655 ret
= PTR_ERR(vqs
[i
]);
662 * We need a data area under 2G to communicate. Our payload is
663 * the address of the indicators.
665 indicatorp
= kmalloc(sizeof(&vcdev
->indicators
), GFP_DMA
| GFP_KERNEL
);
668 *indicatorp
= (unsigned long) &vcdev
->indicators
;
669 if (vcdev
->is_thinint
) {
670 ret
= virtio_ccw_register_adapter_ind(vcdev
, vqs
, nvqs
, ccw
);
672 /* no error, just fall back to legacy interrupts */
673 vcdev
->is_thinint
= false;
675 if (!vcdev
->is_thinint
) {
676 /* Register queue indicators with host. */
677 vcdev
->indicators
= 0;
678 ccw
->cmd_code
= CCW_CMD_SET_IND
;
680 ccw
->count
= sizeof(&vcdev
->indicators
);
681 ccw
->cda
= (__u32
)(unsigned long) indicatorp
;
682 ret
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_SET_IND
);
686 /* Register indicators2 with host for config changes */
687 *indicatorp
= (unsigned long) &vcdev
->indicators2
;
688 vcdev
->indicators2
= 0;
689 ccw
->cmd_code
= CCW_CMD_SET_CONF_IND
;
691 ccw
->count
= sizeof(&vcdev
->indicators2
);
692 ccw
->cda
= (__u32
)(unsigned long) indicatorp
;
693 ret
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_SET_CONF_IND
);
703 virtio_ccw_del_vqs(vdev
);
707 static void virtio_ccw_reset(struct virtio_device
*vdev
)
709 struct virtio_ccw_device
*vcdev
= to_vc_device(vdev
);
712 ccw
= kzalloc(sizeof(*ccw
), GFP_DMA
| GFP_KERNEL
);
716 /* Zero status bits. */
719 /* Send a reset ccw on device. */
720 ccw
->cmd_code
= CCW_CMD_VDEV_RESET
;
724 ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_RESET
);
728 static u64
virtio_ccw_get_features(struct virtio_device
*vdev
)
730 struct virtio_ccw_device
*vcdev
= to_vc_device(vdev
);
731 struct virtio_feature_desc
*features
;
736 ccw
= kzalloc(sizeof(*ccw
), GFP_DMA
| GFP_KERNEL
);
740 features
= kzalloc(sizeof(*features
), GFP_DMA
| GFP_KERNEL
);
745 /* Read the feature bits from the host. */
747 ccw
->cmd_code
= CCW_CMD_READ_FEAT
;
749 ccw
->count
= sizeof(*features
);
750 ccw
->cda
= (__u32
)(unsigned long)features
;
751 ret
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_READ_FEAT
);
757 rc
= le32_to_cpu(features
->features
);
759 if (vcdev
->revision
== 0)
762 /* Read second half of the feature bits from the host. */
764 ccw
->cmd_code
= CCW_CMD_READ_FEAT
;
766 ccw
->count
= sizeof(*features
);
767 ccw
->cda
= (__u32
)(unsigned long)features
;
768 ret
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_READ_FEAT
);
770 rc
|= (u64
)le32_to_cpu(features
->features
) << 32;
778 static void ccw_transport_features(struct virtio_device
*vdev
)
781 * Packed ring isn't enabled on virtio_ccw for now,
782 * because virtio_ccw uses some legacy accessors,
783 * e.g. virtqueue_get_avail() and virtqueue_get_used()
784 * which aren't available in packed ring currently.
786 __virtio_clear_bit(vdev
, VIRTIO_F_RING_PACKED
);
789 static int virtio_ccw_finalize_features(struct virtio_device
*vdev
)
791 struct virtio_ccw_device
*vcdev
= to_vc_device(vdev
);
792 struct virtio_feature_desc
*features
;
796 if (vcdev
->revision
>= 1 &&
797 !__virtio_test_bit(vdev
, VIRTIO_F_VERSION_1
)) {
798 dev_err(&vdev
->dev
, "virtio: device uses revision 1 "
799 "but does not have VIRTIO_F_VERSION_1\n");
803 ccw
= kzalloc(sizeof(*ccw
), GFP_DMA
| GFP_KERNEL
);
807 features
= kzalloc(sizeof(*features
), GFP_DMA
| GFP_KERNEL
);
812 /* Give virtio_ring a chance to accept features. */
813 vring_transport_features(vdev
);
815 /* Give virtio_ccw a chance to accept features. */
816 ccw_transport_features(vdev
);
819 features
->features
= cpu_to_le32((u32
)vdev
->features
);
820 /* Write the first half of the feature bits to the host. */
821 ccw
->cmd_code
= CCW_CMD_WRITE_FEAT
;
823 ccw
->count
= sizeof(*features
);
824 ccw
->cda
= (__u32
)(unsigned long)features
;
825 ret
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_WRITE_FEAT
);
829 if (vcdev
->revision
== 0)
833 features
->features
= cpu_to_le32(vdev
->features
>> 32);
834 /* Write the second half of the feature bits to the host. */
835 ccw
->cmd_code
= CCW_CMD_WRITE_FEAT
;
837 ccw
->count
= sizeof(*features
);
838 ccw
->cda
= (__u32
)(unsigned long)features
;
839 ret
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_WRITE_FEAT
);
848 static void virtio_ccw_get_config(struct virtio_device
*vdev
,
849 unsigned int offset
, void *buf
, unsigned len
)
851 struct virtio_ccw_device
*vcdev
= to_vc_device(vdev
);
857 ccw
= kzalloc(sizeof(*ccw
), GFP_DMA
| GFP_KERNEL
);
861 config_area
= kzalloc(VIRTIO_CCW_CONFIG_SIZE
, GFP_DMA
| GFP_KERNEL
);
865 /* Read the config area from the host. */
866 ccw
->cmd_code
= CCW_CMD_READ_CONF
;
868 ccw
->count
= offset
+ len
;
869 ccw
->cda
= (__u32
)(unsigned long)config_area
;
870 ret
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_READ_CONFIG
);
874 spin_lock_irqsave(&vcdev
->lock
, flags
);
875 memcpy(vcdev
->config
, config_area
, offset
+ len
);
876 if (vcdev
->config_ready
< offset
+ len
)
877 vcdev
->config_ready
= offset
+ len
;
878 spin_unlock_irqrestore(&vcdev
->lock
, flags
);
880 memcpy(buf
, config_area
+ offset
, len
);
887 static void virtio_ccw_set_config(struct virtio_device
*vdev
,
888 unsigned int offset
, const void *buf
,
891 struct virtio_ccw_device
*vcdev
= to_vc_device(vdev
);
896 ccw
= kzalloc(sizeof(*ccw
), GFP_DMA
| GFP_KERNEL
);
900 config_area
= kzalloc(VIRTIO_CCW_CONFIG_SIZE
, GFP_DMA
| GFP_KERNEL
);
904 /* Make sure we don't overwrite fields. */
905 if (vcdev
->config_ready
< offset
)
906 virtio_ccw_get_config(vdev
, 0, NULL
, offset
);
907 spin_lock_irqsave(&vcdev
->lock
, flags
);
908 memcpy(&vcdev
->config
[offset
], buf
, len
);
909 /* Write the config area to the host. */
910 memcpy(config_area
, vcdev
->config
, sizeof(vcdev
->config
));
911 spin_unlock_irqrestore(&vcdev
->lock
, flags
);
912 ccw
->cmd_code
= CCW_CMD_WRITE_CONF
;
914 ccw
->count
= offset
+ len
;
915 ccw
->cda
= (__u32
)(unsigned long)config_area
;
916 ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_WRITE_CONFIG
);
923 static u8
virtio_ccw_get_status(struct virtio_device
*vdev
)
925 struct virtio_ccw_device
*vcdev
= to_vc_device(vdev
);
926 u8 old_status
= *vcdev
->status
;
929 if (vcdev
->revision
< 1)
930 return *vcdev
->status
;
932 ccw
= kzalloc(sizeof(*ccw
), GFP_DMA
| GFP_KERNEL
);
936 ccw
->cmd_code
= CCW_CMD_READ_STATUS
;
938 ccw
->count
= sizeof(*vcdev
->status
);
939 ccw
->cda
= (__u32
)(unsigned long)vcdev
->status
;
940 ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_READ_STATUS
);
942 * If the channel program failed (should only happen if the device
943 * was hotunplugged, and then we clean up via the machine check
944 * handler anyway), vcdev->status was not overwritten and we just
945 * return the old status, which is fine.
949 return *vcdev
->status
;
952 static void virtio_ccw_set_status(struct virtio_device
*vdev
, u8 status
)
954 struct virtio_ccw_device
*vcdev
= to_vc_device(vdev
);
955 u8 old_status
= *vcdev
->status
;
959 ccw
= kzalloc(sizeof(*ccw
), GFP_DMA
| GFP_KERNEL
);
963 /* Write the status to the host. */
964 *vcdev
->status
= status
;
965 ccw
->cmd_code
= CCW_CMD_WRITE_STATUS
;
967 ccw
->count
= sizeof(status
);
968 ccw
->cda
= (__u32
)(unsigned long)vcdev
->status
;
969 ret
= ccw_io_helper(vcdev
, ccw
, VIRTIO_CCW_DOING_WRITE_STATUS
);
970 /* Write failed? We assume status is unchanged. */
972 *vcdev
->status
= old_status
;
976 static const struct virtio_config_ops virtio_ccw_config_ops
= {
977 .get_features
= virtio_ccw_get_features
,
978 .finalize_features
= virtio_ccw_finalize_features
,
979 .get
= virtio_ccw_get_config
,
980 .set
= virtio_ccw_set_config
,
981 .get_status
= virtio_ccw_get_status
,
982 .set_status
= virtio_ccw_set_status
,
983 .reset
= virtio_ccw_reset
,
984 .find_vqs
= virtio_ccw_find_vqs
,
985 .del_vqs
= virtio_ccw_del_vqs
,
990 * ccw bus driver related functions
993 static void virtio_ccw_release_dev(struct device
*_d
)
995 struct virtio_device
*dev
= dev_to_virtio(_d
);
996 struct virtio_ccw_device
*vcdev
= to_vc_device(dev
);
998 kfree(vcdev
->status
);
999 kfree(vcdev
->config_block
);
1003 static int irb_is_error(struct irb
*irb
)
1005 if (scsw_cstat(&irb
->scsw
) != 0)
1007 if (scsw_dstat(&irb
->scsw
) & ~(DEV_STAT_CHN_END
| DEV_STAT_DEV_END
))
1009 if (scsw_cc(&irb
->scsw
) != 0)
1014 static struct virtqueue
*virtio_ccw_vq_by_ind(struct virtio_ccw_device
*vcdev
,
1017 struct virtio_ccw_vq_info
*info
;
1018 unsigned long flags
;
1019 struct virtqueue
*vq
;
1022 spin_lock_irqsave(&vcdev
->lock
, flags
);
1023 list_for_each_entry(info
, &vcdev
->virtqueues
, node
) {
1024 if (info
->vq
->index
== index
) {
1029 spin_unlock_irqrestore(&vcdev
->lock
, flags
);
1033 static void virtio_ccw_check_activity(struct virtio_ccw_device
*vcdev
,
1036 if (vcdev
->curr_io
& activity
) {
1038 case VIRTIO_CCW_DOING_READ_FEAT
:
1039 case VIRTIO_CCW_DOING_WRITE_FEAT
:
1040 case VIRTIO_CCW_DOING_READ_CONFIG
:
1041 case VIRTIO_CCW_DOING_WRITE_CONFIG
:
1042 case VIRTIO_CCW_DOING_WRITE_STATUS
:
1043 case VIRTIO_CCW_DOING_READ_STATUS
:
1044 case VIRTIO_CCW_DOING_SET_VQ
:
1045 case VIRTIO_CCW_DOING_SET_IND
:
1046 case VIRTIO_CCW_DOING_SET_CONF_IND
:
1047 case VIRTIO_CCW_DOING_RESET
:
1048 case VIRTIO_CCW_DOING_READ_VQ_CONF
:
1049 case VIRTIO_CCW_DOING_SET_IND_ADAPTER
:
1050 case VIRTIO_CCW_DOING_SET_VIRTIO_REV
:
1051 vcdev
->curr_io
&= ~activity
;
1052 wake_up(&vcdev
->wait_q
);
1055 /* don't know what to do... */
1056 dev_warn(&vcdev
->cdev
->dev
,
1057 "Suspicious activity '%08x'\n", activity
);
1064 static void virtio_ccw_int_handler(struct ccw_device
*cdev
,
1065 unsigned long intparm
,
1068 __u32 activity
= intparm
& VIRTIO_CCW_INTPARM_MASK
;
1069 struct virtio_ccw_device
*vcdev
= dev_get_drvdata(&cdev
->dev
);
1071 struct virtqueue
*vq
;
1076 vcdev
->err
= PTR_ERR(irb
);
1077 virtio_ccw_check_activity(vcdev
, activity
);
1078 /* Don't poke around indicators, something's wrong. */
1081 /* Check if it's a notification from the host. */
1082 if ((intparm
== 0) &&
1083 (scsw_stctl(&irb
->scsw
) ==
1084 (SCSW_STCTL_ALERT_STATUS
| SCSW_STCTL_STATUS_PEND
))) {
1087 if (irb_is_error(irb
)) {
1088 /* Command reject? */
1089 if ((scsw_dstat(&irb
->scsw
) & DEV_STAT_UNIT_CHECK
) &&
1090 (irb
->ecw
[0] & SNS0_CMD_REJECT
))
1091 vcdev
->err
= -EOPNOTSUPP
;
1093 /* Map everything else to -EIO. */
1096 virtio_ccw_check_activity(vcdev
, activity
);
1097 for_each_set_bit(i
, &vcdev
->indicators
,
1098 sizeof(vcdev
->indicators
) * BITS_PER_BYTE
) {
1099 /* The bit clear must happen before the vring kick. */
1100 clear_bit(i
, &vcdev
->indicators
);
1102 vq
= virtio_ccw_vq_by_ind(vcdev
, i
);
1103 vring_interrupt(0, vq
);
1105 if (test_bit(0, &vcdev
->indicators2
)) {
1106 virtio_config_changed(&vcdev
->vdev
);
1107 clear_bit(0, &vcdev
->indicators2
);
1112 * We usually want to autoonline all devices, but give the admin
1113 * a way to exempt devices from this.
1115 #define __DEV_WORDS ((__MAX_SUBCHANNEL + (8*sizeof(long) - 1)) / \
1117 static unsigned long devs_no_auto
[__MAX_SSID
+ 1][__DEV_WORDS
];
1119 static char *no_auto
= "";
1121 module_param(no_auto
, charp
, 0444);
1122 MODULE_PARM_DESC(no_auto
, "list of ccw bus id ranges not to be auto-onlined");
1124 static int virtio_ccw_check_autoonline(struct ccw_device
*cdev
)
1126 struct ccw_dev_id id
;
1128 ccw_device_get_id(cdev
, &id
);
1129 if (test_bit(id
.devno
, devs_no_auto
[id
.ssid
]))
1134 static void virtio_ccw_auto_online(void *data
, async_cookie_t cookie
)
1136 struct ccw_device
*cdev
= data
;
1139 ret
= ccw_device_set_online(cdev
);
1141 dev_warn(&cdev
->dev
, "Failed to set online: %d\n", ret
);
1144 static int virtio_ccw_probe(struct ccw_device
*cdev
)
1146 cdev
->handler
= virtio_ccw_int_handler
;
1148 if (virtio_ccw_check_autoonline(cdev
))
1149 async_schedule(virtio_ccw_auto_online
, cdev
);
1153 static struct virtio_ccw_device
*virtio_grab_drvdata(struct ccw_device
*cdev
)
1155 unsigned long flags
;
1156 struct virtio_ccw_device
*vcdev
;
1158 spin_lock_irqsave(get_ccwdev_lock(cdev
), flags
);
1159 vcdev
= dev_get_drvdata(&cdev
->dev
);
1160 if (!vcdev
|| vcdev
->going_away
) {
1161 spin_unlock_irqrestore(get_ccwdev_lock(cdev
), flags
);
1164 vcdev
->going_away
= true;
1165 spin_unlock_irqrestore(get_ccwdev_lock(cdev
), flags
);
1169 static void virtio_ccw_remove(struct ccw_device
*cdev
)
1171 unsigned long flags
;
1172 struct virtio_ccw_device
*vcdev
= virtio_grab_drvdata(cdev
);
1174 if (vcdev
&& cdev
->online
) {
1175 if (vcdev
->device_lost
)
1176 virtio_break_device(&vcdev
->vdev
);
1177 unregister_virtio_device(&vcdev
->vdev
);
1178 spin_lock_irqsave(get_ccwdev_lock(cdev
), flags
);
1179 dev_set_drvdata(&cdev
->dev
, NULL
);
1180 spin_unlock_irqrestore(get_ccwdev_lock(cdev
), flags
);
1182 cdev
->handler
= NULL
;
1185 static int virtio_ccw_offline(struct ccw_device
*cdev
)
1187 unsigned long flags
;
1188 struct virtio_ccw_device
*vcdev
= virtio_grab_drvdata(cdev
);
1192 if (vcdev
->device_lost
)
1193 virtio_break_device(&vcdev
->vdev
);
1194 unregister_virtio_device(&vcdev
->vdev
);
1195 spin_lock_irqsave(get_ccwdev_lock(cdev
), flags
);
1196 dev_set_drvdata(&cdev
->dev
, NULL
);
1197 spin_unlock_irqrestore(get_ccwdev_lock(cdev
), flags
);
1201 static int virtio_ccw_set_transport_rev(struct virtio_ccw_device
*vcdev
)
1203 struct virtio_rev_info
*rev
;
1207 ccw
= kzalloc(sizeof(*ccw
), GFP_DMA
| GFP_KERNEL
);
1210 rev
= kzalloc(sizeof(*rev
), GFP_DMA
| GFP_KERNEL
);
1216 /* Set transport revision */
1217 ccw
->cmd_code
= CCW_CMD_SET_VIRTIO_REV
;
1219 ccw
->count
= sizeof(*rev
);
1220 ccw
->cda
= (__u32
)(unsigned long)rev
;
1222 vcdev
->revision
= VIRTIO_CCW_REV_MAX
;
1224 rev
->revision
= vcdev
->revision
;
1225 /* none of our supported revisions carry payload */
1227 ret
= ccw_io_helper(vcdev
, ccw
,
1228 VIRTIO_CCW_DOING_SET_VIRTIO_REV
);
1229 if (ret
== -EOPNOTSUPP
) {
1230 if (vcdev
->revision
== 0)
1232 * The host device does not support setting
1233 * the revision: let's operate it in legacy
1240 } while (ret
== -EOPNOTSUPP
);
1247 static int virtio_ccw_online(struct ccw_device
*cdev
)
1250 struct virtio_ccw_device
*vcdev
;
1251 unsigned long flags
;
1253 vcdev
= kzalloc(sizeof(*vcdev
), GFP_KERNEL
);
1255 dev_warn(&cdev
->dev
, "Could not get memory for virtio\n");
1259 vcdev
->config_block
= kzalloc(sizeof(*vcdev
->config_block
),
1260 GFP_DMA
| GFP_KERNEL
);
1261 if (!vcdev
->config_block
) {
1265 vcdev
->status
= kzalloc(sizeof(*vcdev
->status
), GFP_DMA
| GFP_KERNEL
);
1266 if (!vcdev
->status
) {
1271 vcdev
->is_thinint
= virtio_ccw_use_airq
; /* at least try */
1273 vcdev
->vdev
.dev
.parent
= &cdev
->dev
;
1274 vcdev
->vdev
.dev
.release
= virtio_ccw_release_dev
;
1275 vcdev
->vdev
.config
= &virtio_ccw_config_ops
;
1277 init_waitqueue_head(&vcdev
->wait_q
);
1278 INIT_LIST_HEAD(&vcdev
->virtqueues
);
1279 spin_lock_init(&vcdev
->lock
);
1280 mutex_init(&vcdev
->io_lock
);
1282 spin_lock_irqsave(get_ccwdev_lock(cdev
), flags
);
1283 dev_set_drvdata(&cdev
->dev
, vcdev
);
1284 spin_unlock_irqrestore(get_ccwdev_lock(cdev
), flags
);
1285 vcdev
->vdev
.id
.vendor
= cdev
->id
.cu_type
;
1286 vcdev
->vdev
.id
.device
= cdev
->id
.cu_model
;
1288 ret
= virtio_ccw_set_transport_rev(vcdev
);
1292 ret
= register_virtio_device(&vcdev
->vdev
);
1294 dev_warn(&cdev
->dev
, "Failed to register virtio device: %d\n",
1300 spin_lock_irqsave(get_ccwdev_lock(cdev
), flags
);
1301 dev_set_drvdata(&cdev
->dev
, NULL
);
1302 spin_unlock_irqrestore(get_ccwdev_lock(cdev
), flags
);
1303 put_device(&vcdev
->vdev
.dev
);
1307 kfree(vcdev
->status
);
1308 kfree(vcdev
->config_block
);
1314 static int virtio_ccw_cio_notify(struct ccw_device
*cdev
, int event
)
1317 struct virtio_ccw_device
*vcdev
= dev_get_drvdata(&cdev
->dev
);
1320 * Make sure vcdev is set
1321 * i.e. set_offline/remove callback not already running
1328 vcdev
->device_lost
= true;
1341 static struct ccw_device_id virtio_ids
[] = {
1342 { CCW_DEVICE(0x3832, 0) },
1346 #ifdef CONFIG_PM_SLEEP
1347 static int virtio_ccw_freeze(struct ccw_device
*cdev
)
1349 struct virtio_ccw_device
*vcdev
= dev_get_drvdata(&cdev
->dev
);
1351 return virtio_device_freeze(&vcdev
->vdev
);
1354 static int virtio_ccw_restore(struct ccw_device
*cdev
)
1356 struct virtio_ccw_device
*vcdev
= dev_get_drvdata(&cdev
->dev
);
1359 ret
= virtio_ccw_set_transport_rev(vcdev
);
1363 return virtio_device_restore(&vcdev
->vdev
);
1367 static struct ccw_driver virtio_ccw_driver
= {
1369 .owner
= THIS_MODULE
,
1370 .name
= "virtio_ccw",
1373 .probe
= virtio_ccw_probe
,
1374 .remove
= virtio_ccw_remove
,
1375 .set_offline
= virtio_ccw_offline
,
1376 .set_online
= virtio_ccw_online
,
1377 .notify
= virtio_ccw_cio_notify
,
1378 .int_class
= IRQIO_VIR
,
1379 #ifdef CONFIG_PM_SLEEP
1380 .freeze
= virtio_ccw_freeze
,
1381 .thaw
= virtio_ccw_restore
,
1382 .restore
= virtio_ccw_restore
,
1386 static int __init
pure_hex(char **cp
, unsigned int *val
, int min_digit
,
1387 int max_digit
, int max_val
)
1394 while (diff
<= max_digit
) {
1395 int value
= hex_to_bin(**cp
);
1399 *val
= *val
* 16 + value
;
1404 if ((diff
< min_digit
) || (diff
> max_digit
) || (*val
> max_val
))
1410 static int __init
parse_busid(char *str
, unsigned int *cssid
,
1411 unsigned int *ssid
, unsigned int *devno
)
1422 ret
= pure_hex(&str_work
, cssid
, 1, 2, __MAX_CSSID
);
1423 if (ret
|| (str_work
[0] != '.'))
1426 ret
= pure_hex(&str_work
, ssid
, 1, 1, __MAX_SSID
);
1427 if (ret
|| (str_work
[0] != '.'))
1430 ret
= pure_hex(&str_work
, devno
, 4, 4, __MAX_SUBCHANNEL
);
1431 if (ret
|| (str_work
[0] != '\0'))
1439 static void __init
no_auto_parse(void)
1441 unsigned int from_cssid
, to_cssid
, from_ssid
, to_ssid
, from
, to
;
1446 while ((parm
= strsep(&str
, ","))) {
1447 rc
= parse_busid(strsep(&parm
, "-"), &from_cssid
,
1452 rc
= parse_busid(parm
, &to_cssid
,
1454 if ((from_ssid
> to_ssid
) ||
1455 ((from_ssid
== to_ssid
) && (from
> to
)))
1458 to_cssid
= from_cssid
;
1459 to_ssid
= from_ssid
;
1464 while ((from_ssid
< to_ssid
) ||
1465 ((from_ssid
== to_ssid
) && (from
<= to
))) {
1466 set_bit(from
, devs_no_auto
[from_ssid
]);
1468 if (from
> __MAX_SUBCHANNEL
) {
1476 static int __init
virtio_ccw_init(void)
1478 /* parse no_auto string before we do anything further */
1480 return ccw_driver_register(&virtio_ccw_driver
);
1482 device_initcall(virtio_ccw_init
);