]>
Commit | Line | Data |
---|---|---|
1fcf0512 MT |
1 | /* |
2 | * Virtio PCI driver - modern (virtio 1.0) device support | |
3 | * | |
4 | * This module allows virtio devices to be used over a virtual PCI device. | |
5 | * This can be used with QEMU based VMMs like KVM or Xen. | |
6 | * | |
7 | * Copyright IBM Corp. 2007 | |
8 | * Copyright Red Hat, Inc. 2014 | |
9 | * | |
10 | * Authors: | |
11 | * Anthony Liguori <aliguori@us.ibm.com> | |
12 | * Rusty Russell <rusty@rustcorp.com.au> | |
13 | * Michael S. Tsirkin <mst@redhat.com> | |
14 | * | |
15 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
16 | * See the COPYING file in the top-level directory. | |
17 | * | |
18 | */ | |
19 | ||
05dbcb43 | 20 | #include <linux/delay.h> |
1fcf0512 MT |
21 | #define VIRTIO_PCI_NO_LEGACY |
22 | #include "virtio_pci_common.h" | |
23 | ||
c5d4c2c9 MT |
24 | /* |
25 | * Type-safe wrappers for io accesses. | |
26 | * Use these to enforce at compile time the following spec requirement: | |
27 | * | |
28 | * The driver MUST access each field using the “natural” access | |
29 | * method, i.e. 32-bit accesses for 32-bit fields, 16-bit accesses | |
30 | * for 16-bit fields and 8-bit accesses for 8-bit fields. | |
31 | */ | |
32 | static inline u8 vp_ioread8(u8 __iomem *addr) | |
33 | { | |
34 | return ioread8(addr); | |
35 | } | |
61bd405f | 36 | static inline u16 vp_ioread16 (__le16 __iomem *addr) |
c5d4c2c9 MT |
37 | { |
38 | return ioread16(addr); | |
39 | } | |
40 | ||
61bd405f | 41 | static inline u32 vp_ioread32(__le32 __iomem *addr) |
c5d4c2c9 MT |
42 | { |
43 | return ioread32(addr); | |
44 | } | |
45 | ||
46 | static inline void vp_iowrite8(u8 value, u8 __iomem *addr) | |
47 | { | |
48 | iowrite8(value, addr); | |
49 | } | |
50 | ||
61bd405f | 51 | static inline void vp_iowrite16(u16 value, __le16 __iomem *addr) |
c5d4c2c9 MT |
52 | { |
53 | iowrite16(value, addr); | |
54 | } | |
55 | ||
61bd405f | 56 | static inline void vp_iowrite32(u32 value, __le32 __iomem *addr) |
c5d4c2c9 MT |
57 | { |
58 | iowrite32(value, addr); | |
59 | } | |
60 | ||
a8557d32 MT |
61 | static void vp_iowrite64_twopart(u64 val, |
62 | __le32 __iomem *lo, __le32 __iomem *hi) | |
63 | { | |
64 | vp_iowrite32((u32)val, lo); | |
65 | vp_iowrite32(val >> 32, hi); | |
66 | } | |
67 | ||
1fcf0512 MT |
68 | static void __iomem *map_capability(struct pci_dev *dev, int off, |
69 | size_t minlen, | |
70 | u32 align, | |
71 | u32 start, u32 size, | |
72 | size_t *len) | |
73 | { | |
74 | u8 bar; | |
75 | u32 offset, length; | |
76 | void __iomem *p; | |
77 | ||
78 | pci_read_config_byte(dev, off + offsetof(struct virtio_pci_cap, | |
79 | bar), | |
80 | &bar); | |
81 | pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, offset), | |
82 | &offset); | |
83 | pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length), | |
84 | &length); | |
85 | ||
86 | if (length <= start) { | |
87 | dev_err(&dev->dev, | |
88 | "virtio_pci: bad capability len %u (>%u expected)\n", | |
89 | length, start); | |
90 | return NULL; | |
91 | } | |
92 | ||
93 | if (length - start < minlen) { | |
94 | dev_err(&dev->dev, | |
95 | "virtio_pci: bad capability len %u (>=%zu expected)\n", | |
96 | length, minlen); | |
97 | return NULL; | |
98 | } | |
99 | ||
100 | length -= start; | |
101 | ||
102 | if (start + offset < offset) { | |
103 | dev_err(&dev->dev, | |
104 | "virtio_pci: map wrap-around %u+%u\n", | |
105 | start, offset); | |
106 | return NULL; | |
107 | } | |
108 | ||
109 | offset += start; | |
110 | ||
111 | if (offset & (align - 1)) { | |
112 | dev_err(&dev->dev, | |
113 | "virtio_pci: offset %u not aligned to %u\n", | |
114 | offset, align); | |
115 | return NULL; | |
116 | } | |
117 | ||
118 | if (length > size) | |
119 | length = size; | |
120 | ||
121 | if (len) | |
122 | *len = length; | |
123 | ||
124 | if (minlen + offset < minlen || | |
125 | minlen + offset > pci_resource_len(dev, bar)) { | |
126 | dev_err(&dev->dev, | |
127 | "virtio_pci: map virtio %zu@%u " | |
128 | "out of range on bar %i length %lu\n", | |
129 | minlen, offset, | |
130 | bar, (unsigned long)pci_resource_len(dev, bar)); | |
131 | return NULL; | |
132 | } | |
133 | ||
134 | p = pci_iomap_range(dev, bar, offset, length); | |
135 | if (!p) | |
136 | dev_err(&dev->dev, | |
137 | "virtio_pci: unable to map virtio %u@%u on bar %i\n", | |
138 | length, offset, bar); | |
139 | return p; | |
140 | } | |
141 | ||
1fcf0512 MT |
142 | /* virtio config->get_features() implementation */ |
143 | static u64 vp_get_features(struct virtio_device *vdev) | |
144 | { | |
145 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
146 | u64 features; | |
147 | ||
a8557d32 MT |
148 | vp_iowrite32(0, &vp_dev->common->device_feature_select); |
149 | features = vp_ioread32(&vp_dev->common->device_feature); | |
150 | vp_iowrite32(1, &vp_dev->common->device_feature_select); | |
151 | features |= ((u64)vp_ioread32(&vp_dev->common->device_feature) << 32); | |
1fcf0512 MT |
152 | |
153 | return features; | |
154 | } | |
155 | ||
156 | /* virtio config->finalize_features() implementation */ | |
157 | static int vp_finalize_features(struct virtio_device *vdev) | |
158 | { | |
159 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
160 | ||
161 | /* Give virtio_ring a chance to accept features. */ | |
162 | vring_transport_features(vdev); | |
163 | ||
164 | if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) { | |
165 | dev_err(&vdev->dev, "virtio: device uses modern interface " | |
166 | "but does not have VIRTIO_F_VERSION_1\n"); | |
167 | return -EINVAL; | |
168 | } | |
169 | ||
a8557d32 MT |
170 | vp_iowrite32(0, &vp_dev->common->guest_feature_select); |
171 | vp_iowrite32((u32)vdev->features, &vp_dev->common->guest_feature); | |
172 | vp_iowrite32(1, &vp_dev->common->guest_feature_select); | |
173 | vp_iowrite32(vdev->features >> 32, &vp_dev->common->guest_feature); | |
1fcf0512 MT |
174 | |
175 | return 0; | |
176 | } | |
177 | ||
178 | /* virtio config->get() implementation */ | |
179 | static void vp_get(struct virtio_device *vdev, unsigned offset, | |
180 | void *buf, unsigned len) | |
181 | { | |
182 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
183 | u8 b; | |
184 | __le16 w; | |
185 | __le32 l; | |
186 | ||
187 | BUG_ON(offset + len > vp_dev->device_len); | |
188 | ||
189 | switch (len) { | |
190 | case 1: | |
191 | b = ioread8(vp_dev->device + offset); | |
192 | memcpy(buf, &b, sizeof b); | |
193 | break; | |
194 | case 2: | |
195 | w = cpu_to_le16(ioread16(vp_dev->device + offset)); | |
196 | memcpy(buf, &w, sizeof w); | |
197 | break; | |
198 | case 4: | |
199 | l = cpu_to_le32(ioread32(vp_dev->device + offset)); | |
200 | memcpy(buf, &l, sizeof l); | |
201 | break; | |
202 | case 8: | |
203 | l = cpu_to_le32(ioread32(vp_dev->device + offset)); | |
204 | memcpy(buf, &l, sizeof l); | |
205 | l = cpu_to_le32(ioread32(vp_dev->device + offset + sizeof l)); | |
206 | memcpy(buf + sizeof l, &l, sizeof l); | |
207 | break; | |
208 | default: | |
209 | BUG(); | |
210 | } | |
211 | } | |
212 | ||
213 | /* the config->set() implementation. it's symmetric to the config->get() | |
214 | * implementation */ | |
215 | static void vp_set(struct virtio_device *vdev, unsigned offset, | |
216 | const void *buf, unsigned len) | |
217 | { | |
218 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
219 | u8 b; | |
220 | __le16 w; | |
221 | __le32 l; | |
222 | ||
223 | BUG_ON(offset + len > vp_dev->device_len); | |
224 | ||
225 | switch (len) { | |
226 | case 1: | |
227 | memcpy(&b, buf, sizeof b); | |
228 | iowrite8(b, vp_dev->device + offset); | |
229 | break; | |
230 | case 2: | |
231 | memcpy(&w, buf, sizeof w); | |
232 | iowrite16(le16_to_cpu(w), vp_dev->device + offset); | |
233 | break; | |
234 | case 4: | |
235 | memcpy(&l, buf, sizeof l); | |
236 | iowrite32(le32_to_cpu(l), vp_dev->device + offset); | |
237 | break; | |
238 | case 8: | |
239 | memcpy(&l, buf, sizeof l); | |
240 | iowrite32(le32_to_cpu(l), vp_dev->device + offset); | |
241 | memcpy(&l, buf + sizeof l, sizeof l); | |
242 | iowrite32(le32_to_cpu(l), vp_dev->device + offset + sizeof l); | |
243 | break; | |
244 | default: | |
245 | BUG(); | |
246 | } | |
247 | } | |
248 | ||
249 | static u32 vp_generation(struct virtio_device *vdev) | |
250 | { | |
251 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
a8557d32 | 252 | return vp_ioread8(&vp_dev->common->config_generation); |
1fcf0512 MT |
253 | } |
254 | ||
255 | /* config->{get,set}_status() implementations */ | |
256 | static u8 vp_get_status(struct virtio_device *vdev) | |
257 | { | |
258 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
a8557d32 | 259 | return vp_ioread8(&vp_dev->common->device_status); |
1fcf0512 MT |
260 | } |
261 | ||
262 | static void vp_set_status(struct virtio_device *vdev, u8 status) | |
263 | { | |
264 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
265 | /* We should never be setting status to 0. */ | |
266 | BUG_ON(status == 0); | |
a8557d32 | 267 | vp_iowrite8(status, &vp_dev->common->device_status); |
1fcf0512 MT |
268 | } |
269 | ||
270 | static void vp_reset(struct virtio_device *vdev) | |
271 | { | |
272 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
273 | /* 0 status means a reset. */ | |
a8557d32 | 274 | vp_iowrite8(0, &vp_dev->common->device_status); |
05dbcb43 MT |
275 | /* After writing 0 to device_status, the driver MUST wait for a read of |
276 | * device_status to return 0 before reinitializing the device. | |
277 | * This will flush out the status write, and flush in device writes, | |
278 | * including MSI-X interrupts, if any. | |
279 | */ | |
280 | while (vp_ioread8(&vp_dev->common->device_status)) | |
281 | msleep(1); | |
1fcf0512 MT |
282 | /* Flush pending VQ/configuration callbacks. */ |
283 | vp_synchronize_vectors(vdev); | |
284 | } | |
285 | ||
286 | static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) | |
287 | { | |
288 | /* Setup the vector used for configuration events */ | |
a8557d32 | 289 | vp_iowrite16(vector, &vp_dev->common->msix_config); |
1fcf0512 MT |
290 | /* Verify we had enough resources to assign the vector */ |
291 | /* Will also flush the write out to device */ | |
a8557d32 | 292 | return vp_ioread16(&vp_dev->common->msix_config); |
1fcf0512 MT |
293 | } |
294 | ||
1fcf0512 | 295 | static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, |
0a9b3f47 | 296 | struct virtio_pci_vq_info *info, |
1fcf0512 MT |
297 | unsigned index, |
298 | void (*callback)(struct virtqueue *vq), | |
299 | const char *name, | |
f94682dd | 300 | bool ctx, |
1fcf0512 MT |
301 | u16 msix_vec) |
302 | { | |
303 | struct virtio_pci_common_cfg __iomem *cfg = vp_dev->common; | |
304 | struct virtqueue *vq; | |
305 | u16 num, off; | |
306 | int err; | |
307 | ||
a8557d32 | 308 | if (index >= vp_ioread16(&cfg->num_queues)) |
1fcf0512 MT |
309 | return ERR_PTR(-ENOENT); |
310 | ||
311 | /* Select the queue we're interested in */ | |
a8557d32 | 312 | vp_iowrite16(index, &cfg->queue_select); |
1fcf0512 MT |
313 | |
314 | /* Check if queue is either not available or already active. */ | |
a8557d32 MT |
315 | num = vp_ioread16(&cfg->queue_size); |
316 | if (!num || vp_ioread16(&cfg->queue_enable)) | |
1fcf0512 MT |
317 | return ERR_PTR(-ENOENT); |
318 | ||
319 | if (num & (num - 1)) { | |
320 | dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num); | |
321 | return ERR_PTR(-EINVAL); | |
322 | } | |
323 | ||
324 | /* get offset of notification word for this vq */ | |
a8557d32 | 325 | off = vp_ioread16(&cfg->queue_notify_off); |
1fcf0512 | 326 | |
0a9b3f47 MT |
327 | info->msix_vector = msix_vec; |
328 | ||
1fcf0512 | 329 | /* create the vring */ |
7a5589b2 AL |
330 | vq = vring_create_virtqueue(index, num, |
331 | SMP_CACHE_BYTES, &vp_dev->vdev, | |
f94682dd MT |
332 | true, true, ctx, |
333 | vp_notify, callback, name); | |
7a5589b2 AL |
334 | if (!vq) |
335 | return ERR_PTR(-ENOMEM); | |
1fcf0512 MT |
336 | |
337 | /* activate the queue */ | |
7a5589b2 AL |
338 | vp_iowrite16(virtqueue_get_vring_size(vq), &cfg->queue_size); |
339 | vp_iowrite64_twopart(virtqueue_get_desc_addr(vq), | |
a8557d32 | 340 | &cfg->queue_desc_lo, &cfg->queue_desc_hi); |
7a5589b2 | 341 | vp_iowrite64_twopart(virtqueue_get_avail_addr(vq), |
a8557d32 | 342 | &cfg->queue_avail_lo, &cfg->queue_avail_hi); |
7a5589b2 | 343 | vp_iowrite64_twopart(virtqueue_get_used_addr(vq), |
a8557d32 | 344 | &cfg->queue_used_lo, &cfg->queue_used_hi); |
1fcf0512 | 345 | |
3909213c MT |
346 | if (vp_dev->notify_base) { |
347 | /* offset should not wrap */ | |
348 | if ((u64)off * vp_dev->notify_offset_multiplier + 2 | |
349 | > vp_dev->notify_len) { | |
350 | dev_warn(&vp_dev->pci_dev->dev, | |
351 | "bad notification offset %u (x %u) " | |
352 | "for queue %u > %zd", | |
353 | off, vp_dev->notify_offset_multiplier, | |
354 | index, vp_dev->notify_len); | |
355 | err = -EINVAL; | |
356 | goto err_map_notify; | |
357 | } | |
358 | vq->priv = (void __force *)vp_dev->notify_base + | |
359 | off * vp_dev->notify_offset_multiplier; | |
360 | } else { | |
361 | vq->priv = (void __force *)map_capability(vp_dev->pci_dev, | |
362 | vp_dev->notify_map_cap, 2, 2, | |
363 | off * vp_dev->notify_offset_multiplier, 2, | |
364 | NULL); | |
365 | } | |
1fcf0512 MT |
366 | |
367 | if (!vq->priv) { | |
368 | err = -ENOMEM; | |
369 | goto err_map_notify; | |
370 | } | |
371 | ||
372 | if (msix_vec != VIRTIO_MSI_NO_VECTOR) { | |
a8557d32 MT |
373 | vp_iowrite16(msix_vec, &cfg->queue_msix_vector); |
374 | msix_vec = vp_ioread16(&cfg->queue_msix_vector); | |
1fcf0512 MT |
375 | if (msix_vec == VIRTIO_MSI_NO_VECTOR) { |
376 | err = -EBUSY; | |
377 | goto err_assign_vector; | |
378 | } | |
379 | } | |
380 | ||
381 | return vq; | |
382 | ||
383 | err_assign_vector: | |
3909213c MT |
384 | if (!vp_dev->notify_base) |
385 | pci_iounmap(vp_dev->pci_dev, (void __iomem __force *)vq->priv); | |
1fcf0512 MT |
386 | err_map_notify: |
387 | vring_del_virtqueue(vq); | |
1fcf0512 MT |
388 | return ERR_PTR(err); |
389 | } | |
390 | ||
391 | static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs, | |
f94682dd MT |
392 | struct virtqueue *vqs[], |
393 | vq_callback_t *callbacks[], | |
394 | const char * const names[], const bool *ctx, | |
395 | struct irq_affinity *desc) | |
1fcf0512 MT |
396 | { |
397 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
398 | struct virtqueue *vq; | |
f94682dd | 399 | int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc); |
1fcf0512 MT |
400 | |
401 | if (rc) | |
402 | return rc; | |
403 | ||
404 | /* Select and activate all queues. Has to be done last: once we do | |
405 | * this, there's no way to go back except reset. | |
406 | */ | |
407 | list_for_each_entry(vq, &vdev->vqs, list) { | |
a8557d32 MT |
408 | vp_iowrite16(vq->index, &vp_dev->common->queue_select); |
409 | vp_iowrite16(1, &vp_dev->common->queue_enable); | |
1fcf0512 MT |
410 | } |
411 | ||
412 | return 0; | |
413 | } | |
414 | ||
0a9b3f47 | 415 | static void del_vq(struct virtio_pci_vq_info *info) |
1fcf0512 | 416 | { |
0a9b3f47 | 417 | struct virtqueue *vq = info->vq; |
1fcf0512 MT |
418 | struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); |
419 | ||
a8557d32 | 420 | vp_iowrite16(vq->index, &vp_dev->common->queue_select); |
1fcf0512 | 421 | |
2008c154 | 422 | if (vp_dev->msix_enabled) { |
a8557d32 MT |
423 | vp_iowrite16(VIRTIO_MSI_NO_VECTOR, |
424 | &vp_dev->common->queue_msix_vector); | |
1fcf0512 | 425 | /* Flush the write out to device */ |
a8557d32 | 426 | vp_ioread16(&vp_dev->common->queue_msix_vector); |
1fcf0512 MT |
427 | } |
428 | ||
3909213c MT |
429 | if (!vp_dev->notify_base) |
430 | pci_iounmap(vp_dev->pci_dev, (void __force __iomem *)vq->priv); | |
1fcf0512 MT |
431 | |
432 | vring_del_virtqueue(vq); | |
1fcf0512 MT |
433 | } |
434 | ||
d3f5f065 MT |
435 | static const struct virtio_config_ops virtio_pci_config_nodev_ops = { |
436 | .get = NULL, | |
437 | .set = NULL, | |
438 | .generation = vp_generation, | |
439 | .get_status = vp_get_status, | |
440 | .set_status = vp_set_status, | |
441 | .reset = vp_reset, | |
442 | .find_vqs = vp_modern_find_vqs, | |
443 | .del_vqs = vp_del_vqs, | |
444 | .get_features = vp_get_features, | |
445 | .finalize_features = vp_finalize_features, | |
446 | .bus_name = vp_bus_name, | |
447 | .set_vq_affinity = vp_set_vq_affinity, | |
bbaba479 | 448 | .get_vq_affinity = vp_get_vq_affinity, |
d3f5f065 MT |
449 | }; |
450 | ||
1fcf0512 MT |
451 | static const struct virtio_config_ops virtio_pci_config_ops = { |
452 | .get = vp_get, | |
453 | .set = vp_set, | |
454 | .generation = vp_generation, | |
455 | .get_status = vp_get_status, | |
456 | .set_status = vp_set_status, | |
457 | .reset = vp_reset, | |
458 | .find_vqs = vp_modern_find_vqs, | |
459 | .del_vqs = vp_del_vqs, | |
460 | .get_features = vp_get_features, | |
461 | .finalize_features = vp_finalize_features, | |
462 | .bus_name = vp_bus_name, | |
463 | .set_vq_affinity = vp_set_vq_affinity, | |
bbaba479 | 464 | .get_vq_affinity = vp_get_vq_affinity, |
1fcf0512 MT |
465 | }; |
466 | ||
467 | /** | |
468 | * virtio_pci_find_capability - walk capabilities to find device info. | |
469 | * @dev: the pci device | |
470 | * @cfg_type: the VIRTIO_PCI_CAP_* value we seek | |
471 | * @ioresource_types: IORESOURCE_MEM and/or IORESOURCE_IO. | |
472 | * | |
473 | * Returns offset of the capability, or 0. | |
474 | */ | |
475 | static inline int virtio_pci_find_capability(struct pci_dev *dev, u8 cfg_type, | |
59a5b0f7 | 476 | u32 ioresource_types, int *bars) |
1fcf0512 MT |
477 | { |
478 | int pos; | |
479 | ||
480 | for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); | |
481 | pos > 0; | |
482 | pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { | |
483 | u8 type, bar; | |
484 | pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, | |
485 | cfg_type), | |
486 | &type); | |
487 | pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, | |
488 | bar), | |
489 | &bar); | |
490 | ||
491 | /* Ignore structures with reserved BAR values */ | |
492 | if (bar > 0x5) | |
493 | continue; | |
494 | ||
495 | if (type == cfg_type) { | |
496 | if (pci_resource_len(dev, bar) && | |
59a5b0f7 GH |
497 | pci_resource_flags(dev, bar) & ioresource_types) { |
498 | *bars |= (1 << bar); | |
1fcf0512 | 499 | return pos; |
59a5b0f7 | 500 | } |
1fcf0512 MT |
501 | } |
502 | } | |
503 | return 0; | |
504 | } | |
505 | ||
89461c4a | 506 | /* This is part of the ABI. Don't screw with it. */ |
1fcf0512 MT |
507 | static inline void check_offsets(void) |
508 | { | |
89461c4a RR |
509 | /* Note: disk space was harmed in compilation of this function. */ |
510 | BUILD_BUG_ON(VIRTIO_PCI_CAP_VNDR != | |
511 | offsetof(struct virtio_pci_cap, cap_vndr)); | |
512 | BUILD_BUG_ON(VIRTIO_PCI_CAP_NEXT != | |
513 | offsetof(struct virtio_pci_cap, cap_next)); | |
514 | BUILD_BUG_ON(VIRTIO_PCI_CAP_LEN != | |
515 | offsetof(struct virtio_pci_cap, cap_len)); | |
516 | BUILD_BUG_ON(VIRTIO_PCI_CAP_CFG_TYPE != | |
517 | offsetof(struct virtio_pci_cap, cfg_type)); | |
518 | BUILD_BUG_ON(VIRTIO_PCI_CAP_BAR != | |
519 | offsetof(struct virtio_pci_cap, bar)); | |
520 | BUILD_BUG_ON(VIRTIO_PCI_CAP_OFFSET != | |
521 | offsetof(struct virtio_pci_cap, offset)); | |
522 | BUILD_BUG_ON(VIRTIO_PCI_CAP_LENGTH != | |
523 | offsetof(struct virtio_pci_cap, length)); | |
524 | BUILD_BUG_ON(VIRTIO_PCI_NOTIFY_CAP_MULT != | |
525 | offsetof(struct virtio_pci_notify_cap, | |
526 | notify_off_multiplier)); | |
527 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_DFSELECT != | |
528 | offsetof(struct virtio_pci_common_cfg, | |
529 | device_feature_select)); | |
530 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_DF != | |
531 | offsetof(struct virtio_pci_common_cfg, device_feature)); | |
532 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_GFSELECT != | |
533 | offsetof(struct virtio_pci_common_cfg, | |
534 | guest_feature_select)); | |
535 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_GF != | |
536 | offsetof(struct virtio_pci_common_cfg, guest_feature)); | |
537 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_MSIX != | |
538 | offsetof(struct virtio_pci_common_cfg, msix_config)); | |
539 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_NUMQ != | |
540 | offsetof(struct virtio_pci_common_cfg, num_queues)); | |
541 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_STATUS != | |
542 | offsetof(struct virtio_pci_common_cfg, device_status)); | |
543 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_CFGGENERATION != | |
544 | offsetof(struct virtio_pci_common_cfg, config_generation)); | |
545 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SELECT != | |
546 | offsetof(struct virtio_pci_common_cfg, queue_select)); | |
547 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SIZE != | |
548 | offsetof(struct virtio_pci_common_cfg, queue_size)); | |
549 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_MSIX != | |
550 | offsetof(struct virtio_pci_common_cfg, queue_msix_vector)); | |
551 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_ENABLE != | |
552 | offsetof(struct virtio_pci_common_cfg, queue_enable)); | |
553 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_NOFF != | |
554 | offsetof(struct virtio_pci_common_cfg, queue_notify_off)); | |
555 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCLO != | |
556 | offsetof(struct virtio_pci_common_cfg, queue_desc_lo)); | |
557 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCHI != | |
558 | offsetof(struct virtio_pci_common_cfg, queue_desc_hi)); | |
559 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILLO != | |
560 | offsetof(struct virtio_pci_common_cfg, queue_avail_lo)); | |
561 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILHI != | |
562 | offsetof(struct virtio_pci_common_cfg, queue_avail_hi)); | |
563 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDLO != | |
564 | offsetof(struct virtio_pci_common_cfg, queue_used_lo)); | |
565 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDHI != | |
566 | offsetof(struct virtio_pci_common_cfg, queue_used_hi)); | |
1fcf0512 MT |
567 | } |
568 | ||
569 | /* the PCI probing function */ | |
570 | int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev) | |
571 | { | |
572 | struct pci_dev *pci_dev = vp_dev->pci_dev; | |
573 | int err, common, isr, notify, device; | |
574 | u32 notify_length; | |
3909213c | 575 | u32 notify_offset; |
1fcf0512 MT |
576 | |
577 | check_offsets(); | |
578 | ||
579 | /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ | |
580 | if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f) | |
581 | return -ENODEV; | |
582 | ||
583 | if (pci_dev->device < 0x1040) { | |
584 | /* Transitional devices: use the PCI subsystem device id as | |
585 | * virtio device id, same as legacy driver always did. | |
586 | */ | |
587 | vp_dev->vdev.id.device = pci_dev->subsystem_device; | |
588 | } else { | |
589 | /* Modern devices: simply use PCI device id, but start from 0x1040. */ | |
590 | vp_dev->vdev.id.device = pci_dev->device - 0x1040; | |
591 | } | |
592 | vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor; | |
593 | ||
1fcf0512 MT |
594 | /* check for a common config: if not, use legacy mode (bar 0). */ |
595 | common = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_COMMON_CFG, | |
59a5b0f7 GH |
596 | IORESOURCE_IO | IORESOURCE_MEM, |
597 | &vp_dev->modern_bars); | |
1fcf0512 MT |
598 | if (!common) { |
599 | dev_info(&pci_dev->dev, | |
600 | "virtio_pci: leaving for legacy driver\n"); | |
601 | return -ENODEV; | |
602 | } | |
603 | ||
604 | /* If common is there, these should be too... */ | |
605 | isr = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_ISR_CFG, | |
59a5b0f7 GH |
606 | IORESOURCE_IO | IORESOURCE_MEM, |
607 | &vp_dev->modern_bars); | |
1fcf0512 | 608 | notify = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_NOTIFY_CFG, |
59a5b0f7 GH |
609 | IORESOURCE_IO | IORESOURCE_MEM, |
610 | &vp_dev->modern_bars); | |
1fcf0512 MT |
611 | if (!isr || !notify) { |
612 | dev_err(&pci_dev->dev, | |
613 | "virtio_pci: missing capabilities %i/%i/%i\n", | |
614 | common, isr, notify); | |
615 | return -EINVAL; | |
616 | } | |
617 | ||
7a5589b2 AL |
618 | err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)); |
619 | if (err) | |
620 | err = dma_set_mask_and_coherent(&pci_dev->dev, | |
621 | DMA_BIT_MASK(32)); | |
622 | if (err) | |
623 | dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); | |
624 | ||
1fcf0512 MT |
625 | /* Device capability is only mandatory for devices that have |
626 | * device-specific configuration. | |
627 | */ | |
628 | device = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG, | |
59a5b0f7 GH |
629 | IORESOURCE_IO | IORESOURCE_MEM, |
630 | &vp_dev->modern_bars); | |
631 | ||
632 | err = pci_request_selected_regions(pci_dev, vp_dev->modern_bars, | |
633 | "virtio-pci-modern"); | |
634 | if (err) | |
635 | return err; | |
1fcf0512 MT |
636 | |
637 | err = -EINVAL; | |
638 | vp_dev->common = map_capability(pci_dev, common, | |
639 | sizeof(struct virtio_pci_common_cfg), 4, | |
640 | 0, sizeof(struct virtio_pci_common_cfg), | |
641 | NULL); | |
642 | if (!vp_dev->common) | |
643 | goto err_map_common; | |
644 | vp_dev->isr = map_capability(pci_dev, isr, sizeof(u8), 1, | |
645 | 0, 1, | |
646 | NULL); | |
647 | if (!vp_dev->isr) | |
648 | goto err_map_isr; | |
649 | ||
650 | /* Read notify_off_multiplier from config space. */ | |
651 | pci_read_config_dword(pci_dev, | |
652 | notify + offsetof(struct virtio_pci_notify_cap, | |
653 | notify_off_multiplier), | |
654 | &vp_dev->notify_offset_multiplier); | |
3909213c | 655 | /* Read notify length and offset from config space. */ |
1fcf0512 MT |
656 | pci_read_config_dword(pci_dev, |
657 | notify + offsetof(struct virtio_pci_notify_cap, | |
658 | cap.length), | |
659 | ¬ify_length); | |
660 | ||
3909213c MT |
661 | pci_read_config_dword(pci_dev, |
662 | notify + offsetof(struct virtio_pci_notify_cap, | |
4e94ebdd | 663 | cap.offset), |
3909213c MT |
664 | ¬ify_offset); |
665 | ||
666 | /* We don't know how many VQs we'll map, ahead of the time. | |
667 | * If notify length is small, map it all now. | |
668 | * Otherwise, map each VQ individually later. | |
669 | */ | |
670 | if ((u64)notify_length + (notify_offset % PAGE_SIZE) <= PAGE_SIZE) { | |
671 | vp_dev->notify_base = map_capability(pci_dev, notify, 2, 2, | |
672 | 0, notify_length, | |
673 | &vp_dev->notify_len); | |
674 | if (!vp_dev->notify_base) | |
675 | goto err_map_notify; | |
676 | } else { | |
677 | vp_dev->notify_map_cap = notify; | |
678 | } | |
1fcf0512 MT |
679 | |
680 | /* Again, we don't know how much we should map, but PAGE_SIZE | |
681 | * is more than enough for all existing devices. | |
682 | */ | |
683 | if (device) { | |
684 | vp_dev->device = map_capability(pci_dev, device, 0, 4, | |
685 | 0, PAGE_SIZE, | |
686 | &vp_dev->device_len); | |
687 | if (!vp_dev->device) | |
688 | goto err_map_device; | |
1fcf0512 | 689 | |
d3f5f065 MT |
690 | vp_dev->vdev.config = &virtio_pci_config_ops; |
691 | } else { | |
692 | vp_dev->vdev.config = &virtio_pci_config_nodev_ops; | |
693 | } | |
1fcf0512 MT |
694 | |
695 | vp_dev->config_vector = vp_config_vector; | |
696 | vp_dev->setup_vq = setup_vq; | |
697 | vp_dev->del_vq = del_vq; | |
698 | ||
699 | return 0; | |
700 | ||
701 | err_map_device: | |
3909213c MT |
702 | if (vp_dev->notify_base) |
703 | pci_iounmap(pci_dev, vp_dev->notify_base); | |
704 | err_map_notify: | |
1fcf0512 MT |
705 | pci_iounmap(pci_dev, vp_dev->isr); |
706 | err_map_isr: | |
707 | pci_iounmap(pci_dev, vp_dev->common); | |
708 | err_map_common: | |
709 | return err; | |
710 | } | |
711 | ||
712 | void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev) | |
713 | { | |
714 | struct pci_dev *pci_dev = vp_dev->pci_dev; | |
715 | ||
716 | if (vp_dev->device) | |
717 | pci_iounmap(pci_dev, vp_dev->device); | |
3909213c MT |
718 | if (vp_dev->notify_base) |
719 | pci_iounmap(pci_dev, vp_dev->notify_base); | |
1fcf0512 MT |
720 | pci_iounmap(pci_dev, vp_dev->isr); |
721 | pci_iounmap(pci_dev, vp_dev->common); | |
59a5b0f7 | 722 | pci_release_selected_regions(pci_dev, vp_dev->modern_bars); |
1fcf0512 | 723 | } |