]>
Commit | Line | Data |
---|---|---|
1fcf0512 MT |
1 | /* |
2 | * Virtio PCI driver - modern (virtio 1.0) device support | |
3 | * | |
4 | * This module allows virtio devices to be used over a virtual PCI device. | |
5 | * This can be used with QEMU based VMMs like KVM or Xen. | |
6 | * | |
7 | * Copyright IBM Corp. 2007 | |
8 | * Copyright Red Hat, Inc. 2014 | |
9 | * | |
10 | * Authors: | |
11 | * Anthony Liguori <aliguori@us.ibm.com> | |
12 | * Rusty Russell <rusty@rustcorp.com.au> | |
13 | * Michael S. Tsirkin <mst@redhat.com> | |
14 | * | |
15 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
16 | * See the COPYING file in the top-level directory. | |
17 | * | |
18 | */ | |
19 | ||
20 | #define VIRTIO_PCI_NO_LEGACY | |
21 | #include "virtio_pci_common.h" | |
22 | ||
c5d4c2c9 MT |
23 | /* |
24 | * Type-safe wrappers for io accesses. | |
25 | * Use these to enforce at compile time the following spec requirement: | |
26 | * | |
27 | * The driver MUST access each field using the “natural” access | |
28 | * method, i.e. 32-bit accesses for 32-bit fields, 16-bit accesses | |
29 | * for 16-bit fields and 8-bit accesses for 8-bit fields. | |
30 | */ | |
31 | static inline u8 vp_ioread8(u8 __iomem *addr) | |
32 | { | |
33 | return ioread8(addr); | |
34 | } | |
35 | static inline u16 vp_ioread16 (u16 __iomem *addr) | |
36 | { | |
37 | return ioread16(addr); | |
38 | } | |
39 | ||
40 | static inline u32 vp_ioread32(u32 __iomem *addr) | |
41 | { | |
42 | return ioread32(addr); | |
43 | } | |
44 | ||
45 | static inline void vp_iowrite8(u8 value, u8 __iomem *addr) | |
46 | { | |
47 | iowrite8(value, addr); | |
48 | } | |
49 | ||
50 | static inline void vp_iowrite16(u16 value, u16 __iomem *addr) | |
51 | { | |
52 | iowrite16(value, addr); | |
53 | } | |
54 | ||
55 | static inline void vp_iowrite32(u32 value, u32 __iomem *addr) | |
56 | { | |
57 | iowrite32(value, addr); | |
58 | } | |
59 | ||
a8557d32 MT |
60 | static void vp_iowrite64_twopart(u64 val, |
61 | __le32 __iomem *lo, __le32 __iomem *hi) | |
62 | { | |
63 | vp_iowrite32((u32)val, lo); | |
64 | vp_iowrite32(val >> 32, hi); | |
65 | } | |
66 | ||
1fcf0512 MT |
67 | static void __iomem *map_capability(struct pci_dev *dev, int off, |
68 | size_t minlen, | |
69 | u32 align, | |
70 | u32 start, u32 size, | |
71 | size_t *len) | |
72 | { | |
73 | u8 bar; | |
74 | u32 offset, length; | |
75 | void __iomem *p; | |
76 | ||
77 | pci_read_config_byte(dev, off + offsetof(struct virtio_pci_cap, | |
78 | bar), | |
79 | &bar); | |
80 | pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, offset), | |
81 | &offset); | |
82 | pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length), | |
83 | &length); | |
84 | ||
85 | if (length <= start) { | |
86 | dev_err(&dev->dev, | |
87 | "virtio_pci: bad capability len %u (>%u expected)\n", | |
88 | length, start); | |
89 | return NULL; | |
90 | } | |
91 | ||
92 | if (length - start < minlen) { | |
93 | dev_err(&dev->dev, | |
94 | "virtio_pci: bad capability len %u (>=%zu expected)\n", | |
95 | length, minlen); | |
96 | return NULL; | |
97 | } | |
98 | ||
99 | length -= start; | |
100 | ||
101 | if (start + offset < offset) { | |
102 | dev_err(&dev->dev, | |
103 | "virtio_pci: map wrap-around %u+%u\n", | |
104 | start, offset); | |
105 | return NULL; | |
106 | } | |
107 | ||
108 | offset += start; | |
109 | ||
110 | if (offset & (align - 1)) { | |
111 | dev_err(&dev->dev, | |
112 | "virtio_pci: offset %u not aligned to %u\n", | |
113 | offset, align); | |
114 | return NULL; | |
115 | } | |
116 | ||
117 | if (length > size) | |
118 | length = size; | |
119 | ||
120 | if (len) | |
121 | *len = length; | |
122 | ||
123 | if (minlen + offset < minlen || | |
124 | minlen + offset > pci_resource_len(dev, bar)) { | |
125 | dev_err(&dev->dev, | |
126 | "virtio_pci: map virtio %zu@%u " | |
127 | "out of range on bar %i length %lu\n", | |
128 | minlen, offset, | |
129 | bar, (unsigned long)pci_resource_len(dev, bar)); | |
130 | return NULL; | |
131 | } | |
132 | ||
133 | p = pci_iomap_range(dev, bar, offset, length); | |
134 | if (!p) | |
135 | dev_err(&dev->dev, | |
136 | "virtio_pci: unable to map virtio %u@%u on bar %i\n", | |
137 | length, offset, bar); | |
138 | return p; | |
139 | } | |
140 | ||
1fcf0512 MT |
141 | /* virtio config->get_features() implementation */ |
142 | static u64 vp_get_features(struct virtio_device *vdev) | |
143 | { | |
144 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
145 | u64 features; | |
146 | ||
a8557d32 MT |
147 | vp_iowrite32(0, &vp_dev->common->device_feature_select); |
148 | features = vp_ioread32(&vp_dev->common->device_feature); | |
149 | vp_iowrite32(1, &vp_dev->common->device_feature_select); | |
150 | features |= ((u64)vp_ioread32(&vp_dev->common->device_feature) << 32); | |
1fcf0512 MT |
151 | |
152 | return features; | |
153 | } | |
154 | ||
155 | /* virtio config->finalize_features() implementation */ | |
156 | static int vp_finalize_features(struct virtio_device *vdev) | |
157 | { | |
158 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
159 | ||
160 | /* Give virtio_ring a chance to accept features. */ | |
161 | vring_transport_features(vdev); | |
162 | ||
163 | if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) { | |
164 | dev_err(&vdev->dev, "virtio: device uses modern interface " | |
165 | "but does not have VIRTIO_F_VERSION_1\n"); | |
166 | return -EINVAL; | |
167 | } | |
168 | ||
a8557d32 MT |
169 | vp_iowrite32(0, &vp_dev->common->guest_feature_select); |
170 | vp_iowrite32((u32)vdev->features, &vp_dev->common->guest_feature); | |
171 | vp_iowrite32(1, &vp_dev->common->guest_feature_select); | |
172 | vp_iowrite32(vdev->features >> 32, &vp_dev->common->guest_feature); | |
1fcf0512 MT |
173 | |
174 | return 0; | |
175 | } | |
176 | ||
177 | /* virtio config->get() implementation */ | |
178 | static void vp_get(struct virtio_device *vdev, unsigned offset, | |
179 | void *buf, unsigned len) | |
180 | { | |
181 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
182 | u8 b; | |
183 | __le16 w; | |
184 | __le32 l; | |
185 | ||
186 | BUG_ON(offset + len > vp_dev->device_len); | |
187 | ||
188 | switch (len) { | |
189 | case 1: | |
190 | b = ioread8(vp_dev->device + offset); | |
191 | memcpy(buf, &b, sizeof b); | |
192 | break; | |
193 | case 2: | |
194 | w = cpu_to_le16(ioread16(vp_dev->device + offset)); | |
195 | memcpy(buf, &w, sizeof w); | |
196 | break; | |
197 | case 4: | |
198 | l = cpu_to_le32(ioread32(vp_dev->device + offset)); | |
199 | memcpy(buf, &l, sizeof l); | |
200 | break; | |
201 | case 8: | |
202 | l = cpu_to_le32(ioread32(vp_dev->device + offset)); | |
203 | memcpy(buf, &l, sizeof l); | |
204 | l = cpu_to_le32(ioread32(vp_dev->device + offset + sizeof l)); | |
205 | memcpy(buf + sizeof l, &l, sizeof l); | |
206 | break; | |
207 | default: | |
208 | BUG(); | |
209 | } | |
210 | } | |
211 | ||
212 | /* the config->set() implementation. it's symmetric to the config->get() | |
213 | * implementation */ | |
214 | static void vp_set(struct virtio_device *vdev, unsigned offset, | |
215 | const void *buf, unsigned len) | |
216 | { | |
217 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
218 | u8 b; | |
219 | __le16 w; | |
220 | __le32 l; | |
221 | ||
222 | BUG_ON(offset + len > vp_dev->device_len); | |
223 | ||
224 | switch (len) { | |
225 | case 1: | |
226 | memcpy(&b, buf, sizeof b); | |
227 | iowrite8(b, vp_dev->device + offset); | |
228 | break; | |
229 | case 2: | |
230 | memcpy(&w, buf, sizeof w); | |
231 | iowrite16(le16_to_cpu(w), vp_dev->device + offset); | |
232 | break; | |
233 | case 4: | |
234 | memcpy(&l, buf, sizeof l); | |
235 | iowrite32(le32_to_cpu(l), vp_dev->device + offset); | |
236 | break; | |
237 | case 8: | |
238 | memcpy(&l, buf, sizeof l); | |
239 | iowrite32(le32_to_cpu(l), vp_dev->device + offset); | |
240 | memcpy(&l, buf + sizeof l, sizeof l); | |
241 | iowrite32(le32_to_cpu(l), vp_dev->device + offset + sizeof l); | |
242 | break; | |
243 | default: | |
244 | BUG(); | |
245 | } | |
246 | } | |
247 | ||
248 | static u32 vp_generation(struct virtio_device *vdev) | |
249 | { | |
250 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
a8557d32 | 251 | return vp_ioread8(&vp_dev->common->config_generation); |
1fcf0512 MT |
252 | } |
253 | ||
254 | /* config->{get,set}_status() implementations */ | |
255 | static u8 vp_get_status(struct virtio_device *vdev) | |
256 | { | |
257 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
a8557d32 | 258 | return vp_ioread8(&vp_dev->common->device_status); |
1fcf0512 MT |
259 | } |
260 | ||
261 | static void vp_set_status(struct virtio_device *vdev, u8 status) | |
262 | { | |
263 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
264 | /* We should never be setting status to 0. */ | |
265 | BUG_ON(status == 0); | |
a8557d32 | 266 | vp_iowrite8(status, &vp_dev->common->device_status); |
1fcf0512 MT |
267 | } |
268 | ||
269 | static void vp_reset(struct virtio_device *vdev) | |
270 | { | |
271 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
272 | /* 0 status means a reset. */ | |
a8557d32 | 273 | vp_iowrite8(0, &vp_dev->common->device_status); |
1fcf0512 MT |
274 | /* Flush out the status write, and flush in device writes, |
275 | * including MSI-X interrupts, if any. */ | |
a8557d32 | 276 | vp_ioread8(&vp_dev->common->device_status); |
1fcf0512 MT |
277 | /* Flush pending VQ/configuration callbacks. */ |
278 | vp_synchronize_vectors(vdev); | |
279 | } | |
280 | ||
281 | static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) | |
282 | { | |
283 | /* Setup the vector used for configuration events */ | |
a8557d32 | 284 | vp_iowrite16(vector, &vp_dev->common->msix_config); |
1fcf0512 MT |
285 | /* Verify we had enough resources to assign the vector */ |
286 | /* Will also flush the write out to device */ | |
a8557d32 | 287 | return vp_ioread16(&vp_dev->common->msix_config); |
1fcf0512 MT |
288 | } |
289 | ||
1fcf0512 MT |
290 | static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, |
291 | struct virtio_pci_vq_info *info, | |
292 | unsigned index, | |
293 | void (*callback)(struct virtqueue *vq), | |
294 | const char *name, | |
295 | u16 msix_vec) | |
296 | { | |
297 | struct virtio_pci_common_cfg __iomem *cfg = vp_dev->common; | |
298 | struct virtqueue *vq; | |
299 | u16 num, off; | |
300 | int err; | |
301 | ||
a8557d32 | 302 | if (index >= vp_ioread16(&cfg->num_queues)) |
1fcf0512 MT |
303 | return ERR_PTR(-ENOENT); |
304 | ||
305 | /* Select the queue we're interested in */ | |
a8557d32 | 306 | vp_iowrite16(index, &cfg->queue_select); |
1fcf0512 MT |
307 | |
308 | /* Check if queue is either not available or already active. */ | |
a8557d32 MT |
309 | num = vp_ioread16(&cfg->queue_size); |
310 | if (!num || vp_ioread16(&cfg->queue_enable)) | |
1fcf0512 MT |
311 | return ERR_PTR(-ENOENT); |
312 | ||
313 | if (num & (num - 1)) { | |
314 | dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num); | |
315 | return ERR_PTR(-EINVAL); | |
316 | } | |
317 | ||
318 | /* get offset of notification word for this vq */ | |
a8557d32 | 319 | off = vp_ioread16(&cfg->queue_notify_off); |
1fcf0512 | 320 | |
1fcf0512 MT |
321 | info->msix_vector = msix_vec; |
322 | ||
1fcf0512 | 323 | /* create the vring */ |
7a5589b2 AL |
324 | vq = vring_create_virtqueue(index, num, |
325 | SMP_CACHE_BYTES, &vp_dev->vdev, | |
326 | true, true, vp_notify, callback, name); | |
327 | if (!vq) | |
328 | return ERR_PTR(-ENOMEM); | |
1fcf0512 MT |
329 | |
330 | /* activate the queue */ | |
7a5589b2 AL |
331 | vp_iowrite16(virtqueue_get_vring_size(vq), &cfg->queue_size); |
332 | vp_iowrite64_twopart(virtqueue_get_desc_addr(vq), | |
a8557d32 | 333 | &cfg->queue_desc_lo, &cfg->queue_desc_hi); |
7a5589b2 | 334 | vp_iowrite64_twopart(virtqueue_get_avail_addr(vq), |
a8557d32 | 335 | &cfg->queue_avail_lo, &cfg->queue_avail_hi); |
7a5589b2 | 336 | vp_iowrite64_twopart(virtqueue_get_used_addr(vq), |
a8557d32 | 337 | &cfg->queue_used_lo, &cfg->queue_used_hi); |
1fcf0512 | 338 | |
3909213c MT |
339 | if (vp_dev->notify_base) { |
340 | /* offset should not wrap */ | |
341 | if ((u64)off * vp_dev->notify_offset_multiplier + 2 | |
342 | > vp_dev->notify_len) { | |
343 | dev_warn(&vp_dev->pci_dev->dev, | |
344 | "bad notification offset %u (x %u) " | |
345 | "for queue %u > %zd", | |
346 | off, vp_dev->notify_offset_multiplier, | |
347 | index, vp_dev->notify_len); | |
348 | err = -EINVAL; | |
349 | goto err_map_notify; | |
350 | } | |
351 | vq->priv = (void __force *)vp_dev->notify_base + | |
352 | off * vp_dev->notify_offset_multiplier; | |
353 | } else { | |
354 | vq->priv = (void __force *)map_capability(vp_dev->pci_dev, | |
355 | vp_dev->notify_map_cap, 2, 2, | |
356 | off * vp_dev->notify_offset_multiplier, 2, | |
357 | NULL); | |
358 | } | |
1fcf0512 MT |
359 | |
360 | if (!vq->priv) { | |
361 | err = -ENOMEM; | |
362 | goto err_map_notify; | |
363 | } | |
364 | ||
365 | if (msix_vec != VIRTIO_MSI_NO_VECTOR) { | |
a8557d32 MT |
366 | vp_iowrite16(msix_vec, &cfg->queue_msix_vector); |
367 | msix_vec = vp_ioread16(&cfg->queue_msix_vector); | |
1fcf0512 MT |
368 | if (msix_vec == VIRTIO_MSI_NO_VECTOR) { |
369 | err = -EBUSY; | |
370 | goto err_assign_vector; | |
371 | } | |
372 | } | |
373 | ||
374 | return vq; | |
375 | ||
376 | err_assign_vector: | |
3909213c MT |
377 | if (!vp_dev->notify_base) |
378 | pci_iounmap(vp_dev->pci_dev, (void __iomem __force *)vq->priv); | |
1fcf0512 MT |
379 | err_map_notify: |
380 | vring_del_virtqueue(vq); | |
1fcf0512 MT |
381 | return ERR_PTR(err); |
382 | } | |
383 | ||
384 | static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs, | |
385 | struct virtqueue *vqs[], | |
386 | vq_callback_t *callbacks[], | |
f7ad26ff | 387 | const char * const names[]) |
1fcf0512 MT |
388 | { |
389 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
390 | struct virtqueue *vq; | |
391 | int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names); | |
392 | ||
393 | if (rc) | |
394 | return rc; | |
395 | ||
396 | /* Select and activate all queues. Has to be done last: once we do | |
397 | * this, there's no way to go back except reset. | |
398 | */ | |
399 | list_for_each_entry(vq, &vdev->vqs, list) { | |
a8557d32 MT |
400 | vp_iowrite16(vq->index, &vp_dev->common->queue_select); |
401 | vp_iowrite16(1, &vp_dev->common->queue_enable); | |
1fcf0512 MT |
402 | } |
403 | ||
404 | return 0; | |
405 | } | |
406 | ||
407 | static void del_vq(struct virtio_pci_vq_info *info) | |
408 | { | |
409 | struct virtqueue *vq = info->vq; | |
410 | struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); | |
411 | ||
a8557d32 | 412 | vp_iowrite16(vq->index, &vp_dev->common->queue_select); |
1fcf0512 MT |
413 | |
414 | if (vp_dev->msix_enabled) { | |
a8557d32 MT |
415 | vp_iowrite16(VIRTIO_MSI_NO_VECTOR, |
416 | &vp_dev->common->queue_msix_vector); | |
1fcf0512 | 417 | /* Flush the write out to device */ |
a8557d32 | 418 | vp_ioread16(&vp_dev->common->queue_msix_vector); |
1fcf0512 MT |
419 | } |
420 | ||
3909213c MT |
421 | if (!vp_dev->notify_base) |
422 | pci_iounmap(vp_dev->pci_dev, (void __force __iomem *)vq->priv); | |
1fcf0512 MT |
423 | |
424 | vring_del_virtqueue(vq); | |
1fcf0512 MT |
425 | } |
426 | ||
d3f5f065 MT |
427 | static const struct virtio_config_ops virtio_pci_config_nodev_ops = { |
428 | .get = NULL, | |
429 | .set = NULL, | |
430 | .generation = vp_generation, | |
431 | .get_status = vp_get_status, | |
432 | .set_status = vp_set_status, | |
433 | .reset = vp_reset, | |
434 | .find_vqs = vp_modern_find_vqs, | |
435 | .del_vqs = vp_del_vqs, | |
436 | .get_features = vp_get_features, | |
437 | .finalize_features = vp_finalize_features, | |
438 | .bus_name = vp_bus_name, | |
439 | .set_vq_affinity = vp_set_vq_affinity, | |
440 | }; | |
441 | ||
1fcf0512 MT |
442 | static const struct virtio_config_ops virtio_pci_config_ops = { |
443 | .get = vp_get, | |
444 | .set = vp_set, | |
445 | .generation = vp_generation, | |
446 | .get_status = vp_get_status, | |
447 | .set_status = vp_set_status, | |
448 | .reset = vp_reset, | |
449 | .find_vqs = vp_modern_find_vqs, | |
450 | .del_vqs = vp_del_vqs, | |
451 | .get_features = vp_get_features, | |
452 | .finalize_features = vp_finalize_features, | |
453 | .bus_name = vp_bus_name, | |
454 | .set_vq_affinity = vp_set_vq_affinity, | |
455 | }; | |
456 | ||
457 | /** | |
458 | * virtio_pci_find_capability - walk capabilities to find device info. | |
459 | * @dev: the pci device | |
460 | * @cfg_type: the VIRTIO_PCI_CAP_* value we seek | |
461 | * @ioresource_types: IORESOURCE_MEM and/or IORESOURCE_IO. | |
462 | * | |
463 | * Returns offset of the capability, or 0. | |
464 | */ | |
465 | static inline int virtio_pci_find_capability(struct pci_dev *dev, u8 cfg_type, | |
59a5b0f7 | 466 | u32 ioresource_types, int *bars) |
1fcf0512 MT |
467 | { |
468 | int pos; | |
469 | ||
470 | for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); | |
471 | pos > 0; | |
472 | pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { | |
473 | u8 type, bar; | |
474 | pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, | |
475 | cfg_type), | |
476 | &type); | |
477 | pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, | |
478 | bar), | |
479 | &bar); | |
480 | ||
481 | /* Ignore structures with reserved BAR values */ | |
482 | if (bar > 0x5) | |
483 | continue; | |
484 | ||
485 | if (type == cfg_type) { | |
486 | if (pci_resource_len(dev, bar) && | |
59a5b0f7 GH |
487 | pci_resource_flags(dev, bar) & ioresource_types) { |
488 | *bars |= (1 << bar); | |
1fcf0512 | 489 | return pos; |
59a5b0f7 | 490 | } |
1fcf0512 MT |
491 | } |
492 | } | |
493 | return 0; | |
494 | } | |
495 | ||
89461c4a | 496 | /* This is part of the ABI. Don't screw with it. */ |
1fcf0512 MT |
497 | static inline void check_offsets(void) |
498 | { | |
89461c4a RR |
499 | /* Note: disk space was harmed in compilation of this function. */ |
500 | BUILD_BUG_ON(VIRTIO_PCI_CAP_VNDR != | |
501 | offsetof(struct virtio_pci_cap, cap_vndr)); | |
502 | BUILD_BUG_ON(VIRTIO_PCI_CAP_NEXT != | |
503 | offsetof(struct virtio_pci_cap, cap_next)); | |
504 | BUILD_BUG_ON(VIRTIO_PCI_CAP_LEN != | |
505 | offsetof(struct virtio_pci_cap, cap_len)); | |
506 | BUILD_BUG_ON(VIRTIO_PCI_CAP_CFG_TYPE != | |
507 | offsetof(struct virtio_pci_cap, cfg_type)); | |
508 | BUILD_BUG_ON(VIRTIO_PCI_CAP_BAR != | |
509 | offsetof(struct virtio_pci_cap, bar)); | |
510 | BUILD_BUG_ON(VIRTIO_PCI_CAP_OFFSET != | |
511 | offsetof(struct virtio_pci_cap, offset)); | |
512 | BUILD_BUG_ON(VIRTIO_PCI_CAP_LENGTH != | |
513 | offsetof(struct virtio_pci_cap, length)); | |
514 | BUILD_BUG_ON(VIRTIO_PCI_NOTIFY_CAP_MULT != | |
515 | offsetof(struct virtio_pci_notify_cap, | |
516 | notify_off_multiplier)); | |
517 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_DFSELECT != | |
518 | offsetof(struct virtio_pci_common_cfg, | |
519 | device_feature_select)); | |
520 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_DF != | |
521 | offsetof(struct virtio_pci_common_cfg, device_feature)); | |
522 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_GFSELECT != | |
523 | offsetof(struct virtio_pci_common_cfg, | |
524 | guest_feature_select)); | |
525 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_GF != | |
526 | offsetof(struct virtio_pci_common_cfg, guest_feature)); | |
527 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_MSIX != | |
528 | offsetof(struct virtio_pci_common_cfg, msix_config)); | |
529 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_NUMQ != | |
530 | offsetof(struct virtio_pci_common_cfg, num_queues)); | |
531 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_STATUS != | |
532 | offsetof(struct virtio_pci_common_cfg, device_status)); | |
533 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_CFGGENERATION != | |
534 | offsetof(struct virtio_pci_common_cfg, config_generation)); | |
535 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SELECT != | |
536 | offsetof(struct virtio_pci_common_cfg, queue_select)); | |
537 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SIZE != | |
538 | offsetof(struct virtio_pci_common_cfg, queue_size)); | |
539 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_MSIX != | |
540 | offsetof(struct virtio_pci_common_cfg, queue_msix_vector)); | |
541 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_ENABLE != | |
542 | offsetof(struct virtio_pci_common_cfg, queue_enable)); | |
543 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_NOFF != | |
544 | offsetof(struct virtio_pci_common_cfg, queue_notify_off)); | |
545 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCLO != | |
546 | offsetof(struct virtio_pci_common_cfg, queue_desc_lo)); | |
547 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCHI != | |
548 | offsetof(struct virtio_pci_common_cfg, queue_desc_hi)); | |
549 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILLO != | |
550 | offsetof(struct virtio_pci_common_cfg, queue_avail_lo)); | |
551 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILHI != | |
552 | offsetof(struct virtio_pci_common_cfg, queue_avail_hi)); | |
553 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDLO != | |
554 | offsetof(struct virtio_pci_common_cfg, queue_used_lo)); | |
555 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDHI != | |
556 | offsetof(struct virtio_pci_common_cfg, queue_used_hi)); | |
1fcf0512 MT |
557 | } |
558 | ||
559 | /* the PCI probing function */ | |
560 | int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev) | |
561 | { | |
562 | struct pci_dev *pci_dev = vp_dev->pci_dev; | |
563 | int err, common, isr, notify, device; | |
564 | u32 notify_length; | |
3909213c | 565 | u32 notify_offset; |
1fcf0512 MT |
566 | |
567 | check_offsets(); | |
568 | ||
569 | /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ | |
570 | if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f) | |
571 | return -ENODEV; | |
572 | ||
573 | if (pci_dev->device < 0x1040) { | |
574 | /* Transitional devices: use the PCI subsystem device id as | |
575 | * virtio device id, same as legacy driver always did. | |
576 | */ | |
577 | vp_dev->vdev.id.device = pci_dev->subsystem_device; | |
578 | } else { | |
579 | /* Modern devices: simply use PCI device id, but start from 0x1040. */ | |
580 | vp_dev->vdev.id.device = pci_dev->device - 0x1040; | |
581 | } | |
582 | vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor; | |
583 | ||
1fcf0512 MT |
584 | /* check for a common config: if not, use legacy mode (bar 0). */ |
585 | common = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_COMMON_CFG, | |
59a5b0f7 GH |
586 | IORESOURCE_IO | IORESOURCE_MEM, |
587 | &vp_dev->modern_bars); | |
1fcf0512 MT |
588 | if (!common) { |
589 | dev_info(&pci_dev->dev, | |
590 | "virtio_pci: leaving for legacy driver\n"); | |
591 | return -ENODEV; | |
592 | } | |
593 | ||
594 | /* If common is there, these should be too... */ | |
595 | isr = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_ISR_CFG, | |
59a5b0f7 GH |
596 | IORESOURCE_IO | IORESOURCE_MEM, |
597 | &vp_dev->modern_bars); | |
1fcf0512 | 598 | notify = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_NOTIFY_CFG, |
59a5b0f7 GH |
599 | IORESOURCE_IO | IORESOURCE_MEM, |
600 | &vp_dev->modern_bars); | |
1fcf0512 MT |
601 | if (!isr || !notify) { |
602 | dev_err(&pci_dev->dev, | |
603 | "virtio_pci: missing capabilities %i/%i/%i\n", | |
604 | common, isr, notify); | |
605 | return -EINVAL; | |
606 | } | |
607 | ||
7a5589b2 AL |
608 | err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)); |
609 | if (err) | |
610 | err = dma_set_mask_and_coherent(&pci_dev->dev, | |
611 | DMA_BIT_MASK(32)); | |
612 | if (err) | |
613 | dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); | |
614 | ||
1fcf0512 MT |
615 | /* Device capability is only mandatory for devices that have |
616 | * device-specific configuration. | |
617 | */ | |
618 | device = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG, | |
59a5b0f7 GH |
619 | IORESOURCE_IO | IORESOURCE_MEM, |
620 | &vp_dev->modern_bars); | |
621 | ||
622 | err = pci_request_selected_regions(pci_dev, vp_dev->modern_bars, | |
623 | "virtio-pci-modern"); | |
624 | if (err) | |
625 | return err; | |
1fcf0512 MT |
626 | |
627 | err = -EINVAL; | |
628 | vp_dev->common = map_capability(pci_dev, common, | |
629 | sizeof(struct virtio_pci_common_cfg), 4, | |
630 | 0, sizeof(struct virtio_pci_common_cfg), | |
631 | NULL); | |
632 | if (!vp_dev->common) | |
633 | goto err_map_common; | |
634 | vp_dev->isr = map_capability(pci_dev, isr, sizeof(u8), 1, | |
635 | 0, 1, | |
636 | NULL); | |
637 | if (!vp_dev->isr) | |
638 | goto err_map_isr; | |
639 | ||
640 | /* Read notify_off_multiplier from config space. */ | |
641 | pci_read_config_dword(pci_dev, | |
642 | notify + offsetof(struct virtio_pci_notify_cap, | |
643 | notify_off_multiplier), | |
644 | &vp_dev->notify_offset_multiplier); | |
3909213c | 645 | /* Read notify length and offset from config space. */ |
1fcf0512 MT |
646 | pci_read_config_dword(pci_dev, |
647 | notify + offsetof(struct virtio_pci_notify_cap, | |
648 | cap.length), | |
649 | ¬ify_length); | |
650 | ||
3909213c MT |
651 | pci_read_config_dword(pci_dev, |
652 | notify + offsetof(struct virtio_pci_notify_cap, | |
4e94ebdd | 653 | cap.offset), |
3909213c MT |
654 | ¬ify_offset); |
655 | ||
656 | /* We don't know how many VQs we'll map, ahead of the time. | |
657 | * If notify length is small, map it all now. | |
658 | * Otherwise, map each VQ individually later. | |
659 | */ | |
660 | if ((u64)notify_length + (notify_offset % PAGE_SIZE) <= PAGE_SIZE) { | |
661 | vp_dev->notify_base = map_capability(pci_dev, notify, 2, 2, | |
662 | 0, notify_length, | |
663 | &vp_dev->notify_len); | |
664 | if (!vp_dev->notify_base) | |
665 | goto err_map_notify; | |
666 | } else { | |
667 | vp_dev->notify_map_cap = notify; | |
668 | } | |
1fcf0512 MT |
669 | |
670 | /* Again, we don't know how much we should map, but PAGE_SIZE | |
671 | * is more than enough for all existing devices. | |
672 | */ | |
673 | if (device) { | |
674 | vp_dev->device = map_capability(pci_dev, device, 0, 4, | |
675 | 0, PAGE_SIZE, | |
676 | &vp_dev->device_len); | |
677 | if (!vp_dev->device) | |
678 | goto err_map_device; | |
1fcf0512 | 679 | |
d3f5f065 MT |
680 | vp_dev->vdev.config = &virtio_pci_config_ops; |
681 | } else { | |
682 | vp_dev->vdev.config = &virtio_pci_config_nodev_ops; | |
683 | } | |
1fcf0512 MT |
684 | |
685 | vp_dev->config_vector = vp_config_vector; | |
686 | vp_dev->setup_vq = setup_vq; | |
687 | vp_dev->del_vq = del_vq; | |
688 | ||
689 | return 0; | |
690 | ||
691 | err_map_device: | |
3909213c MT |
692 | if (vp_dev->notify_base) |
693 | pci_iounmap(pci_dev, vp_dev->notify_base); | |
694 | err_map_notify: | |
1fcf0512 MT |
695 | pci_iounmap(pci_dev, vp_dev->isr); |
696 | err_map_isr: | |
697 | pci_iounmap(pci_dev, vp_dev->common); | |
698 | err_map_common: | |
699 | return err; | |
700 | } | |
701 | ||
702 | void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev) | |
703 | { | |
704 | struct pci_dev *pci_dev = vp_dev->pci_dev; | |
705 | ||
706 | if (vp_dev->device) | |
707 | pci_iounmap(pci_dev, vp_dev->device); | |
3909213c MT |
708 | if (vp_dev->notify_base) |
709 | pci_iounmap(pci_dev, vp_dev->notify_base); | |
1fcf0512 MT |
710 | pci_iounmap(pci_dev, vp_dev->isr); |
711 | pci_iounmap(pci_dev, vp_dev->common); | |
59a5b0f7 | 712 | pci_release_selected_regions(pci_dev, vp_dev->modern_bars); |
1fcf0512 | 713 | } |