2 * Back-end of the driver for virtual network devices. This portion of the
3 * driver exports a 'unified' network-device interface that can be accessed
4 * by any operating system that implements a compatible front end. A
5 * reference front-end implementation can be found in:
6 * drivers/net/xen-netfront.c
8 * Copyright (c) 2002-2005, K A Fraser
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #include <linux/kthread.h>
38 #include <linux/if_vlan.h>
39 #include <linux/udp.h>
40 #include <linux/highmem.h>
45 #include <xen/events.h>
46 #include <xen/interface/memory.h>
48 #include <asm/xen/hypercall.h>
49 #include <asm/xen/page.h>
51 /* Provide an option to disable split event channels at load time as
52 * event channels are limited resource. Split event channels are
55 bool separate_tx_rx_irq
= 1;
56 module_param(separate_tx_rx_irq
, bool, 0644);
58 /* The time that packets can stay on the guest Rx internal queue
59 * before they are dropped.
61 unsigned int rx_drain_timeout_msecs
= 10000;
62 module_param(rx_drain_timeout_msecs
, uint
, 0444);
64 /* The length of time before the frontend is considered unresponsive
65 * because it isn't providing Rx slots.
67 unsigned int rx_stall_timeout_msecs
= 60000;
68 module_param(rx_stall_timeout_msecs
, uint
, 0444);
70 unsigned int xenvif_max_queues
;
71 module_param_named(max_queues
, xenvif_max_queues
, uint
, 0644);
72 MODULE_PARM_DESC(max_queues
,
73 "Maximum number of queues per virtual interface");
76 * This is the maximum slots a skb can have. If a guest sends a skb
77 * which exceeds this limit it is considered malicious.
79 #define FATAL_SKB_SLOTS_DEFAULT 20
80 static unsigned int fatal_skb_slots
= FATAL_SKB_SLOTS_DEFAULT
;
81 module_param(fatal_skb_slots
, uint
, 0444);
83 /* The amount to copy out of the first guest Tx slot into the skb's
84 * linear area. If the first slot has more data, it will be mapped
85 * and put into the first frag.
87 * This is sized to avoid pulling headers from the frags for most
90 #define XEN_NETBACK_TX_COPY_LEN 128
93 static void xenvif_idx_release(struct xenvif_queue
*queue
, u16 pending_idx
,
96 static void make_tx_response(struct xenvif_queue
*queue
,
97 struct xen_netif_tx_request
*txp
,
99 static void push_tx_responses(struct xenvif_queue
*queue
);
101 static inline int tx_work_todo(struct xenvif_queue
*queue
);
103 static struct xen_netif_rx_response
*make_rx_response(struct xenvif_queue
*queue
,
110 static inline unsigned long idx_to_pfn(struct xenvif_queue
*queue
,
113 return page_to_pfn(queue
->mmap_pages
[idx
]);
116 static inline unsigned long idx_to_kaddr(struct xenvif_queue
*queue
,
119 return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue
, idx
));
122 #define callback_param(vif, pending_idx) \
123 (vif->pending_tx_info[pending_idx].callback_struct)
125 /* Find the containing VIF's structure from a pointer in pending_tx_info array
127 static inline struct xenvif_queue
*ubuf_to_queue(const struct ubuf_info
*ubuf
)
129 u16 pending_idx
= ubuf
->desc
;
130 struct pending_tx_info
*temp
=
131 container_of(ubuf
, struct pending_tx_info
, callback_struct
);
132 return container_of(temp
- pending_idx
,
137 static u16
frag_get_pending_idx(skb_frag_t
*frag
)
139 return (u16
)frag
->page_offset
;
142 static void frag_set_pending_idx(skb_frag_t
*frag
, u16 pending_idx
)
144 frag
->page_offset
= pending_idx
;
147 static inline pending_ring_idx_t
pending_index(unsigned i
)
149 return i
& (MAX_PENDING_REQS
-1);
152 bool xenvif_rx_ring_slots_available(struct xenvif_queue
*queue
, int needed
)
157 prod
= queue
->rx
.sring
->req_prod
;
158 cons
= queue
->rx
.req_cons
;
160 if (prod
- cons
>= needed
)
163 queue
->rx
.sring
->req_event
= prod
+ 1;
165 /* Make sure event is visible before we check prod
169 } while (queue
->rx
.sring
->req_prod
!= prod
);
174 void xenvif_rx_queue_tail(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
178 spin_lock_irqsave(&queue
->rx_queue
.lock
, flags
);
180 __skb_queue_tail(&queue
->rx_queue
, skb
);
182 queue
->rx_queue_len
+= skb
->len
;
183 if (queue
->rx_queue_len
> queue
->rx_queue_max
)
184 netif_tx_stop_queue(netdev_get_tx_queue(queue
->vif
->dev
, queue
->id
));
186 spin_unlock_irqrestore(&queue
->rx_queue
.lock
, flags
);
189 static struct sk_buff
*xenvif_rx_dequeue(struct xenvif_queue
*queue
)
193 spin_lock_irq(&queue
->rx_queue
.lock
);
195 skb
= __skb_dequeue(&queue
->rx_queue
);
197 queue
->rx_queue_len
-= skb
->len
;
199 spin_unlock_irq(&queue
->rx_queue
.lock
);
204 static void xenvif_rx_queue_maybe_wake(struct xenvif_queue
*queue
)
206 spin_lock_irq(&queue
->rx_queue
.lock
);
208 if (queue
->rx_queue_len
< queue
->rx_queue_max
)
209 netif_tx_wake_queue(netdev_get_tx_queue(queue
->vif
->dev
, queue
->id
));
211 spin_unlock_irq(&queue
->rx_queue
.lock
);
215 static void xenvif_rx_queue_purge(struct xenvif_queue
*queue
)
218 while ((skb
= xenvif_rx_dequeue(queue
)) != NULL
)
222 static void xenvif_rx_queue_drop_expired(struct xenvif_queue
*queue
)
227 skb
= skb_peek(&queue
->rx_queue
);
230 if (time_before(jiffies
, XENVIF_RX_CB(skb
)->expires
))
232 xenvif_rx_dequeue(queue
);
237 struct netrx_pending_operations
{
238 unsigned copy_prod
, copy_cons
;
239 unsigned meta_prod
, meta_cons
;
240 struct gnttab_copy
*copy
;
241 struct xenvif_rx_meta
*meta
;
243 grant_ref_t copy_gref
;
246 static struct xenvif_rx_meta
*get_next_rx_buffer(struct xenvif_queue
*queue
,
247 struct netrx_pending_operations
*npo
)
249 struct xenvif_rx_meta
*meta
;
250 struct xen_netif_rx_request
*req
;
252 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
254 meta
= npo
->meta
+ npo
->meta_prod
++;
255 meta
->gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
261 npo
->copy_gref
= req
->gref
;
267 * Set up the grant operations for this fragment. If it's a flipping
268 * interface, we also set up the unmap request from here.
270 static void xenvif_gop_frag_copy(struct xenvif_queue
*queue
, struct sk_buff
*skb
,
271 struct netrx_pending_operations
*npo
,
272 struct page
*page
, unsigned long size
,
273 unsigned long offset
, int *head
)
275 struct gnttab_copy
*copy_gop
;
276 struct xenvif_rx_meta
*meta
;
278 int gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
280 /* Data must not cross a page boundary. */
281 BUG_ON(size
+ offset
> PAGE_SIZE
<<compound_order(page
));
283 meta
= npo
->meta
+ npo
->meta_prod
- 1;
285 /* Skip unused frames from start of page */
286 page
+= offset
>> PAGE_SHIFT
;
287 offset
&= ~PAGE_MASK
;
290 struct xen_page_foreign
*foreign
;
292 BUG_ON(offset
>= PAGE_SIZE
);
293 BUG_ON(npo
->copy_off
> MAX_BUFFER_OFFSET
);
295 if (npo
->copy_off
== MAX_BUFFER_OFFSET
)
296 meta
= get_next_rx_buffer(queue
, npo
);
298 bytes
= PAGE_SIZE
- offset
;
302 if (npo
->copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
303 bytes
= MAX_BUFFER_OFFSET
- npo
->copy_off
;
305 copy_gop
= npo
->copy
+ npo
->copy_prod
++;
306 copy_gop
->flags
= GNTCOPY_dest_gref
;
307 copy_gop
->len
= bytes
;
309 foreign
= xen_page_foreign(page
);
311 copy_gop
->source
.domid
= foreign
->domid
;
312 copy_gop
->source
.u
.ref
= foreign
->gref
;
313 copy_gop
->flags
|= GNTCOPY_source_gref
;
315 copy_gop
->source
.domid
= DOMID_SELF
;
316 copy_gop
->source
.u
.gmfn
=
317 virt_to_mfn(page_address(page
));
319 copy_gop
->source
.offset
= offset
;
321 copy_gop
->dest
.domid
= queue
->vif
->domid
;
322 copy_gop
->dest
.offset
= npo
->copy_off
;
323 copy_gop
->dest
.u
.ref
= npo
->copy_gref
;
325 npo
->copy_off
+= bytes
;
332 if (offset
== PAGE_SIZE
&& size
) {
333 BUG_ON(!PageCompound(page
));
338 /* Leave a gap for the GSO descriptor. */
339 if (skb_is_gso(skb
)) {
340 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
341 gso_type
= XEN_NETIF_GSO_TYPE_TCPV4
;
342 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
343 gso_type
= XEN_NETIF_GSO_TYPE_TCPV6
;
346 if (*head
&& ((1 << gso_type
) & queue
->vif
->gso_mask
))
347 queue
->rx
.req_cons
++;
349 *head
= 0; /* There must be something in this buffer now. */
355 * Prepare an SKB to be transmitted to the frontend.
357 * This function is responsible for allocating grant operations, meta
360 * It returns the number of meta structures consumed. The number of
361 * ring slots used is always equal to the number of meta slots used
362 * plus the number of GSO descriptors used. Currently, we use either
363 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
364 * frontend-side LRO).
366 static int xenvif_gop_skb(struct sk_buff
*skb
,
367 struct netrx_pending_operations
*npo
,
368 struct xenvif_queue
*queue
)
370 struct xenvif
*vif
= netdev_priv(skb
->dev
);
371 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
373 struct xen_netif_rx_request
*req
;
374 struct xenvif_rx_meta
*meta
;
380 old_meta_prod
= npo
->meta_prod
;
382 gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
383 if (skb_is_gso(skb
)) {
384 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
385 gso_type
= XEN_NETIF_GSO_TYPE_TCPV4
;
386 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
387 gso_type
= XEN_NETIF_GSO_TYPE_TCPV6
;
390 /* Set up a GSO prefix descriptor, if necessary */
391 if ((1 << gso_type
) & vif
->gso_prefix_mask
) {
392 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
393 meta
= npo
->meta
+ npo
->meta_prod
++;
394 meta
->gso_type
= gso_type
;
395 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
400 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
401 meta
= npo
->meta
+ npo
->meta_prod
++;
403 if ((1 << gso_type
) & vif
->gso_mask
) {
404 meta
->gso_type
= gso_type
;
405 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
407 meta
->gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
414 npo
->copy_gref
= req
->gref
;
417 while (data
< skb_tail_pointer(skb
)) {
418 unsigned int offset
= offset_in_page(data
);
419 unsigned int len
= PAGE_SIZE
- offset
;
421 if (data
+ len
> skb_tail_pointer(skb
))
422 len
= skb_tail_pointer(skb
) - data
;
424 xenvif_gop_frag_copy(queue
, skb
, npo
,
425 virt_to_page(data
), len
, offset
, &head
);
429 for (i
= 0; i
< nr_frags
; i
++) {
430 xenvif_gop_frag_copy(queue
, skb
, npo
,
431 skb_frag_page(&skb_shinfo(skb
)->frags
[i
]),
432 skb_frag_size(&skb_shinfo(skb
)->frags
[i
]),
433 skb_shinfo(skb
)->frags
[i
].page_offset
,
437 return npo
->meta_prod
- old_meta_prod
;
441 * This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was
442 * used to set up the operations on the top of
443 * netrx_pending_operations, which have since been done. Check that
444 * they didn't give any errors and advance over them.
446 static int xenvif_check_gop(struct xenvif
*vif
, int nr_meta_slots
,
447 struct netrx_pending_operations
*npo
)
449 struct gnttab_copy
*copy_op
;
450 int status
= XEN_NETIF_RSP_OKAY
;
453 for (i
= 0; i
< nr_meta_slots
; i
++) {
454 copy_op
= npo
->copy
+ npo
->copy_cons
++;
455 if (copy_op
->status
!= GNTST_okay
) {
457 "Bad status %d from copy to DOM%d.\n",
458 copy_op
->status
, vif
->domid
);
459 status
= XEN_NETIF_RSP_ERROR
;
466 static void xenvif_add_frag_responses(struct xenvif_queue
*queue
, int status
,
467 struct xenvif_rx_meta
*meta
,
471 unsigned long offset
;
473 /* No fragments used */
474 if (nr_meta_slots
<= 1)
479 for (i
= 0; i
< nr_meta_slots
; i
++) {
481 if (i
== nr_meta_slots
- 1)
484 flags
= XEN_NETRXF_more_data
;
487 make_rx_response(queue
, meta
[i
].id
, status
, offset
,
488 meta
[i
].size
, flags
);
492 void xenvif_kick_thread(struct xenvif_queue
*queue
)
497 static void xenvif_rx_action(struct xenvif_queue
*queue
)
501 struct xen_netif_rx_response
*resp
;
502 struct sk_buff_head rxq
;
506 unsigned long offset
;
507 bool need_to_notify
= false;
509 struct netrx_pending_operations npo
= {
510 .copy
= queue
->grant_copy_op
,
514 skb_queue_head_init(&rxq
);
516 while (xenvif_rx_ring_slots_available(queue
, XEN_NETBK_RX_SLOTS_MAX
)
517 && (skb
= xenvif_rx_dequeue(queue
)) != NULL
) {
518 RING_IDX old_req_cons
;
519 RING_IDX ring_slots_used
;
521 queue
->last_rx_time
= jiffies
;
523 old_req_cons
= queue
->rx
.req_cons
;
524 XENVIF_RX_CB(skb
)->meta_slots_used
= xenvif_gop_skb(skb
, &npo
, queue
);
525 ring_slots_used
= queue
->rx
.req_cons
- old_req_cons
;
527 __skb_queue_tail(&rxq
, skb
);
530 BUG_ON(npo
.meta_prod
> ARRAY_SIZE(queue
->meta
));
535 BUG_ON(npo
.copy_prod
> MAX_GRANT_COPY_OPS
);
536 gnttab_batch_copy(queue
->grant_copy_op
, npo
.copy_prod
);
538 while ((skb
= __skb_dequeue(&rxq
)) != NULL
) {
540 if ((1 << queue
->meta
[npo
.meta_cons
].gso_type
) &
541 queue
->vif
->gso_prefix_mask
) {
542 resp
= RING_GET_RESPONSE(&queue
->rx
,
543 queue
->rx
.rsp_prod_pvt
++);
545 resp
->flags
= XEN_NETRXF_gso_prefix
| XEN_NETRXF_more_data
;
547 resp
->offset
= queue
->meta
[npo
.meta_cons
].gso_size
;
548 resp
->id
= queue
->meta
[npo
.meta_cons
].id
;
549 resp
->status
= XENVIF_RX_CB(skb
)->meta_slots_used
;
552 XENVIF_RX_CB(skb
)->meta_slots_used
--;
556 queue
->stats
.tx_bytes
+= skb
->len
;
557 queue
->stats
.tx_packets
++;
559 status
= xenvif_check_gop(queue
->vif
,
560 XENVIF_RX_CB(skb
)->meta_slots_used
,
563 if (XENVIF_RX_CB(skb
)->meta_slots_used
== 1)
566 flags
= XEN_NETRXF_more_data
;
568 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) /* local packet? */
569 flags
|= XEN_NETRXF_csum_blank
| XEN_NETRXF_data_validated
;
570 else if (skb
->ip_summed
== CHECKSUM_UNNECESSARY
)
571 /* remote but checksummed. */
572 flags
|= XEN_NETRXF_data_validated
;
575 resp
= make_rx_response(queue
, queue
->meta
[npo
.meta_cons
].id
,
577 queue
->meta
[npo
.meta_cons
].size
,
580 if ((1 << queue
->meta
[npo
.meta_cons
].gso_type
) &
581 queue
->vif
->gso_mask
) {
582 struct xen_netif_extra_info
*gso
=
583 (struct xen_netif_extra_info
*)
584 RING_GET_RESPONSE(&queue
->rx
,
585 queue
->rx
.rsp_prod_pvt
++);
587 resp
->flags
|= XEN_NETRXF_extra_info
;
589 gso
->u
.gso
.type
= queue
->meta
[npo
.meta_cons
].gso_type
;
590 gso
->u
.gso
.size
= queue
->meta
[npo
.meta_cons
].gso_size
;
592 gso
->u
.gso
.features
= 0;
594 gso
->type
= XEN_NETIF_EXTRA_TYPE_GSO
;
598 xenvif_add_frag_responses(queue
, status
,
599 queue
->meta
+ npo
.meta_cons
+ 1,
600 XENVIF_RX_CB(skb
)->meta_slots_used
);
602 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue
->rx
, ret
);
604 need_to_notify
|= !!ret
;
606 npo
.meta_cons
+= XENVIF_RX_CB(skb
)->meta_slots_used
;
612 notify_remote_via_irq(queue
->rx_irq
);
615 void xenvif_napi_schedule_or_enable_events(struct xenvif_queue
*queue
)
619 RING_FINAL_CHECK_FOR_REQUESTS(&queue
->tx
, more_to_do
);
622 napi_schedule(&queue
->napi
);
625 static void tx_add_credit(struct xenvif_queue
*queue
)
627 unsigned long max_burst
, max_credit
;
630 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
631 * Otherwise the interface can seize up due to insufficient credit.
633 max_burst
= RING_GET_REQUEST(&queue
->tx
, queue
->tx
.req_cons
)->size
;
634 max_burst
= min(max_burst
, 131072UL);
635 max_burst
= max(max_burst
, queue
->credit_bytes
);
637 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
638 max_credit
= queue
->remaining_credit
+ queue
->credit_bytes
;
639 if (max_credit
< queue
->remaining_credit
)
640 max_credit
= ULONG_MAX
; /* wrapped: clamp to ULONG_MAX */
642 queue
->remaining_credit
= min(max_credit
, max_burst
);
645 void xenvif_tx_credit_callback(unsigned long data
)
647 struct xenvif_queue
*queue
= (struct xenvif_queue
*)data
;
648 tx_add_credit(queue
);
649 xenvif_napi_schedule_or_enable_events(queue
);
652 static void xenvif_tx_err(struct xenvif_queue
*queue
,
653 struct xen_netif_tx_request
*txp
, RING_IDX end
)
655 RING_IDX cons
= queue
->tx
.req_cons
;
659 spin_lock_irqsave(&queue
->response_lock
, flags
);
660 make_tx_response(queue
, txp
, XEN_NETIF_RSP_ERROR
);
661 push_tx_responses(queue
);
662 spin_unlock_irqrestore(&queue
->response_lock
, flags
);
665 txp
= RING_GET_REQUEST(&queue
->tx
, cons
++);
667 queue
->tx
.req_cons
= cons
;
670 static void xenvif_fatal_tx_err(struct xenvif
*vif
)
672 netdev_err(vif
->dev
, "fatal error; disabling device\n");
673 vif
->disabled
= true;
674 /* Disable the vif from queue 0's kthread */
676 xenvif_kick_thread(&vif
->queues
[0]);
679 static int xenvif_count_requests(struct xenvif_queue
*queue
,
680 struct xen_netif_tx_request
*first
,
681 struct xen_netif_tx_request
*txp
,
684 RING_IDX cons
= queue
->tx
.req_cons
;
689 if (!(first
->flags
& XEN_NETTXF_more_data
))
693 struct xen_netif_tx_request dropped_tx
= { 0 };
695 if (slots
>= work_to_do
) {
696 netdev_err(queue
->vif
->dev
,
697 "Asked for %d slots but exceeds this limit\n",
699 xenvif_fatal_tx_err(queue
->vif
);
703 /* This guest is really using too many slots and
704 * considered malicious.
706 if (unlikely(slots
>= fatal_skb_slots
)) {
707 netdev_err(queue
->vif
->dev
,
708 "Malicious frontend using %d slots, threshold %u\n",
709 slots
, fatal_skb_slots
);
710 xenvif_fatal_tx_err(queue
->vif
);
714 /* Xen network protocol had implicit dependency on
715 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
716 * the historical MAX_SKB_FRAGS value 18 to honor the
717 * same behavior as before. Any packet using more than
718 * 18 slots but less than fatal_skb_slots slots is
721 if (!drop_err
&& slots
>= XEN_NETBK_LEGACY_SLOTS_MAX
) {
723 netdev_dbg(queue
->vif
->dev
,
724 "Too many slots (%d) exceeding limit (%d), dropping packet\n",
725 slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
732 memcpy(txp
, RING_GET_REQUEST(&queue
->tx
, cons
+ slots
),
735 /* If the guest submitted a frame >= 64 KiB then
736 * first->size overflowed and following slots will
737 * appear to be larger than the frame.
739 * This cannot be fatal error as there are buggy
740 * frontends that do this.
742 * Consume all slots and drop the packet.
744 if (!drop_err
&& txp
->size
> first
->size
) {
746 netdev_dbg(queue
->vif
->dev
,
747 "Invalid tx request, slot size %u > remaining size %u\n",
748 txp
->size
, first
->size
);
752 first
->size
-= txp
->size
;
755 if (unlikely((txp
->offset
+ txp
->size
) > PAGE_SIZE
)) {
756 netdev_err(queue
->vif
->dev
, "Cross page boundary, txp->offset: %x, size: %u\n",
757 txp
->offset
, txp
->size
);
758 xenvif_fatal_tx_err(queue
->vif
);
762 more_data
= txp
->flags
& XEN_NETTXF_more_data
;
770 xenvif_tx_err(queue
, first
, cons
+ slots
);
778 struct xenvif_tx_cb
{
782 #define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb)
784 static inline void xenvif_tx_create_map_op(struct xenvif_queue
*queue
,
786 struct xen_netif_tx_request
*txp
,
787 struct gnttab_map_grant_ref
*mop
)
789 queue
->pages_to_map
[mop
-queue
->tx_map_ops
] = queue
->mmap_pages
[pending_idx
];
790 gnttab_set_map_op(mop
, idx_to_kaddr(queue
, pending_idx
),
791 GNTMAP_host_map
| GNTMAP_readonly
,
792 txp
->gref
, queue
->vif
->domid
);
794 memcpy(&queue
->pending_tx_info
[pending_idx
].req
, txp
,
798 static inline struct sk_buff
*xenvif_alloc_skb(unsigned int size
)
800 struct sk_buff
*skb
=
801 alloc_skb(size
+ NET_SKB_PAD
+ NET_IP_ALIGN
,
802 GFP_ATOMIC
| __GFP_NOWARN
);
803 if (unlikely(skb
== NULL
))
806 /* Packets passed to netif_rx() must have some headroom. */
807 skb_reserve(skb
, NET_SKB_PAD
+ NET_IP_ALIGN
);
809 /* Initialize it here to avoid later surprises */
810 skb_shinfo(skb
)->destructor_arg
= NULL
;
815 static struct gnttab_map_grant_ref
*xenvif_get_requests(struct xenvif_queue
*queue
,
817 struct xen_netif_tx_request
*txp
,
818 struct gnttab_map_grant_ref
*gop
)
820 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
821 skb_frag_t
*frags
= shinfo
->frags
;
822 u16 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
824 pending_ring_idx_t index
;
825 unsigned int nr_slots
, frag_overflow
= 0;
827 /* At this point shinfo->nr_frags is in fact the number of
828 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
830 if (shinfo
->nr_frags
> MAX_SKB_FRAGS
) {
831 frag_overflow
= shinfo
->nr_frags
- MAX_SKB_FRAGS
;
832 BUG_ON(frag_overflow
> MAX_SKB_FRAGS
);
833 shinfo
->nr_frags
= MAX_SKB_FRAGS
;
835 nr_slots
= shinfo
->nr_frags
;
837 /* Skip first skb fragment if it is on same page as header fragment. */
838 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
840 for (shinfo
->nr_frags
= start
; shinfo
->nr_frags
< nr_slots
;
841 shinfo
->nr_frags
++, txp
++, gop
++) {
842 index
= pending_index(queue
->pending_cons
++);
843 pending_idx
= queue
->pending_ring
[index
];
844 xenvif_tx_create_map_op(queue
, pending_idx
, txp
, gop
);
845 frag_set_pending_idx(&frags
[shinfo
->nr_frags
], pending_idx
);
849 struct sk_buff
*nskb
= xenvif_alloc_skb(0);
850 if (unlikely(nskb
== NULL
)) {
852 netdev_err(queue
->vif
->dev
,
853 "Can't allocate the frag_list skb.\n");
857 shinfo
= skb_shinfo(nskb
);
858 frags
= shinfo
->frags
;
860 for (shinfo
->nr_frags
= 0; shinfo
->nr_frags
< frag_overflow
;
861 shinfo
->nr_frags
++, txp
++, gop
++) {
862 index
= pending_index(queue
->pending_cons
++);
863 pending_idx
= queue
->pending_ring
[index
];
864 xenvif_tx_create_map_op(queue
, pending_idx
, txp
, gop
);
865 frag_set_pending_idx(&frags
[shinfo
->nr_frags
],
869 skb_shinfo(skb
)->frag_list
= nskb
;
875 static inline void xenvif_grant_handle_set(struct xenvif_queue
*queue
,
877 grant_handle_t handle
)
879 if (unlikely(queue
->grant_tx_handle
[pending_idx
] !=
880 NETBACK_INVALID_HANDLE
)) {
881 netdev_err(queue
->vif
->dev
,
882 "Trying to overwrite active handle! pending_idx: %x\n",
886 queue
->grant_tx_handle
[pending_idx
] = handle
;
889 static inline void xenvif_grant_handle_reset(struct xenvif_queue
*queue
,
892 if (unlikely(queue
->grant_tx_handle
[pending_idx
] ==
893 NETBACK_INVALID_HANDLE
)) {
894 netdev_err(queue
->vif
->dev
,
895 "Trying to unmap invalid handle! pending_idx: %x\n",
899 queue
->grant_tx_handle
[pending_idx
] = NETBACK_INVALID_HANDLE
;
902 static int xenvif_tx_check_gop(struct xenvif_queue
*queue
,
904 struct gnttab_map_grant_ref
**gopp_map
,
905 struct gnttab_copy
**gopp_copy
)
907 struct gnttab_map_grant_ref
*gop_map
= *gopp_map
;
908 u16 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
909 /* This always points to the shinfo of the skb being checked, which
910 * could be either the first or the one on the frag_list
912 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
913 /* If this is non-NULL, we are currently checking the frag_list skb, and
914 * this points to the shinfo of the first one
916 struct skb_shared_info
*first_shinfo
= NULL
;
917 int nr_frags
= shinfo
->nr_frags
;
918 const bool sharedslot
= nr_frags
&&
919 frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
;
922 /* Check status of header. */
923 err
= (*gopp_copy
)->status
;
926 netdev_dbg(queue
->vif
->dev
,
927 "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n",
928 (*gopp_copy
)->status
,
930 (*gopp_copy
)->source
.u
.ref
);
931 /* The first frag might still have this slot mapped */
933 xenvif_idx_release(queue
, pending_idx
,
934 XEN_NETIF_RSP_ERROR
);
939 for (i
= 0; i
< nr_frags
; i
++, gop_map
++) {
942 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[i
]);
944 /* Check error status: if okay then remember grant handle. */
945 newerr
= gop_map
->status
;
947 if (likely(!newerr
)) {
948 xenvif_grant_handle_set(queue
,
951 /* Had a previous error? Invalidate this fragment. */
953 xenvif_idx_unmap(queue
, pending_idx
);
954 /* If the mapping of the first frag was OK, but
955 * the header's copy failed, and they are
956 * sharing a slot, send an error
958 if (i
== 0 && sharedslot
)
959 xenvif_idx_release(queue
, pending_idx
,
960 XEN_NETIF_RSP_ERROR
);
962 xenvif_idx_release(queue
, pending_idx
,
968 /* Error on this fragment: respond to client with an error. */
970 netdev_dbg(queue
->vif
->dev
,
971 "Grant map of %d. frag failed! status: %d pending_idx: %u ref: %u\n",
977 xenvif_idx_release(queue
, pending_idx
, XEN_NETIF_RSP_ERROR
);
979 /* Not the first error? Preceding frags already invalidated. */
983 /* First error: if the header haven't shared a slot with the
984 * first frag, release it as well.
987 xenvif_idx_release(queue
,
988 XENVIF_TX_CB(skb
)->pending_idx
,
991 /* Invalidate preceding fragments of this skb. */
992 for (j
= 0; j
< i
; j
++) {
993 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[j
]);
994 xenvif_idx_unmap(queue
, pending_idx
);
995 xenvif_idx_release(queue
, pending_idx
,
999 /* And if we found the error while checking the frag_list, unmap
1000 * the first skb's frags
1003 for (j
= 0; j
< first_shinfo
->nr_frags
; j
++) {
1004 pending_idx
= frag_get_pending_idx(&first_shinfo
->frags
[j
]);
1005 xenvif_idx_unmap(queue
, pending_idx
);
1006 xenvif_idx_release(queue
, pending_idx
,
1007 XEN_NETIF_RSP_OKAY
);
1011 /* Remember the error: invalidate all subsequent fragments. */
1015 if (skb_has_frag_list(skb
) && !first_shinfo
) {
1016 first_shinfo
= skb_shinfo(skb
);
1017 shinfo
= skb_shinfo(skb_shinfo(skb
)->frag_list
);
1018 nr_frags
= shinfo
->nr_frags
;
1023 *gopp_map
= gop_map
;
1027 static void xenvif_fill_frags(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1029 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1030 int nr_frags
= shinfo
->nr_frags
;
1032 u16 prev_pending_idx
= INVALID_PENDING_IDX
;
1034 for (i
= 0; i
< nr_frags
; i
++) {
1035 skb_frag_t
*frag
= shinfo
->frags
+ i
;
1036 struct xen_netif_tx_request
*txp
;
1040 pending_idx
= frag_get_pending_idx(frag
);
1042 /* If this is not the first frag, chain it to the previous*/
1043 if (prev_pending_idx
== INVALID_PENDING_IDX
)
1044 skb_shinfo(skb
)->destructor_arg
=
1045 &callback_param(queue
, pending_idx
);
1047 callback_param(queue
, prev_pending_idx
).ctx
=
1048 &callback_param(queue
, pending_idx
);
1050 callback_param(queue
, pending_idx
).ctx
= NULL
;
1051 prev_pending_idx
= pending_idx
;
1053 txp
= &queue
->pending_tx_info
[pending_idx
].req
;
1054 page
= virt_to_page(idx_to_kaddr(queue
, pending_idx
));
1055 __skb_fill_page_desc(skb
, i
, page
, txp
->offset
, txp
->size
);
1056 skb
->len
+= txp
->size
;
1057 skb
->data_len
+= txp
->size
;
1058 skb
->truesize
+= txp
->size
;
1060 /* Take an extra reference to offset network stack's put_page */
1061 get_page(queue
->mmap_pages
[pending_idx
]);
1065 static int xenvif_get_extras(struct xenvif_queue
*queue
,
1066 struct xen_netif_extra_info
*extras
,
1069 struct xen_netif_extra_info extra
;
1070 RING_IDX cons
= queue
->tx
.req_cons
;
1073 if (unlikely(work_to_do
-- <= 0)) {
1074 netdev_err(queue
->vif
->dev
, "Missing extra info\n");
1075 xenvif_fatal_tx_err(queue
->vif
);
1079 memcpy(&extra
, RING_GET_REQUEST(&queue
->tx
, cons
),
1081 if (unlikely(!extra
.type
||
1082 extra
.type
>= XEN_NETIF_EXTRA_TYPE_MAX
)) {
1083 queue
->tx
.req_cons
= ++cons
;
1084 netdev_err(queue
->vif
->dev
,
1085 "Invalid extra type: %d\n", extra
.type
);
1086 xenvif_fatal_tx_err(queue
->vif
);
1090 memcpy(&extras
[extra
.type
- 1], &extra
, sizeof(extra
));
1091 queue
->tx
.req_cons
= ++cons
;
1092 } while (extra
.flags
& XEN_NETIF_EXTRA_FLAG_MORE
);
1097 static int xenvif_set_skb_gso(struct xenvif
*vif
,
1098 struct sk_buff
*skb
,
1099 struct xen_netif_extra_info
*gso
)
1101 if (!gso
->u
.gso
.size
) {
1102 netdev_err(vif
->dev
, "GSO size must not be zero.\n");
1103 xenvif_fatal_tx_err(vif
);
1107 switch (gso
->u
.gso
.type
) {
1108 case XEN_NETIF_GSO_TYPE_TCPV4
:
1109 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
1111 case XEN_NETIF_GSO_TYPE_TCPV6
:
1112 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
1115 netdev_err(vif
->dev
, "Bad GSO type %d.\n", gso
->u
.gso
.type
);
1116 xenvif_fatal_tx_err(vif
);
1120 skb_shinfo(skb
)->gso_size
= gso
->u
.gso
.size
;
1121 /* gso_segs will be calculated later */
1126 static int checksum_setup(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1128 bool recalculate_partial_csum
= false;
1130 /* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1131 * peers can fail to set NETRXF_csum_blank when sending a GSO
1132 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1133 * recalculate the partial checksum.
1135 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
&& skb_is_gso(skb
)) {
1136 queue
->stats
.rx_gso_checksum_fixup
++;
1137 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1138 recalculate_partial_csum
= true;
1141 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1142 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1145 return skb_checksum_setup(skb
, recalculate_partial_csum
);
1148 static bool tx_credit_exceeded(struct xenvif_queue
*queue
, unsigned size
)
1150 u64 now
= get_jiffies_64();
1151 u64 next_credit
= queue
->credit_window_start
+
1152 msecs_to_jiffies(queue
->credit_usec
/ 1000);
1154 /* Timer could already be pending in rare cases. */
1155 if (timer_pending(&queue
->credit_timeout
))
1158 /* Passed the point where we can replenish credit? */
1159 if (time_after_eq64(now
, next_credit
)) {
1160 queue
->credit_window_start
= now
;
1161 tx_add_credit(queue
);
1164 /* Still too big to send right now? Set a callback. */
1165 if (size
> queue
->remaining_credit
) {
1166 queue
->credit_timeout
.data
=
1167 (unsigned long)queue
;
1168 mod_timer(&queue
->credit_timeout
,
1170 queue
->credit_window_start
= next_credit
;
1178 static void xenvif_tx_build_gops(struct xenvif_queue
*queue
,
1183 struct gnttab_map_grant_ref
*gop
= queue
->tx_map_ops
, *request_gop
;
1184 struct sk_buff
*skb
;
1187 while (skb_queue_len(&queue
->tx_queue
) < budget
) {
1188 struct xen_netif_tx_request txreq
;
1189 struct xen_netif_tx_request txfrags
[XEN_NETBK_LEGACY_SLOTS_MAX
];
1190 struct xen_netif_extra_info extras
[XEN_NETIF_EXTRA_TYPE_MAX
-1];
1194 unsigned int data_len
;
1195 pending_ring_idx_t index
;
1197 if (queue
->tx
.sring
->req_prod
- queue
->tx
.req_cons
>
1198 XEN_NETIF_TX_RING_SIZE
) {
1199 netdev_err(queue
->vif
->dev
,
1200 "Impossible number of requests. "
1201 "req_prod %d, req_cons %d, size %ld\n",
1202 queue
->tx
.sring
->req_prod
, queue
->tx
.req_cons
,
1203 XEN_NETIF_TX_RING_SIZE
);
1204 xenvif_fatal_tx_err(queue
->vif
);
1208 work_to_do
= RING_HAS_UNCONSUMED_REQUESTS(&queue
->tx
);
1212 idx
= queue
->tx
.req_cons
;
1213 rmb(); /* Ensure that we see the request before we copy it. */
1214 memcpy(&txreq
, RING_GET_REQUEST(&queue
->tx
, idx
), sizeof(txreq
));
1216 /* Credit-based scheduling. */
1217 if (txreq
.size
> queue
->remaining_credit
&&
1218 tx_credit_exceeded(queue
, txreq
.size
))
1221 queue
->remaining_credit
-= txreq
.size
;
1224 queue
->tx
.req_cons
= ++idx
;
1226 memset(extras
, 0, sizeof(extras
));
1227 if (txreq
.flags
& XEN_NETTXF_extra_info
) {
1228 work_to_do
= xenvif_get_extras(queue
, extras
,
1230 idx
= queue
->tx
.req_cons
;
1231 if (unlikely(work_to_do
< 0))
1235 ret
= xenvif_count_requests(queue
, &txreq
, txfrags
, work_to_do
);
1236 if (unlikely(ret
< 0))
1241 if (unlikely(txreq
.size
< ETH_HLEN
)) {
1242 netdev_dbg(queue
->vif
->dev
,
1243 "Bad packet size: %d\n", txreq
.size
);
1244 xenvif_tx_err(queue
, &txreq
, idx
);
1248 /* No crossing a page as the payload mustn't fragment. */
1249 if (unlikely((txreq
.offset
+ txreq
.size
) > PAGE_SIZE
)) {
1250 netdev_err(queue
->vif
->dev
,
1251 "txreq.offset: %x, size: %u, end: %lu\n",
1252 txreq
.offset
, txreq
.size
,
1253 (txreq
.offset
&~PAGE_MASK
) + txreq
.size
);
1254 xenvif_fatal_tx_err(queue
->vif
);
1258 index
= pending_index(queue
->pending_cons
);
1259 pending_idx
= queue
->pending_ring
[index
];
1261 data_len
= (txreq
.size
> XEN_NETBACK_TX_COPY_LEN
&&
1262 ret
< XEN_NETBK_LEGACY_SLOTS_MAX
) ?
1263 XEN_NETBACK_TX_COPY_LEN
: txreq
.size
;
1265 skb
= xenvif_alloc_skb(data_len
);
1266 if (unlikely(skb
== NULL
)) {
1267 netdev_dbg(queue
->vif
->dev
,
1268 "Can't allocate a skb in start_xmit.\n");
1269 xenvif_tx_err(queue
, &txreq
, idx
);
1273 if (extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1].type
) {
1274 struct xen_netif_extra_info
*gso
;
1275 gso
= &extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1];
1277 if (xenvif_set_skb_gso(queue
->vif
, skb
, gso
)) {
1278 /* Failure in xenvif_set_skb_gso is fatal. */
1284 XENVIF_TX_CB(skb
)->pending_idx
= pending_idx
;
1286 __skb_put(skb
, data_len
);
1287 queue
->tx_copy_ops
[*copy_ops
].source
.u
.ref
= txreq
.gref
;
1288 queue
->tx_copy_ops
[*copy_ops
].source
.domid
= queue
->vif
->domid
;
1289 queue
->tx_copy_ops
[*copy_ops
].source
.offset
= txreq
.offset
;
1291 queue
->tx_copy_ops
[*copy_ops
].dest
.u
.gmfn
=
1292 virt_to_mfn(skb
->data
);
1293 queue
->tx_copy_ops
[*copy_ops
].dest
.domid
= DOMID_SELF
;
1294 queue
->tx_copy_ops
[*copy_ops
].dest
.offset
=
1295 offset_in_page(skb
->data
);
1297 queue
->tx_copy_ops
[*copy_ops
].len
= data_len
;
1298 queue
->tx_copy_ops
[*copy_ops
].flags
= GNTCOPY_source_gref
;
1302 skb_shinfo(skb
)->nr_frags
= ret
;
1303 if (data_len
< txreq
.size
) {
1304 skb_shinfo(skb
)->nr_frags
++;
1305 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1307 xenvif_tx_create_map_op(queue
, pending_idx
, &txreq
, gop
);
1310 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1311 INVALID_PENDING_IDX
);
1312 memcpy(&queue
->pending_tx_info
[pending_idx
].req
, &txreq
,
1316 queue
->pending_cons
++;
1318 request_gop
= xenvif_get_requests(queue
, skb
, txfrags
, gop
);
1319 if (request_gop
== NULL
) {
1321 xenvif_tx_err(queue
, &txreq
, idx
);
1326 __skb_queue_tail(&queue
->tx_queue
, skb
);
1328 queue
->tx
.req_cons
= idx
;
1330 if (((gop
-queue
->tx_map_ops
) >= ARRAY_SIZE(queue
->tx_map_ops
)) ||
1331 (*copy_ops
>= ARRAY_SIZE(queue
->tx_copy_ops
)))
1335 (*map_ops
) = gop
- queue
->tx_map_ops
;
1339 /* Consolidate skb with a frag_list into a brand new one with local pages on
1340 * frags. Returns 0 or -ENOMEM if can't allocate new pages.
1342 static int xenvif_handle_frag_list(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1344 unsigned int offset
= skb_headlen(skb
);
1345 skb_frag_t frags
[MAX_SKB_FRAGS
];
1347 struct ubuf_info
*uarg
;
1348 struct sk_buff
*nskb
= skb_shinfo(skb
)->frag_list
;
1350 queue
->stats
.tx_zerocopy_sent
+= 2;
1351 queue
->stats
.tx_frag_overflow
++;
1353 xenvif_fill_frags(queue
, nskb
);
1354 /* Subtract frags size, we will correct it later */
1355 skb
->truesize
-= skb
->data_len
;
1356 skb
->len
+= nskb
->len
;
1357 skb
->data_len
+= nskb
->len
;
1359 /* create a brand new frags array and coalesce there */
1360 for (i
= 0; offset
< skb
->len
; i
++) {
1364 BUG_ON(i
>= MAX_SKB_FRAGS
);
1365 page
= alloc_page(GFP_ATOMIC
);
1368 skb
->truesize
+= skb
->data_len
;
1369 for (j
= 0; j
< i
; j
++)
1370 put_page(frags
[j
].page
.p
);
1374 if (offset
+ PAGE_SIZE
< skb
->len
)
1377 len
= skb
->len
- offset
;
1378 if (skb_copy_bits(skb
, offset
, page_address(page
), len
))
1382 frags
[i
].page
.p
= page
;
1383 frags
[i
].page_offset
= 0;
1384 skb_frag_size_set(&frags
[i
], len
);
1387 /* Copied all the bits from the frag list -- free it. */
1388 skb_frag_list_init(skb
);
1389 xenvif_skb_zerocopy_prepare(queue
, nskb
);
1392 /* Release all the original (foreign) frags. */
1393 for (f
= 0; f
< skb_shinfo(skb
)->nr_frags
; f
++)
1394 skb_frag_unref(skb
, f
);
1395 uarg
= skb_shinfo(skb
)->destructor_arg
;
1396 /* increase inflight counter to offset decrement in callback */
1397 atomic_inc(&queue
->inflight_packets
);
1398 uarg
->callback(uarg
, true);
1399 skb_shinfo(skb
)->destructor_arg
= NULL
;
1401 /* Fill the skb with the new (local) frags. */
1402 memcpy(skb_shinfo(skb
)->frags
, frags
, i
* sizeof(skb_frag_t
));
1403 skb_shinfo(skb
)->nr_frags
= i
;
1404 skb
->truesize
+= i
* PAGE_SIZE
;
1409 static int xenvif_tx_submit(struct xenvif_queue
*queue
)
1411 struct gnttab_map_grant_ref
*gop_map
= queue
->tx_map_ops
;
1412 struct gnttab_copy
*gop_copy
= queue
->tx_copy_ops
;
1413 struct sk_buff
*skb
;
1416 while ((skb
= __skb_dequeue(&queue
->tx_queue
)) != NULL
) {
1417 struct xen_netif_tx_request
*txp
;
1421 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
1422 txp
= &queue
->pending_tx_info
[pending_idx
].req
;
1424 /* Check the remap error code. */
1425 if (unlikely(xenvif_tx_check_gop(queue
, skb
, &gop_map
, &gop_copy
))) {
1426 /* If there was an error, xenvif_tx_check_gop is
1427 * expected to release all the frags which were mapped,
1428 * so kfree_skb shouldn't do it again
1430 skb_shinfo(skb
)->nr_frags
= 0;
1431 if (skb_has_frag_list(skb
)) {
1432 struct sk_buff
*nskb
=
1433 skb_shinfo(skb
)->frag_list
;
1434 skb_shinfo(nskb
)->nr_frags
= 0;
1440 data_len
= skb
->len
;
1441 callback_param(queue
, pending_idx
).ctx
= NULL
;
1442 if (data_len
< txp
->size
) {
1443 /* Append the packet payload as a fragment. */
1444 txp
->offset
+= data_len
;
1445 txp
->size
-= data_len
;
1447 /* Schedule a response immediately. */
1448 xenvif_idx_release(queue
, pending_idx
,
1449 XEN_NETIF_RSP_OKAY
);
1452 if (txp
->flags
& XEN_NETTXF_csum_blank
)
1453 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1454 else if (txp
->flags
& XEN_NETTXF_data_validated
)
1455 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1457 xenvif_fill_frags(queue
, skb
);
1459 if (unlikely(skb_has_frag_list(skb
))) {
1460 if (xenvif_handle_frag_list(queue
, skb
)) {
1461 if (net_ratelimit())
1462 netdev_err(queue
->vif
->dev
,
1463 "Not enough memory to consolidate frag_list!\n");
1464 xenvif_skb_zerocopy_prepare(queue
, skb
);
1470 skb
->dev
= queue
->vif
->dev
;
1471 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1472 skb_reset_network_header(skb
);
1474 if (checksum_setup(queue
, skb
)) {
1475 netdev_dbg(queue
->vif
->dev
,
1476 "Can't setup checksum in net_tx_action\n");
1477 /* We have to set this flag to trigger the callback */
1478 if (skb_shinfo(skb
)->destructor_arg
)
1479 xenvif_skb_zerocopy_prepare(queue
, skb
);
1484 skb_probe_transport_header(skb
, 0);
1486 /* If the packet is GSO then we will have just set up the
1487 * transport header offset in checksum_setup so it's now
1488 * straightforward to calculate gso_segs.
1490 if (skb_is_gso(skb
)) {
1491 int mss
= skb_shinfo(skb
)->gso_size
;
1492 int hdrlen
= skb_transport_header(skb
) -
1493 skb_mac_header(skb
) +
1496 skb_shinfo(skb
)->gso_segs
=
1497 DIV_ROUND_UP(skb
->len
- hdrlen
, mss
);
1500 queue
->stats
.rx_bytes
+= skb
->len
;
1501 queue
->stats
.rx_packets
++;
1505 /* Set this flag right before netif_receive_skb, otherwise
1506 * someone might think this packet already left netback, and
1507 * do a skb_copy_ubufs while we are still in control of the
1508 * skb. E.g. the __pskb_pull_tail earlier can do such thing.
1510 if (skb_shinfo(skb
)->destructor_arg
) {
1511 xenvif_skb_zerocopy_prepare(queue
, skb
);
1512 queue
->stats
.tx_zerocopy_sent
++;
1515 netif_receive_skb(skb
);
1521 void xenvif_zerocopy_callback(struct ubuf_info
*ubuf
, bool zerocopy_success
)
1523 unsigned long flags
;
1524 pending_ring_idx_t index
;
1525 struct xenvif_queue
*queue
= ubuf_to_queue(ubuf
);
1527 /* This is the only place where we grab this lock, to protect callbacks
1530 spin_lock_irqsave(&queue
->callback_lock
, flags
);
1532 u16 pending_idx
= ubuf
->desc
;
1533 ubuf
= (struct ubuf_info
*) ubuf
->ctx
;
1534 BUG_ON(queue
->dealloc_prod
- queue
->dealloc_cons
>=
1536 index
= pending_index(queue
->dealloc_prod
);
1537 queue
->dealloc_ring
[index
] = pending_idx
;
1538 /* Sync with xenvif_tx_dealloc_action:
1539 * insert idx then incr producer.
1542 queue
->dealloc_prod
++;
1544 wake_up(&queue
->dealloc_wq
);
1545 spin_unlock_irqrestore(&queue
->callback_lock
, flags
);
1547 if (likely(zerocopy_success
))
1548 queue
->stats
.tx_zerocopy_success
++;
1550 queue
->stats
.tx_zerocopy_fail
++;
1551 xenvif_skb_zerocopy_complete(queue
);
1554 static inline void xenvif_tx_dealloc_action(struct xenvif_queue
*queue
)
1556 struct gnttab_unmap_grant_ref
*gop
;
1557 pending_ring_idx_t dc
, dp
;
1558 u16 pending_idx
, pending_idx_release
[MAX_PENDING_REQS
];
1561 dc
= queue
->dealloc_cons
;
1562 gop
= queue
->tx_unmap_ops
;
1564 /* Free up any grants we have finished using */
1566 dp
= queue
->dealloc_prod
;
1568 /* Ensure we see all indices enqueued by all
1569 * xenvif_zerocopy_callback().
1574 BUG_ON(gop
- queue
->tx_unmap_ops
> MAX_PENDING_REQS
);
1576 queue
->dealloc_ring
[pending_index(dc
++)];
1578 pending_idx_release
[gop
-queue
->tx_unmap_ops
] =
1580 queue
->pages_to_unmap
[gop
-queue
->tx_unmap_ops
] =
1581 queue
->mmap_pages
[pending_idx
];
1582 gnttab_set_unmap_op(gop
,
1583 idx_to_kaddr(queue
, pending_idx
),
1585 queue
->grant_tx_handle
[pending_idx
]);
1586 xenvif_grant_handle_reset(queue
, pending_idx
);
1590 } while (dp
!= queue
->dealloc_prod
);
1592 queue
->dealloc_cons
= dc
;
1594 if (gop
- queue
->tx_unmap_ops
> 0) {
1596 ret
= gnttab_unmap_refs(queue
->tx_unmap_ops
,
1598 queue
->pages_to_unmap
,
1599 gop
- queue
->tx_unmap_ops
);
1601 netdev_err(queue
->vif
->dev
, "Unmap fail: nr_ops %tx ret %d\n",
1602 gop
- queue
->tx_unmap_ops
, ret
);
1603 for (i
= 0; i
< gop
- queue
->tx_unmap_ops
; ++i
) {
1604 if (gop
[i
].status
!= GNTST_okay
)
1605 netdev_err(queue
->vif
->dev
,
1606 " host_addr: %llx handle: %x status: %d\n",
1615 for (i
= 0; i
< gop
- queue
->tx_unmap_ops
; ++i
)
1616 xenvif_idx_release(queue
, pending_idx_release
[i
],
1617 XEN_NETIF_RSP_OKAY
);
1621 /* Called after netfront has transmitted */
1622 int xenvif_tx_action(struct xenvif_queue
*queue
, int budget
)
1624 unsigned nr_mops
, nr_cops
= 0;
1627 if (unlikely(!tx_work_todo(queue
)))
1630 xenvif_tx_build_gops(queue
, budget
, &nr_cops
, &nr_mops
);
1635 gnttab_batch_copy(queue
->tx_copy_ops
, nr_cops
);
1637 ret
= gnttab_map_refs(queue
->tx_map_ops
,
1639 queue
->pages_to_map
,
1644 work_done
= xenvif_tx_submit(queue
);
1649 static void xenvif_idx_release(struct xenvif_queue
*queue
, u16 pending_idx
,
1652 struct pending_tx_info
*pending_tx_info
;
1653 pending_ring_idx_t index
;
1654 unsigned long flags
;
1656 pending_tx_info
= &queue
->pending_tx_info
[pending_idx
];
1658 spin_lock_irqsave(&queue
->response_lock
, flags
);
1660 make_tx_response(queue
, &pending_tx_info
->req
, status
);
1662 /* Release the pending index before pusing the Tx response so
1663 * its available before a new Tx request is pushed by the
1666 index
= pending_index(queue
->pending_prod
++);
1667 queue
->pending_ring
[index
] = pending_idx
;
1669 push_tx_responses(queue
);
1671 spin_unlock_irqrestore(&queue
->response_lock
, flags
);
1675 static void make_tx_response(struct xenvif_queue
*queue
,
1676 struct xen_netif_tx_request
*txp
,
1679 RING_IDX i
= queue
->tx
.rsp_prod_pvt
;
1680 struct xen_netif_tx_response
*resp
;
1682 resp
= RING_GET_RESPONSE(&queue
->tx
, i
);
1686 if (txp
->flags
& XEN_NETTXF_extra_info
)
1687 RING_GET_RESPONSE(&queue
->tx
, ++i
)->status
= XEN_NETIF_RSP_NULL
;
1689 queue
->tx
.rsp_prod_pvt
= ++i
;
1692 static void push_tx_responses(struct xenvif_queue
*queue
)
1696 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue
->tx
, notify
);
1698 notify_remote_via_irq(queue
->tx_irq
);
1701 static struct xen_netif_rx_response
*make_rx_response(struct xenvif_queue
*queue
,
1708 RING_IDX i
= queue
->rx
.rsp_prod_pvt
;
1709 struct xen_netif_rx_response
*resp
;
1711 resp
= RING_GET_RESPONSE(&queue
->rx
, i
);
1712 resp
->offset
= offset
;
1713 resp
->flags
= flags
;
1715 resp
->status
= (s16
)size
;
1717 resp
->status
= (s16
)st
;
1719 queue
->rx
.rsp_prod_pvt
= ++i
;
1724 void xenvif_idx_unmap(struct xenvif_queue
*queue
, u16 pending_idx
)
1727 struct gnttab_unmap_grant_ref tx_unmap_op
;
1729 gnttab_set_unmap_op(&tx_unmap_op
,
1730 idx_to_kaddr(queue
, pending_idx
),
1732 queue
->grant_tx_handle
[pending_idx
]);
1733 xenvif_grant_handle_reset(queue
, pending_idx
);
1735 ret
= gnttab_unmap_refs(&tx_unmap_op
, NULL
,
1736 &queue
->mmap_pages
[pending_idx
], 1);
1738 netdev_err(queue
->vif
->dev
,
1739 "Unmap fail: ret: %d pending_idx: %d host_addr: %llx handle: %x status: %d\n",
1742 tx_unmap_op
.host_addr
,
1744 tx_unmap_op
.status
);
1749 static inline int tx_work_todo(struct xenvif_queue
*queue
)
1751 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue
->tx
)))
1757 static inline bool tx_dealloc_work_todo(struct xenvif_queue
*queue
)
1759 return queue
->dealloc_cons
!= queue
->dealloc_prod
;
1762 void xenvif_unmap_frontend_rings(struct xenvif_queue
*queue
)
1764 if (queue
->tx
.sring
)
1765 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue
->vif
),
1767 if (queue
->rx
.sring
)
1768 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue
->vif
),
1772 int xenvif_map_frontend_rings(struct xenvif_queue
*queue
,
1773 grant_ref_t tx_ring_ref
,
1774 grant_ref_t rx_ring_ref
)
1777 struct xen_netif_tx_sring
*txs
;
1778 struct xen_netif_rx_sring
*rxs
;
1782 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue
->vif
),
1783 &tx_ring_ref
, 1, &addr
);
1787 txs
= (struct xen_netif_tx_sring
*)addr
;
1788 BACK_RING_INIT(&queue
->tx
, txs
, PAGE_SIZE
);
1790 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue
->vif
),
1791 &rx_ring_ref
, 1, &addr
);
1795 rxs
= (struct xen_netif_rx_sring
*)addr
;
1796 BACK_RING_INIT(&queue
->rx
, rxs
, PAGE_SIZE
);
1801 xenvif_unmap_frontend_rings(queue
);
1805 static void xenvif_queue_carrier_off(struct xenvif_queue
*queue
)
1807 struct xenvif
*vif
= queue
->vif
;
1809 queue
->stalled
= true;
1811 /* At least one queue has stalled? Disable the carrier. */
1812 spin_lock(&vif
->lock
);
1813 if (vif
->stalled_queues
++ == 0) {
1814 netdev_info(vif
->dev
, "Guest Rx stalled");
1815 netif_carrier_off(vif
->dev
);
1817 spin_unlock(&vif
->lock
);
1820 static void xenvif_queue_carrier_on(struct xenvif_queue
*queue
)
1822 struct xenvif
*vif
= queue
->vif
;
1824 queue
->last_rx_time
= jiffies
; /* Reset Rx stall detection. */
1825 queue
->stalled
= false;
1827 /* All queues are ready? Enable the carrier. */
1828 spin_lock(&vif
->lock
);
1829 if (--vif
->stalled_queues
== 0) {
1830 netdev_info(vif
->dev
, "Guest Rx ready");
1831 netif_carrier_on(vif
->dev
);
1833 spin_unlock(&vif
->lock
);
1836 static bool xenvif_rx_queue_stalled(struct xenvif_queue
*queue
)
1838 RING_IDX prod
, cons
;
1840 prod
= queue
->rx
.sring
->req_prod
;
1841 cons
= queue
->rx
.req_cons
;
1843 return !queue
->stalled
1844 && prod
- cons
< XEN_NETBK_RX_SLOTS_MAX
1845 && time_after(jiffies
,
1846 queue
->last_rx_time
+ queue
->vif
->stall_timeout
);
1849 static bool xenvif_rx_queue_ready(struct xenvif_queue
*queue
)
1851 RING_IDX prod
, cons
;
1853 prod
= queue
->rx
.sring
->req_prod
;
1854 cons
= queue
->rx
.req_cons
;
1856 return queue
->stalled
1857 && prod
- cons
>= XEN_NETBK_RX_SLOTS_MAX
;
1860 static bool xenvif_have_rx_work(struct xenvif_queue
*queue
)
1862 return (!skb_queue_empty(&queue
->rx_queue
)
1863 && xenvif_rx_ring_slots_available(queue
, XEN_NETBK_RX_SLOTS_MAX
))
1864 || (queue
->vif
->stall_timeout
&&
1865 (xenvif_rx_queue_stalled(queue
)
1866 || xenvif_rx_queue_ready(queue
)))
1867 || kthread_should_stop()
1868 || queue
->vif
->disabled
;
1871 static long xenvif_rx_queue_timeout(struct xenvif_queue
*queue
)
1873 struct sk_buff
*skb
;
1876 skb
= skb_peek(&queue
->rx_queue
);
1878 return MAX_SCHEDULE_TIMEOUT
;
1880 timeout
= XENVIF_RX_CB(skb
)->expires
- jiffies
;
1881 return timeout
< 0 ? 0 : timeout
;
1884 /* Wait until the guest Rx thread has work.
1886 * The timeout needs to be adjusted based on the current head of the
1887 * queue (and not just the head at the beginning). In particular, if
1888 * the queue is initially empty an infinite timeout is used and this
1889 * needs to be reduced when a skb is queued.
1891 * This cannot be done with wait_event_timeout() because it only
1892 * calculates the timeout once.
1894 static void xenvif_wait_for_rx_work(struct xenvif_queue
*queue
)
1898 if (xenvif_have_rx_work(queue
))
1904 prepare_to_wait(&queue
->wq
, &wait
, TASK_INTERRUPTIBLE
);
1905 if (xenvif_have_rx_work(queue
))
1907 ret
= schedule_timeout(xenvif_rx_queue_timeout(queue
));
1911 finish_wait(&queue
->wq
, &wait
);
1914 int xenvif_kthread_guest_rx(void *data
)
1916 struct xenvif_queue
*queue
= data
;
1917 struct xenvif
*vif
= queue
->vif
;
1919 if (!vif
->stall_timeout
)
1920 xenvif_queue_carrier_on(queue
);
1923 xenvif_wait_for_rx_work(queue
);
1925 if (kthread_should_stop())
1928 /* This frontend is found to be rogue, disable it in
1929 * kthread context. Currently this is only set when
1930 * netback finds out frontend sends malformed packet,
1931 * but we cannot disable the interface in softirq
1932 * context so we defer it here, if this thread is
1933 * associated with queue 0.
1935 if (unlikely(vif
->disabled
&& queue
->id
== 0)) {
1936 xenvif_carrier_off(vif
);
1940 if (!skb_queue_empty(&queue
->rx_queue
))
1941 xenvif_rx_action(queue
);
1943 /* If the guest hasn't provided any Rx slots for a
1944 * while it's probably not responsive, drop the
1945 * carrier so packets are dropped earlier.
1947 if (vif
->stall_timeout
) {
1948 if (xenvif_rx_queue_stalled(queue
))
1949 xenvif_queue_carrier_off(queue
);
1950 else if (xenvif_rx_queue_ready(queue
))
1951 xenvif_queue_carrier_on(queue
);
1954 /* Queued packets may have foreign pages from other
1955 * domains. These cannot be queued indefinitely as
1956 * this would starve guests of grant refs and transmit
1959 xenvif_rx_queue_drop_expired(queue
);
1961 xenvif_rx_queue_maybe_wake(queue
);
1966 /* Bin any remaining skbs */
1967 xenvif_rx_queue_purge(queue
);
1972 static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue
*queue
)
1974 /* Dealloc thread must remain running until all inflight
1977 return kthread_should_stop() &&
1978 !atomic_read(&queue
->inflight_packets
);
1981 int xenvif_dealloc_kthread(void *data
)
1983 struct xenvif_queue
*queue
= data
;
1986 wait_event_interruptible(queue
->dealloc_wq
,
1987 tx_dealloc_work_todo(queue
) ||
1988 xenvif_dealloc_kthread_should_stop(queue
));
1989 if (xenvif_dealloc_kthread_should_stop(queue
))
1992 xenvif_tx_dealloc_action(queue
);
1996 /* Unmap anything remaining*/
1997 if (tx_dealloc_work_todo(queue
))
1998 xenvif_tx_dealloc_action(queue
);
2003 static int __init
netback_init(void)
2010 /* Allow as many queues as there are CPUs, by default */
2011 xenvif_max_queues
= num_online_cpus();
2013 if (fatal_skb_slots
< XEN_NETBK_LEGACY_SLOTS_MAX
) {
2014 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n",
2015 fatal_skb_slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
2016 fatal_skb_slots
= XEN_NETBK_LEGACY_SLOTS_MAX
;
2019 rc
= xenvif_xenbus_init();
2023 #ifdef CONFIG_DEBUG_FS
2024 xen_netback_dbg_root
= debugfs_create_dir("xen-netback", NULL
);
2025 if (IS_ERR_OR_NULL(xen_netback_dbg_root
))
2026 pr_warn("Init of debugfs returned %ld!\n",
2027 PTR_ERR(xen_netback_dbg_root
));
2028 #endif /* CONFIG_DEBUG_FS */
2036 module_init(netback_init
);
2038 static void __exit
netback_fini(void)
2040 #ifdef CONFIG_DEBUG_FS
2041 if (!IS_ERR_OR_NULL(xen_netback_dbg_root
))
2042 debugfs_remove_recursive(xen_netback_dbg_root
);
2043 #endif /* CONFIG_DEBUG_FS */
2044 xenvif_xenbus_fini();
2046 module_exit(netback_fini
);
2048 MODULE_LICENSE("Dual BSD/GPL");
2049 MODULE_ALIAS("xen-backend:vif");