2 * Back-end of the driver for virtual network devices. This portion of the
3 * driver exports a 'unified' network-device interface that can be accessed
4 * by any operating system that implements a compatible front end. A
5 * reference front-end implementation can be found in:
6 * drivers/net/xen-netfront.c
8 * Copyright (c) 2002-2005, K A Fraser
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #include <linux/kthread.h>
38 #include <linux/if_vlan.h>
39 #include <linux/udp.h>
44 #include <xen/events.h>
45 #include <xen/interface/memory.h>
47 #include <asm/xen/hypercall.h>
48 #include <asm/xen/page.h>
50 /* Provide an option to disable split event channels at load time as
51 * event channels are limited resource. Split event channels are
54 bool separate_tx_rx_irq
= 1;
55 module_param(separate_tx_rx_irq
, bool, 0644);
58 * This is the maximum slots a skb can have. If a guest sends a skb
59 * which exceeds this limit it is considered malicious.
61 #define FATAL_SKB_SLOTS_DEFAULT 20
62 static unsigned int fatal_skb_slots
= FATAL_SKB_SLOTS_DEFAULT
;
63 module_param(fatal_skb_slots
, uint
, 0444);
66 * To avoid confusion, we define XEN_NETBK_LEGACY_SLOTS_MAX indicating
67 * the maximum slots a valid packet can use. Now this value is defined
68 * to be XEN_NETIF_NR_SLOTS_MIN, which is supposed to be supported by
71 #define XEN_NETBK_LEGACY_SLOTS_MAX XEN_NETIF_NR_SLOTS_MIN
73 typedef unsigned int pending_ring_idx_t
;
74 #define INVALID_PENDING_RING_IDX (~0U)
76 struct pending_tx_info
{
77 struct xen_netif_tx_request req
; /* coalesced tx request */
79 pending_ring_idx_t head
; /* head != INVALID_PENDING_RING_IDX
80 * if it is head of one or more tx
85 struct netbk_rx_meta
{
91 #define MAX_PENDING_REQS 256
93 /* Discriminate from any valid pending_idx value. */
94 #define INVALID_PENDING_IDX 0xFFFF
96 #define MAX_BUFFER_OFFSET PAGE_SIZE
98 /* extra field used in struct page */
101 #if BITS_PER_LONG < 64
103 #define GROUP_WIDTH (BITS_PER_LONG - IDX_WIDTH)
104 unsigned int group
:GROUP_WIDTH
;
105 unsigned int idx
:IDX_WIDTH
;
107 unsigned int group
, idx
;
114 wait_queue_head_t wq
;
115 struct task_struct
*task
;
117 struct sk_buff_head rx_queue
;
118 struct sk_buff_head tx_queue
;
120 struct timer_list net_timer
;
122 struct page
*mmap_pages
[MAX_PENDING_REQS
];
124 pending_ring_idx_t pending_prod
;
125 pending_ring_idx_t pending_cons
;
126 struct list_head net_schedule_list
;
128 /* Protect the net_schedule_list in netif. */
129 spinlock_t net_schedule_list_lock
;
131 atomic_t netfront_count
;
133 struct pending_tx_info pending_tx_info
[MAX_PENDING_REQS
];
134 /* Coalescing tx requests before copying makes number of grant
135 * copy ops greater or equal to number of slots required. In
136 * worst case a tx request consumes 2 gnttab_copy.
138 struct gnttab_copy tx_copy_ops
[2*MAX_PENDING_REQS
];
140 u16 pending_ring
[MAX_PENDING_REQS
];
143 * Given MAX_BUFFER_OFFSET of 4096 the worst case is that each
144 * head/fragment page uses 2 copy operations because it
145 * straddles two buffers in the frontend.
147 struct gnttab_copy grant_copy_op
[2*XEN_NETIF_RX_RING_SIZE
];
148 struct netbk_rx_meta meta
[2*XEN_NETIF_RX_RING_SIZE
];
151 static struct xen_netbk
*xen_netbk
;
152 static int xen_netbk_group_nr
;
155 * If head != INVALID_PENDING_RING_IDX, it means this tx request is head of
156 * one or more merged tx requests, otherwise it is the continuation of
157 * previous tx request.
159 static inline int pending_tx_is_head(struct xen_netbk
*netbk
, RING_IDX idx
)
161 return netbk
->pending_tx_info
[idx
].head
!= INVALID_PENDING_RING_IDX
;
164 void xen_netbk_add_xenvif(struct xenvif
*vif
)
167 int min_netfront_count
;
169 struct xen_netbk
*netbk
;
171 min_netfront_count
= atomic_read(&xen_netbk
[0].netfront_count
);
172 for (i
= 0; i
< xen_netbk_group_nr
; i
++) {
173 int netfront_count
= atomic_read(&xen_netbk
[i
].netfront_count
);
174 if (netfront_count
< min_netfront_count
) {
176 min_netfront_count
= netfront_count
;
180 netbk
= &xen_netbk
[min_group
];
183 atomic_inc(&netbk
->netfront_count
);
186 void xen_netbk_remove_xenvif(struct xenvif
*vif
)
188 struct xen_netbk
*netbk
= vif
->netbk
;
190 atomic_dec(&netbk
->netfront_count
);
193 static void xen_netbk_idx_release(struct xen_netbk
*netbk
, u16 pending_idx
,
195 static void make_tx_response(struct xenvif
*vif
,
196 struct xen_netif_tx_request
*txp
,
198 static struct xen_netif_rx_response
*make_rx_response(struct xenvif
*vif
,
205 static inline unsigned long idx_to_pfn(struct xen_netbk
*netbk
,
208 return page_to_pfn(netbk
->mmap_pages
[idx
]);
211 static inline unsigned long idx_to_kaddr(struct xen_netbk
*netbk
,
214 return (unsigned long)pfn_to_kaddr(idx_to_pfn(netbk
, idx
));
217 /* extra field used in struct page */
218 static inline void set_page_ext(struct page
*pg
, struct xen_netbk
*netbk
,
221 unsigned int group
= netbk
- xen_netbk
;
222 union page_ext ext
= { .e
= { .group
= group
+ 1, .idx
= idx
} };
224 BUILD_BUG_ON(sizeof(ext
) > sizeof(ext
.mapping
));
225 pg
->mapping
= ext
.mapping
;
228 static int get_page_ext(struct page
*pg
,
229 unsigned int *pgroup
, unsigned int *pidx
)
231 union page_ext ext
= { .mapping
= pg
->mapping
};
232 struct xen_netbk
*netbk
;
233 unsigned int group
, idx
;
235 group
= ext
.e
.group
- 1;
237 if (group
< 0 || group
>= xen_netbk_group_nr
)
240 netbk
= &xen_netbk
[group
];
244 if ((idx
< 0) || (idx
>= MAX_PENDING_REQS
))
247 if (netbk
->mmap_pages
[idx
] != pg
)
257 * This is the amount of packet we copy rather than map, so that the
258 * guest can't fiddle with the contents of the headers while we do
259 * packet processing on them (netfilter, routing, etc).
261 #define PKT_PROT_LEN (ETH_HLEN + \
263 sizeof(struct iphdr) + MAX_IPOPTLEN + \
264 sizeof(struct tcphdr) + MAX_TCP_OPTION_SPACE)
266 static u16
frag_get_pending_idx(skb_frag_t
*frag
)
268 return (u16
)frag
->page_offset
;
271 static void frag_set_pending_idx(skb_frag_t
*frag
, u16 pending_idx
)
273 frag
->page_offset
= pending_idx
;
276 static inline pending_ring_idx_t
pending_index(unsigned i
)
278 return i
& (MAX_PENDING_REQS
-1);
281 static inline pending_ring_idx_t
nr_pending_reqs(struct xen_netbk
*netbk
)
283 return MAX_PENDING_REQS
-
284 netbk
->pending_prod
+ netbk
->pending_cons
;
287 static void xen_netbk_kick_thread(struct xen_netbk
*netbk
)
292 static int max_required_rx_slots(struct xenvif
*vif
)
294 int max
= DIV_ROUND_UP(vif
->dev
->mtu
, PAGE_SIZE
);
296 /* XXX FIXME: RX path dependent on MAX_SKB_FRAGS */
297 if (vif
->can_sg
|| vif
->gso
|| vif
->gso_prefix
)
298 max
+= MAX_SKB_FRAGS
+ 1; /* extra_info + frags */
303 int xen_netbk_rx_ring_full(struct xenvif
*vif
)
305 RING_IDX peek
= vif
->rx_req_cons_peek
;
306 RING_IDX needed
= max_required_rx_slots(vif
);
308 return ((vif
->rx
.sring
->req_prod
- peek
) < needed
) ||
309 ((vif
->rx
.rsp_prod_pvt
+ XEN_NETIF_RX_RING_SIZE
- peek
) < needed
);
312 int xen_netbk_must_stop_queue(struct xenvif
*vif
)
314 if (!xen_netbk_rx_ring_full(vif
))
317 vif
->rx
.sring
->req_event
= vif
->rx_req_cons_peek
+
318 max_required_rx_slots(vif
);
319 mb(); /* request notification /then/ check the queue */
321 return xen_netbk_rx_ring_full(vif
);
325 * Returns true if we should start a new receive buffer instead of
326 * adding 'size' bytes to a buffer which currently contains 'offset'
329 static bool start_new_rx_buffer(int offset
, unsigned long size
, int head
)
331 /* simple case: we have completely filled the current buffer. */
332 if (offset
== MAX_BUFFER_OFFSET
)
336 * complex case: start a fresh buffer if the current frag
337 * would overflow the current buffer but only if:
338 * (i) this frag would fit completely in the next buffer
339 * and (ii) there is already some data in the current buffer
340 * and (iii) this is not the head buffer.
343 * - (i) stops us splitting a frag into two copies
344 * unless the frag is too large for a single buffer.
345 * - (ii) stops us from leaving a buffer pointlessly empty.
346 * - (iii) stops us leaving the first buffer
347 * empty. Strictly speaking this is already covered
348 * by (ii) but is explicitly checked because
349 * netfront relies on the first buffer being
350 * non-empty and can crash otherwise.
352 * This means we will effectively linearise small
353 * frags but do not needlessly split large buffers
354 * into multiple copies tend to give large frags their
355 * own buffers as before.
357 if ((offset
+ size
> MAX_BUFFER_OFFSET
) &&
358 (size
<= MAX_BUFFER_OFFSET
) && offset
&& !head
)
365 * Figure out how many ring slots we're going to need to send @skb to
366 * the guest. This function is essentially a dry run of
367 * netbk_gop_frag_copy.
369 unsigned int xen_netbk_count_skb_slots(struct xenvif
*vif
, struct sk_buff
*skb
)
374 count
= DIV_ROUND_UP(skb_headlen(skb
), PAGE_SIZE
);
376 copy_off
= skb_headlen(skb
) % PAGE_SIZE
;
378 if (skb_shinfo(skb
)->gso_size
)
381 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
382 unsigned long size
= skb_frag_size(&skb_shinfo(skb
)->frags
[i
]);
383 unsigned long offset
= skb_shinfo(skb
)->frags
[i
].page_offset
;
386 offset
&= ~PAGE_MASK
;
389 BUG_ON(offset
>= PAGE_SIZE
);
390 BUG_ON(copy_off
> MAX_BUFFER_OFFSET
);
392 bytes
= PAGE_SIZE
- offset
;
397 if (start_new_rx_buffer(copy_off
, bytes
, 0)) {
402 if (copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
403 bytes
= MAX_BUFFER_OFFSET
- copy_off
;
410 if (offset
== PAGE_SIZE
)
417 struct netrx_pending_operations
{
418 unsigned copy_prod
, copy_cons
;
419 unsigned meta_prod
, meta_cons
;
420 struct gnttab_copy
*copy
;
421 struct netbk_rx_meta
*meta
;
423 grant_ref_t copy_gref
;
426 static struct netbk_rx_meta
*get_next_rx_buffer(struct xenvif
*vif
,
427 struct netrx_pending_operations
*npo
)
429 struct netbk_rx_meta
*meta
;
430 struct xen_netif_rx_request
*req
;
432 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
434 meta
= npo
->meta
+ npo
->meta_prod
++;
440 npo
->copy_gref
= req
->gref
;
446 * Set up the grant operations for this fragment. If it's a flipping
447 * interface, we also set up the unmap request from here.
449 static void netbk_gop_frag_copy(struct xenvif
*vif
, struct sk_buff
*skb
,
450 struct netrx_pending_operations
*npo
,
451 struct page
*page
, unsigned long size
,
452 unsigned long offset
, int *head
)
454 struct gnttab_copy
*copy_gop
;
455 struct netbk_rx_meta
*meta
;
457 * These variables are used iff get_page_ext returns true,
458 * in which case they are guaranteed to be initialized.
460 unsigned int uninitialized_var(group
), uninitialized_var(idx
);
461 int foreign
= get_page_ext(page
, &group
, &idx
);
464 /* Data must not cross a page boundary. */
465 BUG_ON(size
+ offset
> PAGE_SIZE
<<compound_order(page
));
467 meta
= npo
->meta
+ npo
->meta_prod
- 1;
469 /* Skip unused frames from start of page */
470 page
+= offset
>> PAGE_SHIFT
;
471 offset
&= ~PAGE_MASK
;
474 BUG_ON(offset
>= PAGE_SIZE
);
475 BUG_ON(npo
->copy_off
> MAX_BUFFER_OFFSET
);
477 bytes
= PAGE_SIZE
- offset
;
482 if (start_new_rx_buffer(npo
->copy_off
, bytes
, *head
)) {
484 * Netfront requires there to be some data in the head
489 meta
= get_next_rx_buffer(vif
, npo
);
492 if (npo
->copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
493 bytes
= MAX_BUFFER_OFFSET
- npo
->copy_off
;
495 copy_gop
= npo
->copy
+ npo
->copy_prod
++;
496 copy_gop
->flags
= GNTCOPY_dest_gref
;
498 struct xen_netbk
*netbk
= &xen_netbk
[group
];
499 struct pending_tx_info
*src_pend
;
501 src_pend
= &netbk
->pending_tx_info
[idx
];
503 copy_gop
->source
.domid
= src_pend
->vif
->domid
;
504 copy_gop
->source
.u
.ref
= src_pend
->req
.gref
;
505 copy_gop
->flags
|= GNTCOPY_source_gref
;
507 void *vaddr
= page_address(page
);
508 copy_gop
->source
.domid
= DOMID_SELF
;
509 copy_gop
->source
.u
.gmfn
= virt_to_mfn(vaddr
);
511 copy_gop
->source
.offset
= offset
;
512 copy_gop
->dest
.domid
= vif
->domid
;
514 copy_gop
->dest
.offset
= npo
->copy_off
;
515 copy_gop
->dest
.u
.ref
= npo
->copy_gref
;
516 copy_gop
->len
= bytes
;
518 npo
->copy_off
+= bytes
;
525 if (offset
== PAGE_SIZE
&& size
) {
526 BUG_ON(!PageCompound(page
));
531 /* Leave a gap for the GSO descriptor. */
532 if (*head
&& skb_shinfo(skb
)->gso_size
&& !vif
->gso_prefix
)
535 *head
= 0; /* There must be something in this buffer now. */
541 * Prepare an SKB to be transmitted to the frontend.
543 * This function is responsible for allocating grant operations, meta
546 * It returns the number of meta structures consumed. The number of
547 * ring slots used is always equal to the number of meta slots used
548 * plus the number of GSO descriptors used. Currently, we use either
549 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
550 * frontend-side LRO).
552 static int netbk_gop_skb(struct sk_buff
*skb
,
553 struct netrx_pending_operations
*npo
)
555 struct xenvif
*vif
= netdev_priv(skb
->dev
);
556 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
558 struct xen_netif_rx_request
*req
;
559 struct netbk_rx_meta
*meta
;
564 old_meta_prod
= npo
->meta_prod
;
566 /* Set up a GSO prefix descriptor, if necessary */
567 if (skb_shinfo(skb
)->gso_size
&& vif
->gso_prefix
) {
568 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
569 meta
= npo
->meta
+ npo
->meta_prod
++;
570 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
575 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
576 meta
= npo
->meta
+ npo
->meta_prod
++;
578 if (!vif
->gso_prefix
)
579 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
586 npo
->copy_gref
= req
->gref
;
589 while (data
< skb_tail_pointer(skb
)) {
590 unsigned int offset
= offset_in_page(data
);
591 unsigned int len
= PAGE_SIZE
- offset
;
593 if (data
+ len
> skb_tail_pointer(skb
))
594 len
= skb_tail_pointer(skb
) - data
;
596 netbk_gop_frag_copy(vif
, skb
, npo
,
597 virt_to_page(data
), len
, offset
, &head
);
601 for (i
= 0; i
< nr_frags
; i
++) {
602 netbk_gop_frag_copy(vif
, skb
, npo
,
603 skb_frag_page(&skb_shinfo(skb
)->frags
[i
]),
604 skb_frag_size(&skb_shinfo(skb
)->frags
[i
]),
605 skb_shinfo(skb
)->frags
[i
].page_offset
,
609 return npo
->meta_prod
- old_meta_prod
;
613 * This is a twin to netbk_gop_skb. Assume that netbk_gop_skb was
614 * used to set up the operations on the top of
615 * netrx_pending_operations, which have since been done. Check that
616 * they didn't give any errors and advance over them.
618 static int netbk_check_gop(struct xenvif
*vif
, int nr_meta_slots
,
619 struct netrx_pending_operations
*npo
)
621 struct gnttab_copy
*copy_op
;
622 int status
= XEN_NETIF_RSP_OKAY
;
625 for (i
= 0; i
< nr_meta_slots
; i
++) {
626 copy_op
= npo
->copy
+ npo
->copy_cons
++;
627 if (copy_op
->status
!= GNTST_okay
) {
629 "Bad status %d from copy to DOM%d.\n",
630 copy_op
->status
, vif
->domid
);
631 status
= XEN_NETIF_RSP_ERROR
;
638 static void netbk_add_frag_responses(struct xenvif
*vif
, int status
,
639 struct netbk_rx_meta
*meta
,
643 unsigned long offset
;
645 /* No fragments used */
646 if (nr_meta_slots
<= 1)
651 for (i
= 0; i
< nr_meta_slots
; i
++) {
653 if (i
== nr_meta_slots
- 1)
656 flags
= XEN_NETRXF_more_data
;
659 make_rx_response(vif
, meta
[i
].id
, status
, offset
,
660 meta
[i
].size
, flags
);
664 struct skb_cb_overlay
{
668 static void xen_netbk_rx_action(struct xen_netbk
*netbk
)
670 struct xenvif
*vif
= NULL
, *tmp
;
673 struct xen_netif_rx_response
*resp
;
674 struct sk_buff_head rxq
;
680 unsigned long offset
;
681 struct skb_cb_overlay
*sco
;
683 struct netrx_pending_operations npo
= {
684 .copy
= netbk
->grant_copy_op
,
688 skb_queue_head_init(&rxq
);
692 while ((skb
= skb_dequeue(&netbk
->rx_queue
)) != NULL
) {
693 vif
= netdev_priv(skb
->dev
);
694 nr_frags
= skb_shinfo(skb
)->nr_frags
;
696 sco
= (struct skb_cb_overlay
*)skb
->cb
;
697 sco
->meta_slots_used
= netbk_gop_skb(skb
, &npo
);
699 count
+= nr_frags
+ 1;
701 __skb_queue_tail(&rxq
, skb
);
703 /* Filled the batch queue? */
704 /* XXX FIXME: RX path dependent on MAX_SKB_FRAGS */
705 if (count
+ MAX_SKB_FRAGS
>= XEN_NETIF_RX_RING_SIZE
)
709 BUG_ON(npo
.meta_prod
> ARRAY_SIZE(netbk
->meta
));
714 BUG_ON(npo
.copy_prod
> ARRAY_SIZE(netbk
->grant_copy_op
));
715 gnttab_batch_copy(netbk
->grant_copy_op
, npo
.copy_prod
);
717 while ((skb
= __skb_dequeue(&rxq
)) != NULL
) {
718 sco
= (struct skb_cb_overlay
*)skb
->cb
;
720 vif
= netdev_priv(skb
->dev
);
722 if (netbk
->meta
[npo
.meta_cons
].gso_size
&& vif
->gso_prefix
) {
723 resp
= RING_GET_RESPONSE(&vif
->rx
,
724 vif
->rx
.rsp_prod_pvt
++);
726 resp
->flags
= XEN_NETRXF_gso_prefix
| XEN_NETRXF_more_data
;
728 resp
->offset
= netbk
->meta
[npo
.meta_cons
].gso_size
;
729 resp
->id
= netbk
->meta
[npo
.meta_cons
].id
;
730 resp
->status
= sco
->meta_slots_used
;
733 sco
->meta_slots_used
--;
737 vif
->dev
->stats
.tx_bytes
+= skb
->len
;
738 vif
->dev
->stats
.tx_packets
++;
740 status
= netbk_check_gop(vif
, sco
->meta_slots_used
, &npo
);
742 if (sco
->meta_slots_used
== 1)
745 flags
= XEN_NETRXF_more_data
;
747 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) /* local packet? */
748 flags
|= XEN_NETRXF_csum_blank
| XEN_NETRXF_data_validated
;
749 else if (skb
->ip_summed
== CHECKSUM_UNNECESSARY
)
750 /* remote but checksummed. */
751 flags
|= XEN_NETRXF_data_validated
;
754 resp
= make_rx_response(vif
, netbk
->meta
[npo
.meta_cons
].id
,
756 netbk
->meta
[npo
.meta_cons
].size
,
759 if (netbk
->meta
[npo
.meta_cons
].gso_size
&& !vif
->gso_prefix
) {
760 struct xen_netif_extra_info
*gso
=
761 (struct xen_netif_extra_info
*)
762 RING_GET_RESPONSE(&vif
->rx
,
763 vif
->rx
.rsp_prod_pvt
++);
765 resp
->flags
|= XEN_NETRXF_extra_info
;
767 gso
->u
.gso
.size
= netbk
->meta
[npo
.meta_cons
].gso_size
;
768 gso
->u
.gso
.type
= XEN_NETIF_GSO_TYPE_TCPV4
;
770 gso
->u
.gso
.features
= 0;
772 gso
->type
= XEN_NETIF_EXTRA_TYPE_GSO
;
776 netbk_add_frag_responses(vif
, status
,
777 netbk
->meta
+ npo
.meta_cons
+ 1,
778 sco
->meta_slots_used
);
780 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif
->rx
, ret
);
781 if (ret
&& list_empty(&vif
->notify_list
))
782 list_add_tail(&vif
->notify_list
, ¬ify
);
784 xenvif_notify_tx_completion(vif
);
787 npo
.meta_cons
+= sco
->meta_slots_used
;
791 list_for_each_entry_safe(vif
, tmp
, ¬ify
, notify_list
) {
792 notify_remote_via_irq(vif
->rx_irq
);
793 list_del_init(&vif
->notify_list
);
796 /* More work to do? */
797 if (!skb_queue_empty(&netbk
->rx_queue
) &&
798 !timer_pending(&netbk
->net_timer
))
799 xen_netbk_kick_thread(netbk
);
802 void xen_netbk_queue_tx_skb(struct xenvif
*vif
, struct sk_buff
*skb
)
804 struct xen_netbk
*netbk
= vif
->netbk
;
806 skb_queue_tail(&netbk
->rx_queue
, skb
);
808 xen_netbk_kick_thread(netbk
);
811 static void xen_netbk_alarm(unsigned long data
)
813 struct xen_netbk
*netbk
= (struct xen_netbk
*)data
;
814 xen_netbk_kick_thread(netbk
);
817 static int __on_net_schedule_list(struct xenvif
*vif
)
819 return !list_empty(&vif
->schedule_list
);
822 /* Must be called with net_schedule_list_lock held */
823 static void remove_from_net_schedule_list(struct xenvif
*vif
)
825 if (likely(__on_net_schedule_list(vif
))) {
826 list_del_init(&vif
->schedule_list
);
831 static struct xenvif
*poll_net_schedule_list(struct xen_netbk
*netbk
)
833 struct xenvif
*vif
= NULL
;
835 spin_lock_irq(&netbk
->net_schedule_list_lock
);
836 if (list_empty(&netbk
->net_schedule_list
))
839 vif
= list_first_entry(&netbk
->net_schedule_list
,
840 struct xenvif
, schedule_list
);
846 remove_from_net_schedule_list(vif
);
848 spin_unlock_irq(&netbk
->net_schedule_list_lock
);
852 void xen_netbk_schedule_xenvif(struct xenvif
*vif
)
855 struct xen_netbk
*netbk
= vif
->netbk
;
857 if (__on_net_schedule_list(vif
))
860 spin_lock_irqsave(&netbk
->net_schedule_list_lock
, flags
);
861 if (!__on_net_schedule_list(vif
) &&
862 likely(xenvif_schedulable(vif
))) {
863 list_add_tail(&vif
->schedule_list
, &netbk
->net_schedule_list
);
866 spin_unlock_irqrestore(&netbk
->net_schedule_list_lock
, flags
);
870 if ((nr_pending_reqs(netbk
) < (MAX_PENDING_REQS
/2)) &&
871 !list_empty(&netbk
->net_schedule_list
))
872 xen_netbk_kick_thread(netbk
);
875 void xen_netbk_deschedule_xenvif(struct xenvif
*vif
)
877 struct xen_netbk
*netbk
= vif
->netbk
;
878 spin_lock_irq(&netbk
->net_schedule_list_lock
);
879 remove_from_net_schedule_list(vif
);
880 spin_unlock_irq(&netbk
->net_schedule_list_lock
);
883 void xen_netbk_check_rx_xenvif(struct xenvif
*vif
)
887 RING_FINAL_CHECK_FOR_REQUESTS(&vif
->tx
, more_to_do
);
890 xen_netbk_schedule_xenvif(vif
);
893 static void tx_add_credit(struct xenvif
*vif
)
895 unsigned long max_burst
, max_credit
;
898 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
899 * Otherwise the interface can seize up due to insufficient credit.
901 max_burst
= RING_GET_REQUEST(&vif
->tx
, vif
->tx
.req_cons
)->size
;
902 max_burst
= min(max_burst
, 131072UL);
903 max_burst
= max(max_burst
, vif
->credit_bytes
);
905 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
906 max_credit
= vif
->remaining_credit
+ vif
->credit_bytes
;
907 if (max_credit
< vif
->remaining_credit
)
908 max_credit
= ULONG_MAX
; /* wrapped: clamp to ULONG_MAX */
910 vif
->remaining_credit
= min(max_credit
, max_burst
);
913 static void tx_credit_callback(unsigned long data
)
915 struct xenvif
*vif
= (struct xenvif
*)data
;
917 xen_netbk_check_rx_xenvif(vif
);
920 static void netbk_tx_err(struct xenvif
*vif
,
921 struct xen_netif_tx_request
*txp
, RING_IDX end
)
923 RING_IDX cons
= vif
->tx
.req_cons
;
926 make_tx_response(vif
, txp
, XEN_NETIF_RSP_ERROR
);
929 txp
= RING_GET_REQUEST(&vif
->tx
, cons
++);
931 vif
->tx
.req_cons
= cons
;
932 xen_netbk_check_rx_xenvif(vif
);
936 static void netbk_fatal_tx_err(struct xenvif
*vif
)
938 netdev_err(vif
->dev
, "fatal error; disabling device\n");
939 xenvif_carrier_off(vif
);
943 static int netbk_count_requests(struct xenvif
*vif
,
944 struct xen_netif_tx_request
*first
,
945 struct xen_netif_tx_request
*txp
,
948 RING_IDX cons
= vif
->tx
.req_cons
;
953 if (!(first
->flags
& XEN_NETTXF_more_data
))
957 struct xen_netif_tx_request dropped_tx
= { 0 };
959 if (slots
>= work_to_do
) {
961 "Asked for %d slots but exceeds this limit\n",
963 netbk_fatal_tx_err(vif
);
967 /* This guest is really using too many slots and
968 * considered malicious.
970 if (unlikely(slots
>= fatal_skb_slots
)) {
972 "Malicious frontend using %d slots, threshold %u\n",
973 slots
, fatal_skb_slots
);
974 netbk_fatal_tx_err(vif
);
978 /* Xen network protocol had implicit dependency on
979 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
980 * the historical MAX_SKB_FRAGS value 18 to honor the
981 * same behavior as before. Any packet using more than
982 * 18 slots but less than fatal_skb_slots slots is
985 if (!drop_err
&& slots
>= XEN_NETBK_LEGACY_SLOTS_MAX
) {
988 "Too many slots (%d) exceeding limit (%d), dropping packet\n",
989 slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
996 memcpy(txp
, RING_GET_REQUEST(&vif
->tx
, cons
+ slots
),
999 /* If the guest submitted a frame >= 64 KiB then
1000 * first->size overflowed and following slots will
1001 * appear to be larger than the frame.
1003 * This cannot be fatal error as there are buggy
1004 * frontends that do this.
1006 * Consume all slots and drop the packet.
1008 if (!drop_err
&& txp
->size
> first
->size
) {
1009 if (net_ratelimit())
1010 netdev_dbg(vif
->dev
,
1011 "Invalid tx request, slot size %u > remaining size %u\n",
1012 txp
->size
, first
->size
);
1016 first
->size
-= txp
->size
;
1019 if (unlikely((txp
->offset
+ txp
->size
) > PAGE_SIZE
)) {
1020 netdev_err(vif
->dev
, "Cross page boundary, txp->offset: %x, size: %u\n",
1021 txp
->offset
, txp
->size
);
1022 netbk_fatal_tx_err(vif
);
1026 more_data
= txp
->flags
& XEN_NETTXF_more_data
;
1031 } while (more_data
);
1034 netbk_tx_err(vif
, first
, cons
+ slots
);
1041 static struct page
*xen_netbk_alloc_page(struct xen_netbk
*netbk
,
1045 page
= alloc_page(GFP_KERNEL
|__GFP_COLD
);
1048 set_page_ext(page
, netbk
, pending_idx
);
1049 netbk
->mmap_pages
[pending_idx
] = page
;
1053 static struct gnttab_copy
*xen_netbk_get_requests(struct xen_netbk
*netbk
,
1055 struct sk_buff
*skb
,
1056 struct xen_netif_tx_request
*txp
,
1057 struct gnttab_copy
*gop
)
1059 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1060 skb_frag_t
*frags
= shinfo
->frags
;
1061 u16 pending_idx
= *((u16
*)skb
->data
);
1065 pending_ring_idx_t index
, start_idx
= 0;
1066 uint16_t dst_offset
;
1067 unsigned int nr_slots
;
1068 struct pending_tx_info
*first
= NULL
;
1070 /* At this point shinfo->nr_frags is in fact the number of
1071 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
1073 nr_slots
= shinfo
->nr_frags
;
1075 /* Skip first skb fragment if it is on same page as header fragment. */
1076 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
1078 /* Coalesce tx requests, at this point the packet passed in
1079 * should be <= 64K. Any packets larger than 64K have been
1080 * handled in netbk_count_requests().
1082 for (shinfo
->nr_frags
= slot
= start
; slot
< nr_slots
;
1083 shinfo
->nr_frags
++) {
1084 struct pending_tx_info
*pending_tx_info
=
1085 netbk
->pending_tx_info
;
1087 page
= alloc_page(GFP_KERNEL
|__GFP_COLD
);
1093 while (dst_offset
< PAGE_SIZE
&& slot
< nr_slots
) {
1094 gop
->flags
= GNTCOPY_source_gref
;
1096 gop
->source
.u
.ref
= txp
->gref
;
1097 gop
->source
.domid
= vif
->domid
;
1098 gop
->source
.offset
= txp
->offset
;
1100 gop
->dest
.domid
= DOMID_SELF
;
1102 gop
->dest
.offset
= dst_offset
;
1103 gop
->dest
.u
.gmfn
= virt_to_mfn(page_address(page
));
1105 if (dst_offset
+ txp
->size
> PAGE_SIZE
) {
1106 /* This page can only merge a portion
1107 * of tx request. Do not increment any
1108 * pointer / counter here. The txp
1109 * will be dealt with in future
1110 * rounds, eventually hitting the
1113 gop
->len
= PAGE_SIZE
- dst_offset
;
1114 txp
->offset
+= gop
->len
;
1115 txp
->size
-= gop
->len
;
1116 dst_offset
+= gop
->len
; /* quit loop */
1118 /* This tx request can be merged in the page */
1119 gop
->len
= txp
->size
;
1120 dst_offset
+= gop
->len
;
1122 index
= pending_index(netbk
->pending_cons
++);
1124 pending_idx
= netbk
->pending_ring
[index
];
1126 memcpy(&pending_tx_info
[pending_idx
].req
, txp
,
1130 pending_tx_info
[pending_idx
].vif
= vif
;
1132 /* Poison these fields, corresponding
1133 * fields for head tx req will be set
1134 * to correct values after the loop.
1136 netbk
->mmap_pages
[pending_idx
] = (void *)(~0UL);
1137 pending_tx_info
[pending_idx
].head
=
1138 INVALID_PENDING_RING_IDX
;
1141 first
= &pending_tx_info
[pending_idx
];
1143 head_idx
= pending_idx
;
1153 first
->req
.offset
= 0;
1154 first
->req
.size
= dst_offset
;
1155 first
->head
= start_idx
;
1156 set_page_ext(page
, netbk
, head_idx
);
1157 netbk
->mmap_pages
[head_idx
] = page
;
1158 frag_set_pending_idx(&frags
[shinfo
->nr_frags
], head_idx
);
1161 BUG_ON(shinfo
->nr_frags
> MAX_SKB_FRAGS
);
1165 /* Unwind, freeing all pages and sending error responses. */
1166 while (shinfo
->nr_frags
-- > start
) {
1167 xen_netbk_idx_release(netbk
,
1168 frag_get_pending_idx(&frags
[shinfo
->nr_frags
]),
1169 XEN_NETIF_RSP_ERROR
);
1171 /* The head too, if necessary. */
1173 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_ERROR
);
1178 static int xen_netbk_tx_check_gop(struct xen_netbk
*netbk
,
1179 struct sk_buff
*skb
,
1180 struct gnttab_copy
**gopp
)
1182 struct gnttab_copy
*gop
= *gopp
;
1183 u16 pending_idx
= *((u16
*)skb
->data
);
1184 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1185 struct pending_tx_info
*tx_info
;
1186 int nr_frags
= shinfo
->nr_frags
;
1188 u16 peek
; /* peek into next tx request */
1190 /* Check status of header. */
1193 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_ERROR
);
1195 /* Skip first skb fragment if it is on same page as header fragment. */
1196 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
1198 for (i
= start
; i
< nr_frags
; i
++) {
1200 pending_ring_idx_t head
;
1202 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[i
]);
1203 tx_info
= &netbk
->pending_tx_info
[pending_idx
];
1204 head
= tx_info
->head
;
1206 /* Check error status: if okay then remember grant handle. */
1208 newerr
= (++gop
)->status
;
1211 peek
= netbk
->pending_ring
[pending_index(++head
)];
1212 } while (!pending_tx_is_head(netbk
, peek
));
1214 if (likely(!newerr
)) {
1215 /* Had a previous error? Invalidate this fragment. */
1217 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1221 /* Error on this fragment: respond to client with an error. */
1222 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_ERROR
);
1224 /* Not the first error? Preceding frags already invalidated. */
1228 /* First error: invalidate header and preceding fragments. */
1229 pending_idx
= *((u16
*)skb
->data
);
1230 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1231 for (j
= start
; j
< i
; j
++) {
1232 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[j
]);
1233 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1236 /* Remember the error: invalidate all subsequent fragments. */
1244 static void xen_netbk_fill_frags(struct xen_netbk
*netbk
, struct sk_buff
*skb
)
1246 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1247 int nr_frags
= shinfo
->nr_frags
;
1250 for (i
= 0; i
< nr_frags
; i
++) {
1251 skb_frag_t
*frag
= shinfo
->frags
+ i
;
1252 struct xen_netif_tx_request
*txp
;
1256 pending_idx
= frag_get_pending_idx(frag
);
1258 txp
= &netbk
->pending_tx_info
[pending_idx
].req
;
1259 page
= virt_to_page(idx_to_kaddr(netbk
, pending_idx
));
1260 __skb_fill_page_desc(skb
, i
, page
, txp
->offset
, txp
->size
);
1261 skb
->len
+= txp
->size
;
1262 skb
->data_len
+= txp
->size
;
1263 skb
->truesize
+= txp
->size
;
1265 /* Take an extra reference to offset xen_netbk_idx_release */
1266 get_page(netbk
->mmap_pages
[pending_idx
]);
1267 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1271 static int xen_netbk_get_extras(struct xenvif
*vif
,
1272 struct xen_netif_extra_info
*extras
,
1275 struct xen_netif_extra_info extra
;
1276 RING_IDX cons
= vif
->tx
.req_cons
;
1279 if (unlikely(work_to_do
-- <= 0)) {
1280 netdev_err(vif
->dev
, "Missing extra info\n");
1281 netbk_fatal_tx_err(vif
);
1285 memcpy(&extra
, RING_GET_REQUEST(&vif
->tx
, cons
),
1287 if (unlikely(!extra
.type
||
1288 extra
.type
>= XEN_NETIF_EXTRA_TYPE_MAX
)) {
1289 vif
->tx
.req_cons
= ++cons
;
1290 netdev_err(vif
->dev
,
1291 "Invalid extra type: %d\n", extra
.type
);
1292 netbk_fatal_tx_err(vif
);
1296 memcpy(&extras
[extra
.type
- 1], &extra
, sizeof(extra
));
1297 vif
->tx
.req_cons
= ++cons
;
1298 } while (extra
.flags
& XEN_NETIF_EXTRA_FLAG_MORE
);
1303 static int netbk_set_skb_gso(struct xenvif
*vif
,
1304 struct sk_buff
*skb
,
1305 struct xen_netif_extra_info
*gso
)
1307 if (!gso
->u
.gso
.size
) {
1308 netdev_err(vif
->dev
, "GSO size must not be zero.\n");
1309 netbk_fatal_tx_err(vif
);
1313 /* Currently only TCPv4 S.O. is supported. */
1314 if (gso
->u
.gso
.type
!= XEN_NETIF_GSO_TYPE_TCPV4
) {
1315 netdev_err(vif
->dev
, "Bad GSO type %d.\n", gso
->u
.gso
.type
);
1316 netbk_fatal_tx_err(vif
);
1320 skb_shinfo(skb
)->gso_size
= gso
->u
.gso
.size
;
1321 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
1323 /* Header must be checked, and gso_segs computed. */
1324 skb_shinfo(skb
)->gso_type
|= SKB_GSO_DODGY
;
1325 skb_shinfo(skb
)->gso_segs
= 0;
1330 static int checksum_setup(struct xenvif
*vif
, struct sk_buff
*skb
)
1334 int recalculate_partial_csum
= 0;
1337 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1338 * peers can fail to set NETRXF_csum_blank when sending a GSO
1339 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1340 * recalculate the partial checksum.
1342 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
&& skb_is_gso(skb
)) {
1343 vif
->rx_gso_checksum_fixup
++;
1344 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1345 recalculate_partial_csum
= 1;
1348 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1349 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1352 if (skb
->protocol
!= htons(ETH_P_IP
))
1355 iph
= (void *)skb
->data
;
1356 switch (iph
->protocol
) {
1358 if (!skb_partial_csum_set(skb
, 4 * iph
->ihl
,
1359 offsetof(struct tcphdr
, check
)))
1362 if (recalculate_partial_csum
) {
1363 struct tcphdr
*tcph
= tcp_hdr(skb
);
1364 tcph
->check
= ~csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
1365 skb
->len
- iph
->ihl
*4,
1370 if (!skb_partial_csum_set(skb
, 4 * iph
->ihl
,
1371 offsetof(struct udphdr
, check
)))
1374 if (recalculate_partial_csum
) {
1375 struct udphdr
*udph
= udp_hdr(skb
);
1376 udph
->check
= ~csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
1377 skb
->len
- iph
->ihl
*4,
1382 if (net_ratelimit())
1383 netdev_err(vif
->dev
,
1384 "Attempting to checksum a non-TCP/UDP packet, dropping a protocol %d packet\n",
1395 static bool tx_credit_exceeded(struct xenvif
*vif
, unsigned size
)
1397 unsigned long now
= jiffies
;
1398 unsigned long next_credit
=
1399 vif
->credit_timeout
.expires
+
1400 msecs_to_jiffies(vif
->credit_usec
/ 1000);
1402 /* Timer could already be pending in rare cases. */
1403 if (timer_pending(&vif
->credit_timeout
))
1406 /* Passed the point where we can replenish credit? */
1407 if (time_after_eq(now
, next_credit
)) {
1408 vif
->credit_timeout
.expires
= now
;
1412 /* Still too big to send right now? Set a callback. */
1413 if (size
> vif
->remaining_credit
) {
1414 vif
->credit_timeout
.data
=
1416 vif
->credit_timeout
.function
=
1418 mod_timer(&vif
->credit_timeout
,
1427 static unsigned xen_netbk_tx_build_gops(struct xen_netbk
*netbk
)
1429 struct gnttab_copy
*gop
= netbk
->tx_copy_ops
, *request_gop
;
1430 struct sk_buff
*skb
;
1433 while ((nr_pending_reqs(netbk
) + XEN_NETBK_LEGACY_SLOTS_MAX
1434 < MAX_PENDING_REQS
) &&
1435 !list_empty(&netbk
->net_schedule_list
)) {
1437 struct xen_netif_tx_request txreq
;
1438 struct xen_netif_tx_request txfrags
[XEN_NETBK_LEGACY_SLOTS_MAX
];
1440 struct xen_netif_extra_info extras
[XEN_NETIF_EXTRA_TYPE_MAX
-1];
1444 unsigned int data_len
;
1445 pending_ring_idx_t index
;
1447 /* Get a netif from the list with work to do. */
1448 vif
= poll_net_schedule_list(netbk
);
1449 /* This can sometimes happen because the test of
1450 * list_empty(net_schedule_list) at the top of the
1451 * loop is unlocked. Just go back and have another
1457 if (vif
->tx
.sring
->req_prod
- vif
->tx
.req_cons
>
1458 XEN_NETIF_TX_RING_SIZE
) {
1459 netdev_err(vif
->dev
,
1460 "Impossible number of requests. "
1461 "req_prod %d, req_cons %d, size %ld\n",
1462 vif
->tx
.sring
->req_prod
, vif
->tx
.req_cons
,
1463 XEN_NETIF_TX_RING_SIZE
);
1464 netbk_fatal_tx_err(vif
);
1468 RING_FINAL_CHECK_FOR_REQUESTS(&vif
->tx
, work_to_do
);
1474 idx
= vif
->tx
.req_cons
;
1475 rmb(); /* Ensure that we see the request before we copy it. */
1476 memcpy(&txreq
, RING_GET_REQUEST(&vif
->tx
, idx
), sizeof(txreq
));
1478 /* Credit-based scheduling. */
1479 if (txreq
.size
> vif
->remaining_credit
&&
1480 tx_credit_exceeded(vif
, txreq
.size
)) {
1485 vif
->remaining_credit
-= txreq
.size
;
1488 vif
->tx
.req_cons
= ++idx
;
1490 memset(extras
, 0, sizeof(extras
));
1491 if (txreq
.flags
& XEN_NETTXF_extra_info
) {
1492 work_to_do
= xen_netbk_get_extras(vif
, extras
,
1494 idx
= vif
->tx
.req_cons
;
1495 if (unlikely(work_to_do
< 0))
1499 ret
= netbk_count_requests(vif
, &txreq
, txfrags
, work_to_do
);
1500 if (unlikely(ret
< 0))
1505 if (unlikely(txreq
.size
< ETH_HLEN
)) {
1506 netdev_dbg(vif
->dev
,
1507 "Bad packet size: %d\n", txreq
.size
);
1508 netbk_tx_err(vif
, &txreq
, idx
);
1512 /* No crossing a page as the payload mustn't fragment. */
1513 if (unlikely((txreq
.offset
+ txreq
.size
) > PAGE_SIZE
)) {
1514 netdev_err(vif
->dev
,
1515 "txreq.offset: %x, size: %u, end: %lu\n",
1516 txreq
.offset
, txreq
.size
,
1517 (txreq
.offset
&~PAGE_MASK
) + txreq
.size
);
1518 netbk_fatal_tx_err(vif
);
1522 index
= pending_index(netbk
->pending_cons
);
1523 pending_idx
= netbk
->pending_ring
[index
];
1525 data_len
= (txreq
.size
> PKT_PROT_LEN
&&
1526 ret
< XEN_NETBK_LEGACY_SLOTS_MAX
) ?
1527 PKT_PROT_LEN
: txreq
.size
;
1529 skb
= alloc_skb(data_len
+ NET_SKB_PAD
+ NET_IP_ALIGN
,
1530 GFP_ATOMIC
| __GFP_NOWARN
);
1531 if (unlikely(skb
== NULL
)) {
1532 netdev_dbg(vif
->dev
,
1533 "Can't allocate a skb in start_xmit.\n");
1534 netbk_tx_err(vif
, &txreq
, idx
);
1538 /* Packets passed to netif_rx() must have some headroom. */
1539 skb_reserve(skb
, NET_SKB_PAD
+ NET_IP_ALIGN
);
1541 if (extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1].type
) {
1542 struct xen_netif_extra_info
*gso
;
1543 gso
= &extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1];
1545 if (netbk_set_skb_gso(vif
, skb
, gso
)) {
1546 /* Failure in netbk_set_skb_gso is fatal. */
1552 /* XXX could copy straight to head */
1553 page
= xen_netbk_alloc_page(netbk
, pending_idx
);
1556 netbk_tx_err(vif
, &txreq
, idx
);
1560 gop
->source
.u
.ref
= txreq
.gref
;
1561 gop
->source
.domid
= vif
->domid
;
1562 gop
->source
.offset
= txreq
.offset
;
1564 gop
->dest
.u
.gmfn
= virt_to_mfn(page_address(page
));
1565 gop
->dest
.domid
= DOMID_SELF
;
1566 gop
->dest
.offset
= txreq
.offset
;
1568 gop
->len
= txreq
.size
;
1569 gop
->flags
= GNTCOPY_source_gref
;
1573 memcpy(&netbk
->pending_tx_info
[pending_idx
].req
,
1574 &txreq
, sizeof(txreq
));
1575 netbk
->pending_tx_info
[pending_idx
].vif
= vif
;
1576 netbk
->pending_tx_info
[pending_idx
].head
= index
;
1577 *((u16
*)skb
->data
) = pending_idx
;
1579 __skb_put(skb
, data_len
);
1581 skb_shinfo(skb
)->nr_frags
= ret
;
1582 if (data_len
< txreq
.size
) {
1583 skb_shinfo(skb
)->nr_frags
++;
1584 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1587 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1588 INVALID_PENDING_IDX
);
1591 netbk
->pending_cons
++;
1593 request_gop
= xen_netbk_get_requests(netbk
, vif
,
1595 if (request_gop
== NULL
) {
1597 netbk_tx_err(vif
, &txreq
, idx
);
1602 __skb_queue_tail(&netbk
->tx_queue
, skb
);
1604 vif
->tx
.req_cons
= idx
;
1605 xen_netbk_check_rx_xenvif(vif
);
1607 if ((gop
-netbk
->tx_copy_ops
) >= ARRAY_SIZE(netbk
->tx_copy_ops
))
1611 return gop
- netbk
->tx_copy_ops
;
1614 static void xen_netbk_tx_submit(struct xen_netbk
*netbk
)
1616 struct gnttab_copy
*gop
= netbk
->tx_copy_ops
;
1617 struct sk_buff
*skb
;
1619 while ((skb
= __skb_dequeue(&netbk
->tx_queue
)) != NULL
) {
1620 struct xen_netif_tx_request
*txp
;
1625 pending_idx
= *((u16
*)skb
->data
);
1626 vif
= netbk
->pending_tx_info
[pending_idx
].vif
;
1627 txp
= &netbk
->pending_tx_info
[pending_idx
].req
;
1629 /* Check the remap error code. */
1630 if (unlikely(xen_netbk_tx_check_gop(netbk
, skb
, &gop
))) {
1631 netdev_dbg(vif
->dev
, "netback grant failed.\n");
1632 skb_shinfo(skb
)->nr_frags
= 0;
1637 data_len
= skb
->len
;
1639 (void *)(idx_to_kaddr(netbk
, pending_idx
)|txp
->offset
),
1641 if (data_len
< txp
->size
) {
1642 /* Append the packet payload as a fragment. */
1643 txp
->offset
+= data_len
;
1644 txp
->size
-= data_len
;
1646 /* Schedule a response immediately. */
1647 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1650 if (txp
->flags
& XEN_NETTXF_csum_blank
)
1651 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1652 else if (txp
->flags
& XEN_NETTXF_data_validated
)
1653 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1655 xen_netbk_fill_frags(netbk
, skb
);
1658 * If the initial fragment was < PKT_PROT_LEN then
1659 * pull through some bytes from the other fragments to
1660 * increase the linear region to PKT_PROT_LEN bytes.
1662 if (skb_headlen(skb
) < PKT_PROT_LEN
&& skb_is_nonlinear(skb
)) {
1663 int target
= min_t(int, skb
->len
, PKT_PROT_LEN
);
1664 __pskb_pull_tail(skb
, target
- skb_headlen(skb
));
1667 skb
->dev
= vif
->dev
;
1668 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1669 skb_reset_network_header(skb
);
1671 if (checksum_setup(vif
, skb
)) {
1672 netdev_dbg(vif
->dev
,
1673 "Can't setup checksum in net_tx_action\n");
1678 skb_probe_transport_header(skb
, 0);
1680 vif
->dev
->stats
.rx_bytes
+= skb
->len
;
1681 vif
->dev
->stats
.rx_packets
++;
1683 xenvif_receive_skb(vif
, skb
);
1687 /* Called after netfront has transmitted */
1688 static void xen_netbk_tx_action(struct xen_netbk
*netbk
)
1692 nr_gops
= xen_netbk_tx_build_gops(netbk
);
1697 gnttab_batch_copy(netbk
->tx_copy_ops
, nr_gops
);
1699 xen_netbk_tx_submit(netbk
);
1702 static void xen_netbk_idx_release(struct xen_netbk
*netbk
, u16 pending_idx
,
1706 struct pending_tx_info
*pending_tx_info
;
1707 pending_ring_idx_t head
;
1708 u16 peek
; /* peek into next tx request */
1710 BUG_ON(netbk
->mmap_pages
[pending_idx
] == (void *)(~0UL));
1712 /* Already complete? */
1713 if (netbk
->mmap_pages
[pending_idx
] == NULL
)
1716 pending_tx_info
= &netbk
->pending_tx_info
[pending_idx
];
1718 vif
= pending_tx_info
->vif
;
1719 head
= pending_tx_info
->head
;
1721 BUG_ON(!pending_tx_is_head(netbk
, head
));
1722 BUG_ON(netbk
->pending_ring
[pending_index(head
)] != pending_idx
);
1725 pending_ring_idx_t index
;
1726 pending_ring_idx_t idx
= pending_index(head
);
1727 u16 info_idx
= netbk
->pending_ring
[idx
];
1729 pending_tx_info
= &netbk
->pending_tx_info
[info_idx
];
1730 make_tx_response(vif
, &pending_tx_info
->req
, status
);
1732 /* Setting any number other than
1733 * INVALID_PENDING_RING_IDX indicates this slot is
1734 * starting a new packet / ending a previous packet.
1736 pending_tx_info
->head
= 0;
1738 index
= pending_index(netbk
->pending_prod
++);
1739 netbk
->pending_ring
[index
] = netbk
->pending_ring
[info_idx
];
1743 peek
= netbk
->pending_ring
[pending_index(++head
)];
1745 } while (!pending_tx_is_head(netbk
, peek
));
1747 netbk
->mmap_pages
[pending_idx
]->mapping
= 0;
1748 put_page(netbk
->mmap_pages
[pending_idx
]);
1749 netbk
->mmap_pages
[pending_idx
] = NULL
;
1753 static void make_tx_response(struct xenvif
*vif
,
1754 struct xen_netif_tx_request
*txp
,
1757 RING_IDX i
= vif
->tx
.rsp_prod_pvt
;
1758 struct xen_netif_tx_response
*resp
;
1761 resp
= RING_GET_RESPONSE(&vif
->tx
, i
);
1765 if (txp
->flags
& XEN_NETTXF_extra_info
)
1766 RING_GET_RESPONSE(&vif
->tx
, ++i
)->status
= XEN_NETIF_RSP_NULL
;
1768 vif
->tx
.rsp_prod_pvt
= ++i
;
1769 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif
->tx
, notify
);
1771 notify_remote_via_irq(vif
->tx_irq
);
1774 static struct xen_netif_rx_response
*make_rx_response(struct xenvif
*vif
,
1781 RING_IDX i
= vif
->rx
.rsp_prod_pvt
;
1782 struct xen_netif_rx_response
*resp
;
1784 resp
= RING_GET_RESPONSE(&vif
->rx
, i
);
1785 resp
->offset
= offset
;
1786 resp
->flags
= flags
;
1788 resp
->status
= (s16
)size
;
1790 resp
->status
= (s16
)st
;
1792 vif
->rx
.rsp_prod_pvt
= ++i
;
1797 static inline int rx_work_todo(struct xen_netbk
*netbk
)
1799 return !skb_queue_empty(&netbk
->rx_queue
);
1802 static inline int tx_work_todo(struct xen_netbk
*netbk
)
1805 if ((nr_pending_reqs(netbk
) + XEN_NETBK_LEGACY_SLOTS_MAX
1806 < MAX_PENDING_REQS
) &&
1807 !list_empty(&netbk
->net_schedule_list
))
1813 static int xen_netbk_kthread(void *data
)
1815 struct xen_netbk
*netbk
= data
;
1816 while (!kthread_should_stop()) {
1817 wait_event_interruptible(netbk
->wq
,
1818 rx_work_todo(netbk
) ||
1819 tx_work_todo(netbk
) ||
1820 kthread_should_stop());
1823 if (kthread_should_stop())
1826 if (rx_work_todo(netbk
))
1827 xen_netbk_rx_action(netbk
);
1829 if (tx_work_todo(netbk
))
1830 xen_netbk_tx_action(netbk
);
1836 void xen_netbk_unmap_frontend_rings(struct xenvif
*vif
)
1839 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif
),
1842 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif
),
1846 int xen_netbk_map_frontend_rings(struct xenvif
*vif
,
1847 grant_ref_t tx_ring_ref
,
1848 grant_ref_t rx_ring_ref
)
1851 struct xen_netif_tx_sring
*txs
;
1852 struct xen_netif_rx_sring
*rxs
;
1856 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif
),
1857 tx_ring_ref
, &addr
);
1861 txs
= (struct xen_netif_tx_sring
*)addr
;
1862 BACK_RING_INIT(&vif
->tx
, txs
, PAGE_SIZE
);
1864 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif
),
1865 rx_ring_ref
, &addr
);
1869 rxs
= (struct xen_netif_rx_sring
*)addr
;
1870 BACK_RING_INIT(&vif
->rx
, rxs
, PAGE_SIZE
);
1872 vif
->rx_req_cons_peek
= 0;
1877 xen_netbk_unmap_frontend_rings(vif
);
1881 static int __init
netback_init(void)
1890 if (fatal_skb_slots
< XEN_NETBK_LEGACY_SLOTS_MAX
) {
1892 "xen-netback: fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n",
1893 fatal_skb_slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
1894 fatal_skb_slots
= XEN_NETBK_LEGACY_SLOTS_MAX
;
1897 xen_netbk_group_nr
= num_online_cpus();
1898 xen_netbk
= vzalloc(sizeof(struct xen_netbk
) * xen_netbk_group_nr
);
1902 for (group
= 0; group
< xen_netbk_group_nr
; group
++) {
1903 struct xen_netbk
*netbk
= &xen_netbk
[group
];
1904 skb_queue_head_init(&netbk
->rx_queue
);
1905 skb_queue_head_init(&netbk
->tx_queue
);
1907 init_timer(&netbk
->net_timer
);
1908 netbk
->net_timer
.data
= (unsigned long)netbk
;
1909 netbk
->net_timer
.function
= xen_netbk_alarm
;
1911 netbk
->pending_cons
= 0;
1912 netbk
->pending_prod
= MAX_PENDING_REQS
;
1913 for (i
= 0; i
< MAX_PENDING_REQS
; i
++)
1914 netbk
->pending_ring
[i
] = i
;
1916 init_waitqueue_head(&netbk
->wq
);
1917 netbk
->task
= kthread_create(xen_netbk_kthread
,
1919 "netback/%u", group
);
1921 if (IS_ERR(netbk
->task
)) {
1922 printk(KERN_ALERT
"kthread_create() fails at netback\n");
1923 del_timer(&netbk
->net_timer
);
1924 rc
= PTR_ERR(netbk
->task
);
1928 kthread_bind(netbk
->task
, group
);
1930 INIT_LIST_HEAD(&netbk
->net_schedule_list
);
1932 spin_lock_init(&netbk
->net_schedule_list_lock
);
1934 atomic_set(&netbk
->netfront_count
, 0);
1936 wake_up_process(netbk
->task
);
1939 rc
= xenvif_xenbus_init();
1946 while (--group
>= 0) {
1947 struct xen_netbk
*netbk
= &xen_netbk
[group
];
1948 del_timer(&netbk
->net_timer
);
1949 kthread_stop(netbk
->task
);
1956 module_init(netback_init
);
1958 static void __exit
netback_fini(void)
1962 xenvif_xenbus_fini();
1964 for (i
= 0; i
< xen_netbk_group_nr
; i
++) {
1965 struct xen_netbk
*netbk
= &xen_netbk
[i
];
1966 del_timer_sync(&netbk
->net_timer
);
1967 kthread_stop(netbk
->task
);
1968 for (j
= 0; j
< MAX_PENDING_REQS
; j
++) {
1969 if (netbk
->mmap_pages
[i
])
1970 __free_page(netbk
->mmap_pages
[i
]);
1976 module_exit(netback_fini
);
1978 MODULE_LICENSE("Dual BSD/GPL");
1979 MODULE_ALIAS("xen-backend:vif");