]>
Commit | Line | Data |
---|---|---|
c9e6bc64 ED |
1 | #ifndef _NET_GRO_CELLS_H |
2 | #define _NET_GRO_CELLS_H | |
3 | ||
4 | #include <linux/skbuff.h> | |
5 | #include <linux/slab.h> | |
6 | #include <linux/netdevice.h> | |
7 | ||
8 | struct gro_cell { | |
9 | struct sk_buff_head napi_skbs; | |
10 | struct napi_struct napi; | |
11 | } ____cacheline_aligned_in_smp; | |
12 | ||
13 | struct gro_cells { | |
14 | unsigned int gro_cells_mask; | |
15 | struct gro_cell *cells; | |
16 | }; | |
17 | ||
18 | static inline void gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) | |
19 | { | |
20 | unsigned long flags; | |
21 | struct gro_cell *cell = gcells->cells; | |
22 | struct net_device *dev = skb->dev; | |
23 | ||
24 | if (!cell || skb_cloned(skb) || !(dev->features & NETIF_F_GRO)) { | |
25 | netif_rx(skb); | |
26 | return; | |
27 | } | |
28 | ||
29 | if (skb_rx_queue_recorded(skb)) | |
30 | cell += skb_get_rx_queue(skb) & gcells->gro_cells_mask; | |
31 | ||
32 | if (skb_queue_len(&cell->napi_skbs) > netdev_max_backlog) { | |
33 | atomic_long_inc(&dev->rx_dropped); | |
34 | kfree_skb(skb); | |
35 | return; | |
36 | } | |
37 | ||
38 | spin_lock_irqsave(&cell->napi_skbs.lock, flags); | |
39 | ||
40 | __skb_queue_tail(&cell->napi_skbs, skb); | |
41 | if (skb_queue_len(&cell->napi_skbs) == 1) | |
42 | napi_schedule(&cell->napi); | |
43 | ||
44 | spin_unlock_irqrestore(&cell->napi_skbs.lock, flags); | |
45 | } | |
46 | ||
47 | static inline int gro_cell_poll(struct napi_struct *napi, int budget) | |
48 | { | |
49 | struct gro_cell *cell = container_of(napi, struct gro_cell, napi); | |
50 | struct sk_buff *skb; | |
51 | int work_done = 0; | |
52 | ||
53 | while (work_done < budget) { | |
54 | skb = skb_dequeue(&cell->napi_skbs); | |
55 | if (!skb) | |
56 | break; | |
57 | ||
58 | napi_gro_receive(napi, skb); | |
59 | work_done++; | |
60 | } | |
61 | ||
62 | if (work_done < budget) | |
63 | napi_complete(napi); | |
64 | return work_done; | |
65 | } | |
66 | ||
67 | static inline int gro_cells_init(struct gro_cells *gcells, struct net_device *dev) | |
68 | { | |
69 | int i; | |
70 | ||
71 | gcells->gro_cells_mask = roundup_pow_of_two(netif_get_num_default_rss_queues()) - 1; | |
72 | gcells->cells = kcalloc(sizeof(struct gro_cell), | |
73 | gcells->gro_cells_mask + 1, | |
74 | GFP_KERNEL); | |
75 | if (!gcells->cells) | |
76 | return -ENOMEM; | |
77 | ||
78 | for (i = 0; i <= gcells->gro_cells_mask; i++) { | |
79 | struct gro_cell *cell = gcells->cells + i; | |
80 | ||
81 | skb_queue_head_init(&cell->napi_skbs); | |
82 | netif_napi_add(dev, &cell->napi, gro_cell_poll, 64); | |
83 | napi_enable(&cell->napi); | |
84 | } | |
85 | return 0; | |
86 | } | |
87 | ||
88 | static inline void gro_cells_destroy(struct gro_cells *gcells) | |
89 | { | |
90 | struct gro_cell *cell = gcells->cells; | |
91 | int i; | |
92 | ||
93 | if (!cell) | |
94 | return; | |
95 | for (i = 0; i <= gcells->gro_cells_mask; i++,cell++) { | |
96 | netif_napi_del(&cell->napi); | |
97 | skb_queue_purge(&cell->napi_skbs); | |
98 | } | |
99 | kfree(gcells->cells); | |
100 | gcells->cells = NULL; | |
101 | } | |
102 | ||
103 | #endif |