]>
Commit | Line | Data |
---|---|---|
06021292 ET |
1 | /* |
2 | * Low Latency Sockets | |
3 | * Copyright(c) 2013 Intel Corporation. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program; if not, write to the Free Software Foundation, Inc., | |
16 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
17 | * | |
18 | * Author: Eliezer Tamir | |
19 | * | |
20 | * Contact Information: | |
21 | * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> | |
22 | */ | |
23 | ||
06021292 ET |
24 | #ifndef _LINUX_NET_LL_POLL_H |
25 | #define _LINUX_NET_LL_POLL_H | |
26 | ||
27 | #include <linux/netdevice.h> | |
28 | #include <net/ip.h> | |
29 | ||
30 | #ifdef CONFIG_NET_LL_RX_POLL | |
31 | ||
32 | struct napi_struct; | |
2d48d67f | 33 | extern unsigned int sysctl_net_ll_read __read_mostly; |
eb6db622 | 34 | extern unsigned int sysctl_net_ll_poll __read_mostly; |
06021292 ET |
35 | |
36 | /* return values from ndo_ll_poll */ | |
37 | #define LL_FLUSH_FAILED -1 | |
38 | #define LL_FLUSH_BUSY -2 | |
39 | ||
cbf55001 | 40 | static inline bool net_busy_loop_on(void) |
91e2fd33 | 41 | { |
cbf55001 | 42 | return sysctl_net_ll_poll; |
91e2fd33 ET |
43 | } |
44 | ||
ad6276e0 ET |
45 | /* a wrapper to make debug_smp_processor_id() happy |
46 | * we can use sched_clock() because we don't care much about precision | |
9a3c71aa | 47 | * we only care that the average is bounded |
ad6276e0 ET |
48 | */ |
49 | #ifdef CONFIG_DEBUG_PREEMPT | |
cbf55001 | 50 | static inline u64 busy_loop_sched_clock(void) |
ad6276e0 ET |
51 | { |
52 | u64 rc; | |
53 | ||
54 | preempt_disable_notrace(); | |
55 | rc = sched_clock(); | |
56 | preempt_enable_no_resched_notrace(); | |
57 | ||
58 | return rc; | |
59 | } | |
60 | #else /* CONFIG_DEBUG_PREEMPT */ | |
cbf55001 | 61 | static inline u64 busy_loop_sched_clock(void) |
ad6276e0 ET |
62 | { |
63 | return sched_clock(); | |
64 | } | |
65 | #endif /* CONFIG_DEBUG_PREEMPT */ | |
66 | ||
67 | /* we don't mind a ~2.5% imprecision so <<10 instead of *1000 | |
2d48d67f | 68 | * sk->sk_ll_usec is a u_int so this can't overflow |
9a3c71aa | 69 | */ |
cbf55001 | 70 | static inline u64 sk_busy_loop_end_time(struct sock *sk) |
06021292 | 71 | { |
1bc2774d | 72 | return (u64)ACCESS_ONCE(sk->sk_ll_usec) << 10; |
2d48d67f | 73 | } |
9a3c71aa | 74 | |
91e2fd33 ET |
75 | /* in poll/select we use the global sysctl_net_ll_poll value |
76 | * only call sched_clock() if enabled | |
77 | */ | |
cbf55001 | 78 | static inline u64 busy_loop_end_time(void) |
2d48d67f | 79 | { |
1bc2774d ET |
80 | return (u64)ACCESS_ONCE(sysctl_net_ll_poll) << 10; |
81 | } | |
91e2fd33 | 82 | |
cbf55001 ET |
83 | /* if flag is not set we don't need to know the time |
84 | * so we want to avoid a potentially expensive sched_clock() | |
85 | */ | |
86 | static inline u64 busy_loop_start_time(unsigned int flag) | |
1bc2774d | 87 | { |
cbf55001 | 88 | return flag ? busy_loop_sched_clock() : 0; |
06021292 ET |
89 | } |
90 | ||
cbf55001 | 91 | static inline bool sk_can_busy_loop(struct sock *sk) |
06021292 | 92 | { |
dafcc438 | 93 | return sk->sk_ll_usec && sk->sk_napi_id && |
06021292 ET |
94 | !need_resched() && !signal_pending(current); |
95 | } | |
96 | ||
1bc2774d | 97 | /* careful! time_in_range64 will evaluate now twice */ |
cbf55001 | 98 | static inline bool busy_loop_range(u64 start_time, u64 run_time) |
06021292 | 99 | { |
cbf55001 | 100 | u64 now = busy_loop_sched_clock(); |
1bc2774d ET |
101 | |
102 | return time_in_range64(now, start_time, start_time + run_time); | |
06021292 ET |
103 | } |
104 | ||
2d48d67f ET |
105 | /* when used in sock_poll() nonblock is known at compile time to be true |
106 | * so the loop and end_time will be optimized out | |
107 | */ | |
cbf55001 | 108 | static inline bool sk_busy_loop(struct sock *sk, int nonblock) |
06021292 | 109 | { |
cbf55001 ET |
110 | u64 start_time = busy_loop_start_time(!nonblock); |
111 | u64 end_time = sk_busy_loop_end_time(sk); | |
06021292 ET |
112 | const struct net_device_ops *ops; |
113 | struct napi_struct *napi; | |
114 | int rc = false; | |
115 | ||
116 | /* | |
117 | * rcu read lock for napi hash | |
118 | * bh so we don't race with net_rx_action | |
119 | */ | |
120 | rcu_read_lock_bh(); | |
121 | ||
122 | napi = napi_by_id(sk->sk_napi_id); | |
123 | if (!napi) | |
124 | goto out; | |
125 | ||
126 | ops = napi->dev->netdev_ops; | |
127 | if (!ops->ndo_ll_poll) | |
128 | goto out; | |
129 | ||
130 | do { | |
06021292 ET |
131 | rc = ops->ndo_ll_poll(napi); |
132 | ||
133 | if (rc == LL_FLUSH_FAILED) | |
134 | break; /* permanent failure */ | |
135 | ||
136 | if (rc > 0) | |
137 | /* local bh are disabled so it is ok to use _BH */ | |
138 | NET_ADD_STATS_BH(sock_net(sk), | |
139 | LINUX_MIB_LOWLATENCYRXPACKETS, rc); | |
140 | ||
2d48d67f | 141 | } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) && |
cbf55001 | 142 | busy_loop_range(start_time, end_time)); |
06021292 ET |
143 | |
144 | rc = !skb_queue_empty(&sk->sk_receive_queue); | |
145 | out: | |
146 | rcu_read_unlock_bh(); | |
147 | return rc; | |
148 | } | |
149 | ||
150 | /* used in the NIC receive handler to mark the skb */ | |
151 | static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) | |
152 | { | |
153 | skb->napi_id = napi->napi_id; | |
154 | } | |
155 | ||
156 | /* used in the protocol hanlder to propagate the napi_id to the socket */ | |
157 | static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) | |
158 | { | |
159 | sk->sk_napi_id = skb->napi_id; | |
160 | } | |
161 | ||
162 | #else /* CONFIG_NET_LL_RX_POLL */ | |
cbf55001 | 163 | static inline unsigned long net_busy_loop_on(void) |
91e2fd33 ET |
164 | { |
165 | return 0; | |
166 | } | |
06021292 | 167 | |
cbf55001 | 168 | static inline u64 busy_loop_start_time(unsigned int flag) |
2d48d67f ET |
169 | { |
170 | return 0; | |
171 | } | |
172 | ||
cbf55001 | 173 | static inline u64 busy_loop_end_time(void) |
06021292 ET |
174 | { |
175 | return 0; | |
176 | } | |
177 | ||
cbf55001 | 178 | static inline bool sk_can_busy_loop(struct sock *sk) |
06021292 ET |
179 | { |
180 | return false; | |
181 | } | |
182 | ||
cbf55001 | 183 | static inline bool sk_busy_poll(struct sock *sk, int nonblock) |
06021292 ET |
184 | { |
185 | return false; | |
186 | } | |
187 | ||
188 | static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) | |
189 | { | |
190 | } | |
191 | ||
192 | static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) | |
193 | { | |
194 | } | |
195 | ||
cbf55001 | 196 | static inline bool busy_loop_range(u64 start_time, u64 run_time) |
06021292 ET |
197 | { |
198 | return false; | |
199 | } | |
200 | ||
201 | #endif /* CONFIG_NET_LL_RX_POLL */ | |
202 | #endif /* _LINUX_NET_LL_POLL_H */ |