]>
Commit | Line | Data |
---|---|---|
06021292 ET |
1 | /* |
2 | * Low Latency Sockets | |
3 | * Copyright(c) 2013 Intel Corporation. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program; if not, write to the Free Software Foundation, Inc., | |
16 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
17 | * | |
18 | * Author: Eliezer Tamir | |
19 | * | |
20 | * Contact Information: | |
21 | * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> | |
22 | */ | |
23 | ||
06021292 ET |
24 | #ifndef _LINUX_NET_LL_POLL_H |
25 | #define _LINUX_NET_LL_POLL_H | |
26 | ||
27 | #include <linux/netdevice.h> | |
28 | #include <net/ip.h> | |
29 | ||
30 | #ifdef CONFIG_NET_LL_RX_POLL | |
31 | ||
32 | struct napi_struct; | |
2d48d67f | 33 | extern unsigned int sysctl_net_ll_read __read_mostly; |
eb6db622 | 34 | extern unsigned int sysctl_net_ll_poll __read_mostly; |
06021292 ET |
35 | |
36 | /* return values from ndo_ll_poll */ | |
37 | #define LL_FLUSH_FAILED -1 | |
38 | #define LL_FLUSH_BUSY -2 | |
39 | ||
91e2fd33 ET |
40 | static inline unsigned int ll_get_flag(void) |
41 | { | |
42 | return sysctl_net_ll_poll ? POLL_LL : 0; | |
43 | } | |
44 | ||
ad6276e0 ET |
45 | /* a wrapper to make debug_smp_processor_id() happy |
46 | * we can use sched_clock() because we don't care much about precision | |
9a3c71aa | 47 | * we only care that the average is bounded |
ad6276e0 ET |
48 | */ |
49 | #ifdef CONFIG_DEBUG_PREEMPT | |
50 | static inline u64 ll_sched_clock(void) | |
51 | { | |
52 | u64 rc; | |
53 | ||
54 | preempt_disable_notrace(); | |
55 | rc = sched_clock(); | |
56 | preempt_enable_no_resched_notrace(); | |
57 | ||
58 | return rc; | |
59 | } | |
60 | #else /* CONFIG_DEBUG_PREEMPT */ | |
61 | static inline u64 ll_sched_clock(void) | |
62 | { | |
63 | return sched_clock(); | |
64 | } | |
65 | #endif /* CONFIG_DEBUG_PREEMPT */ | |
66 | ||
67 | /* we don't mind a ~2.5% imprecision so <<10 instead of *1000 | |
2d48d67f | 68 | * sk->sk_ll_usec is a u_int so this can't overflow |
9a3c71aa | 69 | */ |
1bc2774d | 70 | static inline u64 ll_sk_run_time(struct sock *sk) |
06021292 | 71 | { |
1bc2774d | 72 | return (u64)ACCESS_ONCE(sk->sk_ll_usec) << 10; |
2d48d67f | 73 | } |
9a3c71aa | 74 | |
91e2fd33 ET |
75 | /* in poll/select we use the global sysctl_net_ll_poll value |
76 | * only call sched_clock() if enabled | |
77 | */ | |
1bc2774d | 78 | static inline u64 ll_run_time(void) |
2d48d67f | 79 | { |
1bc2774d ET |
80 | return (u64)ACCESS_ONCE(sysctl_net_ll_poll) << 10; |
81 | } | |
91e2fd33 | 82 | |
1bc2774d ET |
83 | /* if flag is not set we don't need to know the time */ |
84 | static inline u64 ll_start_time(unsigned int flag) | |
85 | { | |
86 | return flag ? ll_sched_clock() : 0; | |
06021292 ET |
87 | } |
88 | ||
89 | static inline bool sk_valid_ll(struct sock *sk) | |
90 | { | |
dafcc438 | 91 | return sk->sk_ll_usec && sk->sk_napi_id && |
06021292 ET |
92 | !need_resched() && !signal_pending(current); |
93 | } | |
94 | ||
1bc2774d ET |
95 | /* careful! time_in_range64 will evaluate now twice */ |
96 | static inline bool can_poll_ll(u64 start_time, u64 run_time) | |
06021292 | 97 | { |
1bc2774d ET |
98 | u64 now = ll_sched_clock(); |
99 | ||
100 | return time_in_range64(now, start_time, start_time + run_time); | |
06021292 ET |
101 | } |
102 | ||
2d48d67f ET |
103 | /* when used in sock_poll() nonblock is known at compile time to be true |
104 | * so the loop and end_time will be optimized out | |
105 | */ | |
06021292 ET |
106 | static inline bool sk_poll_ll(struct sock *sk, int nonblock) |
107 | { | |
1bc2774d ET |
108 | u64 start_time = ll_start_time(!nonblock); |
109 | u64 run_time = ll_sk_run_time(sk); | |
06021292 ET |
110 | const struct net_device_ops *ops; |
111 | struct napi_struct *napi; | |
112 | int rc = false; | |
113 | ||
114 | /* | |
115 | * rcu read lock for napi hash | |
116 | * bh so we don't race with net_rx_action | |
117 | */ | |
118 | rcu_read_lock_bh(); | |
119 | ||
120 | napi = napi_by_id(sk->sk_napi_id); | |
121 | if (!napi) | |
122 | goto out; | |
123 | ||
124 | ops = napi->dev->netdev_ops; | |
125 | if (!ops->ndo_ll_poll) | |
126 | goto out; | |
127 | ||
128 | do { | |
06021292 ET |
129 | rc = ops->ndo_ll_poll(napi); |
130 | ||
131 | if (rc == LL_FLUSH_FAILED) | |
132 | break; /* permanent failure */ | |
133 | ||
134 | if (rc > 0) | |
135 | /* local bh are disabled so it is ok to use _BH */ | |
136 | NET_ADD_STATS_BH(sock_net(sk), | |
137 | LINUX_MIB_LOWLATENCYRXPACKETS, rc); | |
138 | ||
2d48d67f | 139 | } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) && |
1bc2774d | 140 | can_poll_ll(start_time, run_time)); |
06021292 ET |
141 | |
142 | rc = !skb_queue_empty(&sk->sk_receive_queue); | |
143 | out: | |
144 | rcu_read_unlock_bh(); | |
145 | return rc; | |
146 | } | |
147 | ||
148 | /* used in the NIC receive handler to mark the skb */ | |
149 | static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) | |
150 | { | |
151 | skb->napi_id = napi->napi_id; | |
152 | } | |
153 | ||
154 | /* used in the protocol hanlder to propagate the napi_id to the socket */ | |
155 | static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) | |
156 | { | |
157 | sk->sk_napi_id = skb->napi_id; | |
158 | } | |
159 | ||
160 | #else /* CONFIG_NET_LL_RX_POLL */ | |
91e2fd33 ET |
161 | static inline unsigned long ll_get_flag(void) |
162 | { | |
163 | return 0; | |
164 | } | |
06021292 | 165 | |
419076f5 | 166 | static inline u64 ll_start_time(unsigned int flag) |
2d48d67f ET |
167 | { |
168 | return 0; | |
169 | } | |
170 | ||
419076f5 | 171 | static inline u64 ll_run_time(void) |
06021292 ET |
172 | { |
173 | return 0; | |
174 | } | |
175 | ||
176 | static inline bool sk_valid_ll(struct sock *sk) | |
177 | { | |
178 | return false; | |
179 | } | |
180 | ||
181 | static inline bool sk_poll_ll(struct sock *sk, int nonblock) | |
182 | { | |
183 | return false; | |
184 | } | |
185 | ||
186 | static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) | |
187 | { | |
188 | } | |
189 | ||
190 | static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) | |
191 | { | |
192 | } | |
193 | ||
419076f5 | 194 | static inline bool can_poll_ll(u64 start_time, u64 run_time) |
06021292 ET |
195 | { |
196 | return false; | |
197 | } | |
198 | ||
199 | #endif /* CONFIG_NET_LL_RX_POLL */ | |
200 | #endif /* _LINUX_NET_LL_POLL_H */ |