]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * net/core/dst.c Protocol independent destination cache. | |
3 | * | |
4 | * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> | |
5 | * | |
6 | */ | |
7 | ||
8 | #include <linux/bitops.h> | |
9 | #include <linux/errno.h> | |
10 | #include <linux/init.h> | |
11 | #include <linux/kernel.h> | |
86bba269 | 12 | #include <linux/workqueue.h> |
1da177e4 LT |
13 | #include <linux/mm.h> |
14 | #include <linux/module.h> | |
5a0e3ad6 | 15 | #include <linux/slab.h> |
1da177e4 | 16 | #include <linux/netdevice.h> |
1da177e4 LT |
17 | #include <linux/skbuff.h> |
18 | #include <linux/string.h> | |
19 | #include <linux/types.h> | |
e9dc8653 | 20 | #include <net/net_namespace.h> |
2fc1b5dd | 21 | #include <linux/sched.h> |
1da177e4 LT |
22 | |
23 | #include <net/dst.h> | |
24 | ||
86bba269 ED |
25 | /* |
26 | * Theory of operations: | |
27 | * 1) We use a list, protected by a spinlock, to add | |
28 | * new entries from both BH and non-BH context. | |
29 | * 2) In order to keep spinlock held for a small delay, | |
30 | * we use a second list where are stored long lived | |
31 | * entries, that are handled by the garbage collect thread | |
32 | * fired by a workqueue. | |
33 | * 3) This list is guarded by a mutex, | |
34 | * so that the gc_task and dst_dev_event() can be synchronized. | |
1da177e4 | 35 | */ |
4ec93edb | 36 | #if RT_CACHE_DEBUG >= 2 |
1da177e4 LT |
37 | static atomic_t dst_total = ATOMIC_INIT(0); |
38 | #endif | |
1da177e4 | 39 | |
86bba269 ED |
40 | /* |
41 | * We want to keep lock & list close together | |
42 | * to dirty as few cache lines as possible in __dst_free(). | |
43 | * As this is not a very strong hint, we dont force an alignment on SMP. | |
44 | */ | |
45 | static struct { | |
46 | spinlock_t lock; | |
598ed936 | 47 | struct dst_entry *list; |
86bba269 ED |
48 | unsigned long timer_inc; |
49 | unsigned long timer_expires; | |
50 | } dst_garbage = { | |
51 | .lock = __SPIN_LOCK_UNLOCKED(dst_garbage.lock), | |
52 | .timer_inc = DST_GC_MAX, | |
53 | }; | |
54 | static void dst_gc_task(struct work_struct *work); | |
598ed936 | 55 | static void ___dst_free(struct dst_entry *dst); |
1da177e4 | 56 | |
86bba269 | 57 | static DECLARE_DELAYED_WORK(dst_gc_work, dst_gc_task); |
1da177e4 | 58 | |
86bba269 ED |
59 | static DEFINE_MUTEX(dst_gc_mutex); |
60 | /* | |
61 | * long lived entries are maintained in this list, guarded by dst_gc_mutex | |
62 | */ | |
63 | static struct dst_entry *dst_busy_list; | |
64 | ||
65 | static void dst_gc_task(struct work_struct *work) | |
1da177e4 LT |
66 | { |
67 | int delayed = 0; | |
86bba269 ED |
68 | int work_performed = 0; |
69 | unsigned long expires = ~0L; | |
70 | struct dst_entry *dst, *next, head; | |
71 | struct dst_entry *last = &head; | |
72 | #if RT_CACHE_DEBUG >= 2 | |
73 | ktime_t time_start = ktime_get(); | |
74 | struct timespec elapsed; | |
75 | #endif | |
1da177e4 | 76 | |
86bba269 ED |
77 | mutex_lock(&dst_gc_mutex); |
78 | next = dst_busy_list; | |
1da177e4 | 79 | |
86bba269 ED |
80 | loop: |
81 | while ((dst = next) != NULL) { | |
82 | next = dst->next; | |
83 | prefetch(&next->next); | |
2fc1b5dd | 84 | cond_resched(); |
86bba269 ED |
85 | if (likely(atomic_read(&dst->__refcnt))) { |
86 | last->next = dst; | |
87 | last = dst; | |
1da177e4 LT |
88 | delayed++; |
89 | continue; | |
90 | } | |
86bba269 | 91 | work_performed++; |
1da177e4 LT |
92 | |
93 | dst = dst_destroy(dst); | |
94 | if (dst) { | |
95 | /* NOHASH and still referenced. Unless it is already | |
96 | * on gc list, invalidate it and add to gc list. | |
97 | * | |
98 | * Note: this is temporary. Actually, NOHASH dst's | |
99 | * must be obsoleted when parent is obsoleted. | |
100 | * But we do not have state "obsoleted, but | |
101 | * referenced by parent", so it is right. | |
102 | */ | |
103 | if (dst->obsolete > 1) | |
104 | continue; | |
105 | ||
106 | ___dst_free(dst); | |
86bba269 ED |
107 | dst->next = next; |
108 | next = dst; | |
1da177e4 LT |
109 | } |
110 | } | |
86bba269 ED |
111 | |
112 | spin_lock_bh(&dst_garbage.lock); | |
113 | next = dst_garbage.list; | |
114 | if (next) { | |
115 | dst_garbage.list = NULL; | |
116 | spin_unlock_bh(&dst_garbage.lock); | |
117 | goto loop; | |
1da177e4 | 118 | } |
86bba269 ED |
119 | last->next = NULL; |
120 | dst_busy_list = head.next; | |
121 | if (!dst_busy_list) | |
122 | dst_garbage.timer_inc = DST_GC_MAX; | |
123 | else { | |
124 | /* | |
125 | * if we freed less than 1/10 of delayed entries, | |
126 | * we can sleep longer. | |
127 | */ | |
128 | if (work_performed <= delayed/10) { | |
129 | dst_garbage.timer_expires += dst_garbage.timer_inc; | |
130 | if (dst_garbage.timer_expires > DST_GC_MAX) | |
131 | dst_garbage.timer_expires = DST_GC_MAX; | |
132 | dst_garbage.timer_inc += DST_GC_INC; | |
133 | } else { | |
134 | dst_garbage.timer_inc = DST_GC_INC; | |
135 | dst_garbage.timer_expires = DST_GC_MIN; | |
136 | } | |
137 | expires = dst_garbage.timer_expires; | |
138 | /* | |
598ed936 | 139 | * if the next desired timer is more than 4 seconds in the |
140 | * future then round the timer to whole seconds | |
86bba269 ED |
141 | */ |
142 | if (expires > 4*HZ) | |
143 | expires = round_jiffies_relative(expires); | |
144 | schedule_delayed_work(&dst_gc_work, expires); | |
f0098f78 | 145 | } |
86bba269 ED |
146 | |
147 | spin_unlock_bh(&dst_garbage.lock); | |
148 | mutex_unlock(&dst_gc_mutex); | |
1da177e4 | 149 | #if RT_CACHE_DEBUG >= 2 |
86bba269 ED |
150 | elapsed = ktime_to_timespec(ktime_sub(ktime_get(), time_start)); |
151 | printk(KERN_DEBUG "dst_total: %d delayed: %d work_perf: %d" | |
152 | " expires: %lu elapsed: %lu us\n", | |
153 | atomic_read(&dst_total), delayed, work_performed, | |
154 | expires, | |
598ed936 | 155 | elapsed.tv_sec * USEC_PER_SEC + |
156 | elapsed.tv_nsec / NSEC_PER_USEC); | |
1da177e4 | 157 | #endif |
1da177e4 LT |
158 | } |
159 | ||
352e512c | 160 | int dst_discard(struct sk_buff *skb) |
1da177e4 LT |
161 | { |
162 | kfree_skb(skb); | |
163 | return 0; | |
164 | } | |
352e512c | 165 | EXPORT_SYMBOL(dst_discard); |
1da177e4 | 166 | |
725d1e1b | 167 | const u32 dst_default_metrics[RTAX_MAX]; |
62fa8a84 | 168 | |
3c7bd1a1 | 169 | void *dst_alloc(struct dst_ops *ops, int initial_ref) |
1da177e4 | 170 | { |
598ed936 | 171 | struct dst_entry *dst; |
1da177e4 | 172 | |
fc66f95c | 173 | if (ops->gc && dst_entries_get_fast(ops) > ops->gc_thresh) { |
569d3645 | 174 | if (ops->gc(ops)) |
1da177e4 LT |
175 | return NULL; |
176 | } | |
c3762229 | 177 | dst = kmem_cache_zalloc(ops->kmem_cachep, GFP_ATOMIC); |
1da177e4 LT |
178 | if (!dst) |
179 | return NULL; | |
3c7bd1a1 | 180 | atomic_set(&dst->__refcnt, initial_ref); |
1da177e4 LT |
181 | dst->ops = ops; |
182 | dst->lastuse = jiffies; | |
183 | dst->path = dst; | |
c4b1010f | 184 | dst->input = dst->output = dst_discard; |
62fa8a84 | 185 | dst_init_metrics(dst, dst_default_metrics, true); |
4ec93edb | 186 | #if RT_CACHE_DEBUG >= 2 |
1da177e4 LT |
187 | atomic_inc(&dst_total); |
188 | #endif | |
fc66f95c | 189 | dst_entries_add(ops, 1); |
1da177e4 LT |
190 | return dst; |
191 | } | |
598ed936 | 192 | EXPORT_SYMBOL(dst_alloc); |
1da177e4 | 193 | |
598ed936 | 194 | static void ___dst_free(struct dst_entry *dst) |
1da177e4 LT |
195 | { |
196 | /* The first case (dev==NULL) is required, when | |
197 | protocol module is unloaded. | |
198 | */ | |
598ed936 | 199 | if (dst->dev == NULL || !(dst->dev->flags&IFF_UP)) |
c4b1010f | 200 | dst->input = dst->output = dst_discard; |
1da177e4 LT |
201 | dst->obsolete = 2; |
202 | } | |
203 | ||
598ed936 | 204 | void __dst_free(struct dst_entry *dst) |
1da177e4 | 205 | { |
86bba269 | 206 | spin_lock_bh(&dst_garbage.lock); |
1da177e4 | 207 | ___dst_free(dst); |
86bba269 ED |
208 | dst->next = dst_garbage.list; |
209 | dst_garbage.list = dst; | |
210 | if (dst_garbage.timer_inc > DST_GC_INC) { | |
211 | dst_garbage.timer_inc = DST_GC_INC; | |
212 | dst_garbage.timer_expires = DST_GC_MIN; | |
f262b59b | 213 | cancel_delayed_work(&dst_gc_work); |
86bba269 | 214 | schedule_delayed_work(&dst_gc_work, dst_garbage.timer_expires); |
1da177e4 | 215 | } |
86bba269 | 216 | spin_unlock_bh(&dst_garbage.lock); |
1da177e4 | 217 | } |
d79d9913 | 218 | EXPORT_SYMBOL(__dst_free); |
1da177e4 LT |
219 | |
220 | struct dst_entry *dst_destroy(struct dst_entry * dst) | |
221 | { | |
222 | struct dst_entry *child; | |
223 | struct neighbour *neigh; | |
224 | struct hh_cache *hh; | |
225 | ||
226 | smp_rmb(); | |
227 | ||
228 | again: | |
229 | neigh = dst->neighbour; | |
230 | hh = dst->hh; | |
231 | child = dst->child; | |
232 | ||
233 | dst->hh = NULL; | |
34d101dd ED |
234 | if (hh) |
235 | hh_cache_put(hh); | |
1da177e4 LT |
236 | |
237 | if (neigh) { | |
238 | dst->neighbour = NULL; | |
239 | neigh_release(neigh); | |
240 | } | |
241 | ||
fc66f95c | 242 | dst_entries_add(dst->ops, -1); |
1da177e4 LT |
243 | |
244 | if (dst->ops->destroy) | |
245 | dst->ops->destroy(dst); | |
246 | if (dst->dev) | |
247 | dev_put(dst->dev); | |
4ec93edb | 248 | #if RT_CACHE_DEBUG >= 2 |
1da177e4 LT |
249 | atomic_dec(&dst_total); |
250 | #endif | |
251 | kmem_cache_free(dst->ops->kmem_cachep, dst); | |
252 | ||
253 | dst = child; | |
254 | if (dst) { | |
6775cab9 HX |
255 | int nohash = dst->flags & DST_NOHASH; |
256 | ||
1da177e4 LT |
257 | if (atomic_dec_and_test(&dst->__refcnt)) { |
258 | /* We were real parent of this dst, so kill child. */ | |
6775cab9 | 259 | if (nohash) |
1da177e4 LT |
260 | goto again; |
261 | } else { | |
262 | /* Child is still referenced, return it for freeing. */ | |
6775cab9 | 263 | if (nohash) |
1da177e4 LT |
264 | return dst; |
265 | /* Child is still in his hash table */ | |
266 | } | |
267 | } | |
268 | return NULL; | |
269 | } | |
598ed936 | 270 | EXPORT_SYMBOL(dst_destroy); |
1da177e4 | 271 | |
8d330868 IJ |
272 | void dst_release(struct dst_entry *dst) |
273 | { | |
274 | if (dst) { | |
598ed936 | 275 | int newrefcnt; |
ef711cf1 | 276 | |
598ed936 | 277 | newrefcnt = atomic_dec_return(&dst->__refcnt); |
278 | WARN_ON(newrefcnt < 0); | |
27b75c95 ED |
279 | if (unlikely(dst->flags & DST_NOCACHE) && !newrefcnt) { |
280 | dst = dst_destroy(dst); | |
281 | if (dst) | |
282 | __dst_free(dst); | |
283 | } | |
8d330868 IJ |
284 | } |
285 | } | |
286 | EXPORT_SYMBOL(dst_release); | |
287 | ||
62fa8a84 DM |
288 | u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old) |
289 | { | |
290 | u32 *p = kmalloc(sizeof(u32) * RTAX_MAX, GFP_ATOMIC); | |
291 | ||
292 | if (p) { | |
293 | u32 *old_p = __DST_METRICS_PTR(old); | |
294 | unsigned long prev, new; | |
295 | ||
296 | memcpy(p, old_p, sizeof(u32) * RTAX_MAX); | |
297 | ||
298 | new = (unsigned long) p; | |
299 | prev = cmpxchg(&dst->_metrics, old, new); | |
300 | ||
301 | if (prev != old) { | |
302 | kfree(p); | |
303 | p = __DST_METRICS_PTR(prev); | |
304 | if (prev & DST_METRICS_READ_ONLY) | |
305 | p = NULL; | |
306 | } | |
307 | } | |
308 | return p; | |
309 | } | |
310 | EXPORT_SYMBOL(dst_cow_metrics_generic); | |
311 | ||
312 | /* Caller asserts that dst_metrics_read_only(dst) is false. */ | |
313 | void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old) | |
314 | { | |
315 | unsigned long prev, new; | |
316 | ||
317 | new = (unsigned long) dst_default_metrics; | |
318 | prev = cmpxchg(&dst->_metrics, old, new); | |
319 | if (prev == old) | |
320 | kfree(__DST_METRICS_PTR(old)); | |
321 | } | |
322 | EXPORT_SYMBOL(__dst_destroy_metrics_generic); | |
323 | ||
27b75c95 ED |
324 | /** |
325 | * skb_dst_set_noref - sets skb dst, without a reference | |
326 | * @skb: buffer | |
327 | * @dst: dst entry | |
328 | * | |
329 | * Sets skb dst, assuming a reference was not taken on dst | |
330 | * skb_dst_drop() should not dst_release() this dst | |
331 | */ | |
332 | void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst) | |
333 | { | |
334 | WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); | |
335 | /* If dst not in cache, we must take a reference, because | |
336 | * dst_release() will destroy dst as soon as its refcount becomes zero | |
337 | */ | |
338 | if (unlikely(dst->flags & DST_NOCACHE)) { | |
339 | dst_hold(dst); | |
340 | skb_dst_set(skb, dst); | |
341 | } else { | |
342 | skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF; | |
343 | } | |
344 | } | |
345 | EXPORT_SYMBOL(skb_dst_set_noref); | |
346 | ||
1da177e4 LT |
347 | /* Dirty hack. We did it in 2.2 (in __dst_free), |
348 | * we have _very_ good reasons not to repeat | |
349 | * this mistake in 2.3, but we have no choice | |
350 | * now. _It_ _is_ _explicit_ _deliberate_ | |
351 | * _race_ _condition_. | |
352 | * | |
353 | * Commented and originally written by Alexey. | |
354 | */ | |
56115511 | 355 | static void dst_ifdown(struct dst_entry *dst, struct net_device *dev, |
356 | int unregister) | |
1da177e4 LT |
357 | { |
358 | if (dst->ops->ifdown) | |
359 | dst->ops->ifdown(dst, dev, unregister); | |
360 | ||
361 | if (dev != dst->dev) | |
362 | return; | |
363 | ||
364 | if (!unregister) { | |
c4b1010f | 365 | dst->input = dst->output = dst_discard; |
1da177e4 | 366 | } else { |
c346dca1 | 367 | dst->dev = dev_net(dst->dev)->loopback_dev; |
de3cb747 | 368 | dev_hold(dst->dev); |
1da177e4 LT |
369 | dev_put(dev); |
370 | if (dst->neighbour && dst->neighbour->dev == dev) { | |
5a3e55d6 | 371 | dst->neighbour->dev = dst->dev; |
64b7d961 | 372 | dev_hold(dst->dev); |
1da177e4 | 373 | dev_put(dev); |
1da177e4 LT |
374 | } |
375 | } | |
376 | } | |
377 | ||
598ed936 | 378 | static int dst_dev_event(struct notifier_block *this, unsigned long event, |
379 | void *ptr) | |
1da177e4 LT |
380 | { |
381 | struct net_device *dev = ptr; | |
86bba269 | 382 | struct dst_entry *dst, *last = NULL; |
1da177e4 LT |
383 | |
384 | switch (event) { | |
385 | case NETDEV_UNREGISTER: | |
386 | case NETDEV_DOWN: | |
86bba269 ED |
387 | mutex_lock(&dst_gc_mutex); |
388 | for (dst = dst_busy_list; dst; dst = dst->next) { | |
389 | last = dst; | |
390 | dst_ifdown(dst, dev, event != NETDEV_DOWN); | |
391 | } | |
392 | ||
393 | spin_lock_bh(&dst_garbage.lock); | |
394 | dst = dst_garbage.list; | |
395 | dst_garbage.list = NULL; | |
396 | spin_unlock_bh(&dst_garbage.lock); | |
397 | ||
398 | if (last) | |
399 | last->next = dst; | |
400 | else | |
401 | dst_busy_list = dst; | |
598ed936 | 402 | for (; dst; dst = dst->next) |
1da177e4 | 403 | dst_ifdown(dst, dev, event != NETDEV_DOWN); |
86bba269 | 404 | mutex_unlock(&dst_gc_mutex); |
1da177e4 LT |
405 | break; |
406 | } | |
407 | return NOTIFY_DONE; | |
408 | } | |
409 | ||
410 | static struct notifier_block dst_dev_notifier = { | |
411 | .notifier_call = dst_dev_event, | |
332dd96f | 412 | .priority = -10, /* must be called after other network notifiers */ |
1da177e4 LT |
413 | }; |
414 | ||
415 | void __init dst_init(void) | |
416 | { | |
417 | register_netdevice_notifier(&dst_dev_notifier); | |
418 | } |