]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - fs/dcache.c
fs: dcache scale dentry refcount
[mirror_ubuntu-zesty-kernel.git] / fs / dcache.c
1 /*
2 * fs/dcache.c
3 *
4 * Complete reimplementation
5 * (C) 1997 Thomas Schoebel-Theuer,
6 * with heavy changes by Linus Torvalds
7 */
8
9 /*
10 * Notes on the allocation strategy:
11 *
12 * The dcache is a master of the icache - whenever a dcache entry
13 * exists, the inode will always exist. "iput()" is done either when
14 * the dcache entry is deleted or garbage collected.
15 */
16
17 #include <linux/syscalls.h>
18 #include <linux/string.h>
19 #include <linux/mm.h>
20 #include <linux/fs.h>
21 #include <linux/fsnotify.h>
22 #include <linux/slab.h>
23 #include <linux/init.h>
24 #include <linux/hash.h>
25 #include <linux/cache.h>
26 #include <linux/module.h>
27 #include <linux/mount.h>
28 #include <linux/file.h>
29 #include <asm/uaccess.h>
30 #include <linux/security.h>
31 #include <linux/seqlock.h>
32 #include <linux/swap.h>
33 #include <linux/bootmem.h>
34 #include <linux/fs_struct.h>
35 #include <linux/hardirq.h>
36 #include "internal.h"
37
38 /*
39 * Usage:
40 * dcache_hash_lock protects:
41 * - the dcache hash table, s_anon lists
42 * dcache_lru_lock protects:
43 * - the dcache lru lists and counters
44 * d_lock protects:
45 * - d_flags
46 * - d_name
47 * - d_lru
48 * - d_count
49 *
50 * Ordering:
51 * dcache_lock
52 * dentry->d_lock
53 * dcache_lru_lock
54 * dcache_hash_lock
55 *
56 * if (dentry1 < dentry2)
57 * dentry1->d_lock
58 * dentry2->d_lock
59 */
60 int sysctl_vfs_cache_pressure __read_mostly = 100;
61 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
62
63 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock);
64 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);
65 __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock);
66 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
67
68 EXPORT_SYMBOL(dcache_lock);
69
70 static struct kmem_cache *dentry_cache __read_mostly;
71
72 #define DNAME_INLINE_LEN (sizeof(struct dentry)-offsetof(struct dentry,d_iname))
73
74 /*
75 * This is the single most critical data structure when it comes
76 * to the dcache: the hashtable for lookups. Somebody should try
77 * to make this good - I've just made it work.
78 *
79 * This hash-function tries to avoid losing too many bits of hash
80 * information, yet avoid using a prime hash-size or similar.
81 */
82 #define D_HASHBITS d_hash_shift
83 #define D_HASHMASK d_hash_mask
84
85 static unsigned int d_hash_mask __read_mostly;
86 static unsigned int d_hash_shift __read_mostly;
87 static struct hlist_head *dentry_hashtable __read_mostly;
88
89 /* Statistics gathering. */
90 struct dentry_stat_t dentry_stat = {
91 .age_limit = 45,
92 };
93
94 static DEFINE_PER_CPU(unsigned int, nr_dentry);
95
96 #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
97 static int get_nr_dentry(void)
98 {
99 int i;
100 int sum = 0;
101 for_each_possible_cpu(i)
102 sum += per_cpu(nr_dentry, i);
103 return sum < 0 ? 0 : sum;
104 }
105
106 int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,
107 size_t *lenp, loff_t *ppos)
108 {
109 dentry_stat.nr_dentry = get_nr_dentry();
110 return proc_dointvec(table, write, buffer, lenp, ppos);
111 }
112 #endif
113
114 static void __d_free(struct rcu_head *head)
115 {
116 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
117
118 WARN_ON(!list_empty(&dentry->d_alias));
119 if (dname_external(dentry))
120 kfree(dentry->d_name.name);
121 kmem_cache_free(dentry_cache, dentry);
122 }
123
124 /*
125 * no dcache_lock, please.
126 */
127 static void d_free(struct dentry *dentry)
128 {
129 BUG_ON(dentry->d_count);
130 this_cpu_dec(nr_dentry);
131 if (dentry->d_op && dentry->d_op->d_release)
132 dentry->d_op->d_release(dentry);
133
134 /* if dentry was never inserted into hash, immediate free is OK */
135 if (hlist_unhashed(&dentry->d_hash))
136 __d_free(&dentry->d_u.d_rcu);
137 else
138 call_rcu(&dentry->d_u.d_rcu, __d_free);
139 }
140
141 /*
142 * Release the dentry's inode, using the filesystem
143 * d_iput() operation if defined.
144 */
145 static void dentry_iput(struct dentry * dentry)
146 __releases(dentry->d_lock)
147 __releases(dcache_lock)
148 {
149 struct inode *inode = dentry->d_inode;
150 if (inode) {
151 dentry->d_inode = NULL;
152 list_del_init(&dentry->d_alias);
153 spin_unlock(&dentry->d_lock);
154 spin_unlock(&dcache_lock);
155 if (!inode->i_nlink)
156 fsnotify_inoderemove(inode);
157 if (dentry->d_op && dentry->d_op->d_iput)
158 dentry->d_op->d_iput(dentry, inode);
159 else
160 iput(inode);
161 } else {
162 spin_unlock(&dentry->d_lock);
163 spin_unlock(&dcache_lock);
164 }
165 }
166
167 /*
168 * dentry_lru_(add|del|move_tail) must be called with d_lock held.
169 */
170 static void dentry_lru_add(struct dentry *dentry)
171 {
172 if (list_empty(&dentry->d_lru)) {
173 spin_lock(&dcache_lru_lock);
174 list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
175 dentry->d_sb->s_nr_dentry_unused++;
176 dentry_stat.nr_unused++;
177 spin_unlock(&dcache_lru_lock);
178 }
179 }
180
181 static void __dentry_lru_del(struct dentry *dentry)
182 {
183 list_del_init(&dentry->d_lru);
184 dentry->d_sb->s_nr_dentry_unused--;
185 dentry_stat.nr_unused--;
186 }
187
188 static void dentry_lru_del(struct dentry *dentry)
189 {
190 if (!list_empty(&dentry->d_lru)) {
191 spin_lock(&dcache_lru_lock);
192 __dentry_lru_del(dentry);
193 spin_unlock(&dcache_lru_lock);
194 }
195 }
196
197 static void dentry_lru_move_tail(struct dentry *dentry)
198 {
199 spin_lock(&dcache_lru_lock);
200 if (list_empty(&dentry->d_lru)) {
201 list_add_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
202 dentry->d_sb->s_nr_dentry_unused++;
203 dentry_stat.nr_unused++;
204 } else {
205 list_move_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
206 }
207 spin_unlock(&dcache_lru_lock);
208 }
209
210 /**
211 * d_kill - kill dentry and return parent
212 * @dentry: dentry to kill
213 *
214 * The dentry must already be unhashed and removed from the LRU.
215 *
216 * If this is the root of the dentry tree, return NULL.
217 *
218 * dcache_lock and d_lock must be held by caller, are dropped by d_kill.
219 */
220 static struct dentry *d_kill(struct dentry *dentry)
221 __releases(dentry->d_lock)
222 __releases(dcache_lock)
223 {
224 struct dentry *parent;
225
226 list_del(&dentry->d_u.d_child);
227 dentry_iput(dentry);
228 /*
229 * dentry_iput drops the locks, at which point nobody (except
230 * transient RCU lookups) can reach this dentry.
231 */
232 if (IS_ROOT(dentry))
233 parent = NULL;
234 else
235 parent = dentry->d_parent;
236 d_free(dentry);
237 return parent;
238 }
239
240 /**
241 * d_drop - drop a dentry
242 * @dentry: dentry to drop
243 *
244 * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
245 * be found through a VFS lookup any more. Note that this is different from
246 * deleting the dentry - d_delete will try to mark the dentry negative if
247 * possible, giving a successful _negative_ lookup, while d_drop will
248 * just make the cache lookup fail.
249 *
250 * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
251 * reason (NFS timeouts or autofs deletes).
252 *
253 * __d_drop requires dentry->d_lock.
254 */
255 void __d_drop(struct dentry *dentry)
256 {
257 if (!(dentry->d_flags & DCACHE_UNHASHED)) {
258 dentry->d_flags |= DCACHE_UNHASHED;
259 spin_lock(&dcache_hash_lock);
260 hlist_del_rcu(&dentry->d_hash);
261 spin_unlock(&dcache_hash_lock);
262 }
263 }
264 EXPORT_SYMBOL(__d_drop);
265
266 void d_drop(struct dentry *dentry)
267 {
268 spin_lock(&dcache_lock);
269 spin_lock(&dentry->d_lock);
270 __d_drop(dentry);
271 spin_unlock(&dentry->d_lock);
272 spin_unlock(&dcache_lock);
273 }
274 EXPORT_SYMBOL(d_drop);
275
276 /*
277 * This is dput
278 *
279 * This is complicated by the fact that we do not want to put
280 * dentries that are no longer on any hash chain on the unused
281 * list: we'd much rather just get rid of them immediately.
282 *
283 * However, that implies that we have to traverse the dentry
284 * tree upwards to the parents which might _also_ now be
285 * scheduled for deletion (it may have been only waiting for
286 * its last child to go away).
287 *
288 * This tail recursion is done by hand as we don't want to depend
289 * on the compiler to always get this right (gcc generally doesn't).
290 * Real recursion would eat up our stack space.
291 */
292
293 /*
294 * dput - release a dentry
295 * @dentry: dentry to release
296 *
297 * Release a dentry. This will drop the usage count and if appropriate
298 * call the dentry unlink method as well as removing it from the queues and
299 * releasing its resources. If the parent dentries were scheduled for release
300 * they too may now get deleted.
301 *
302 * no dcache lock, please.
303 */
304
305 void dput(struct dentry *dentry)
306 {
307 if (!dentry)
308 return;
309
310 repeat:
311 if (dentry->d_count == 1)
312 might_sleep();
313 spin_lock(&dentry->d_lock);
314 if (dentry->d_count == 1) {
315 if (!spin_trylock(&dcache_lock)) {
316 /*
317 * Something of a livelock possibility we could avoid
318 * by taking dcache_lock and trying again, but we
319 * want to reduce dcache_lock anyway so this will
320 * get improved.
321 */
322 spin_unlock(&dentry->d_lock);
323 goto repeat;
324 }
325 }
326 dentry->d_count--;
327 if (dentry->d_count) {
328 spin_unlock(&dentry->d_lock);
329 spin_unlock(&dcache_lock);
330 return;
331 }
332
333 /*
334 * AV: ->d_delete() is _NOT_ allowed to block now.
335 */
336 if (dentry->d_op && dentry->d_op->d_delete) {
337 if (dentry->d_op->d_delete(dentry))
338 goto unhash_it;
339 }
340
341 /* Unreachable? Get rid of it */
342 if (d_unhashed(dentry))
343 goto kill_it;
344
345 /* Otherwise leave it cached and ensure it's on the LRU */
346 dentry->d_flags |= DCACHE_REFERENCED;
347 dentry_lru_add(dentry);
348
349 spin_unlock(&dentry->d_lock);
350 spin_unlock(&dcache_lock);
351 return;
352
353 unhash_it:
354 __d_drop(dentry);
355 kill_it:
356 /* if dentry was on the d_lru list delete it from there */
357 dentry_lru_del(dentry);
358 dentry = d_kill(dentry);
359 if (dentry)
360 goto repeat;
361 }
362 EXPORT_SYMBOL(dput);
363
364 /**
365 * d_invalidate - invalidate a dentry
366 * @dentry: dentry to invalidate
367 *
368 * Try to invalidate the dentry if it turns out to be
369 * possible. If there are other dentries that can be
370 * reached through this one we can't delete it and we
371 * return -EBUSY. On success we return 0.
372 *
373 * no dcache lock.
374 */
375
376 int d_invalidate(struct dentry * dentry)
377 {
378 /*
379 * If it's already been dropped, return OK.
380 */
381 spin_lock(&dcache_lock);
382 if (d_unhashed(dentry)) {
383 spin_unlock(&dcache_lock);
384 return 0;
385 }
386 /*
387 * Check whether to do a partial shrink_dcache
388 * to get rid of unused child entries.
389 */
390 if (!list_empty(&dentry->d_subdirs)) {
391 spin_unlock(&dcache_lock);
392 shrink_dcache_parent(dentry);
393 spin_lock(&dcache_lock);
394 }
395
396 /*
397 * Somebody else still using it?
398 *
399 * If it's a directory, we can't drop it
400 * for fear of somebody re-populating it
401 * with children (even though dropping it
402 * would make it unreachable from the root,
403 * we might still populate it if it was a
404 * working directory or similar).
405 */
406 spin_lock(&dentry->d_lock);
407 if (dentry->d_count > 1) {
408 if (dentry->d_inode && S_ISDIR(dentry->d_inode->i_mode)) {
409 spin_unlock(&dentry->d_lock);
410 spin_unlock(&dcache_lock);
411 return -EBUSY;
412 }
413 }
414
415 __d_drop(dentry);
416 spin_unlock(&dentry->d_lock);
417 spin_unlock(&dcache_lock);
418 return 0;
419 }
420 EXPORT_SYMBOL(d_invalidate);
421
422 /* This must be called with dcache_lock and d_lock held */
423 static inline struct dentry * __dget_locked_dlock(struct dentry *dentry)
424 {
425 dentry->d_count++;
426 dentry_lru_del(dentry);
427 return dentry;
428 }
429
430 /* This should be called _only_ with dcache_lock held */
431 static inline struct dentry * __dget_locked(struct dentry *dentry)
432 {
433 spin_lock(&dentry->d_lock);
434 __dget_locked_dlock(dentry);
435 spin_unlock(&dentry->d_lock);
436 return dentry;
437 }
438
439 struct dentry * dget_locked_dlock(struct dentry *dentry)
440 {
441 return __dget_locked_dlock(dentry);
442 }
443
444 struct dentry * dget_locked(struct dentry *dentry)
445 {
446 return __dget_locked(dentry);
447 }
448 EXPORT_SYMBOL(dget_locked);
449
450 struct dentry *dget_parent(struct dentry *dentry)
451 {
452 struct dentry *ret;
453
454 repeat:
455 spin_lock(&dentry->d_lock);
456 ret = dentry->d_parent;
457 if (!ret)
458 goto out;
459 if (dentry == ret) {
460 ret->d_count++;
461 goto out;
462 }
463 if (!spin_trylock(&ret->d_lock)) {
464 spin_unlock(&dentry->d_lock);
465 cpu_relax();
466 goto repeat;
467 }
468 BUG_ON(!ret->d_count);
469 ret->d_count++;
470 spin_unlock(&ret->d_lock);
471 out:
472 spin_unlock(&dentry->d_lock);
473 return ret;
474 }
475 EXPORT_SYMBOL(dget_parent);
476
477 /**
478 * d_find_alias - grab a hashed alias of inode
479 * @inode: inode in question
480 * @want_discon: flag, used by d_splice_alias, to request
481 * that only a DISCONNECTED alias be returned.
482 *
483 * If inode has a hashed alias, or is a directory and has any alias,
484 * acquire the reference to alias and return it. Otherwise return NULL.
485 * Notice that if inode is a directory there can be only one alias and
486 * it can be unhashed only if it has no children, or if it is the root
487 * of a filesystem.
488 *
489 * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
490 * any other hashed alias over that one unless @want_discon is set,
491 * in which case only return an IS_ROOT, DCACHE_DISCONNECTED alias.
492 */
493
494 static struct dentry * __d_find_alias(struct inode *inode, int want_discon)
495 {
496 struct list_head *head, *next, *tmp;
497 struct dentry *alias, *discon_alias=NULL;
498
499 head = &inode->i_dentry;
500 next = inode->i_dentry.next;
501 while (next != head) {
502 tmp = next;
503 next = tmp->next;
504 prefetch(next);
505 alias = list_entry(tmp, struct dentry, d_alias);
506 if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
507 if (IS_ROOT(alias) &&
508 (alias->d_flags & DCACHE_DISCONNECTED))
509 discon_alias = alias;
510 else if (!want_discon) {
511 __dget_locked(alias);
512 return alias;
513 }
514 }
515 }
516 if (discon_alias)
517 __dget_locked(discon_alias);
518 return discon_alias;
519 }
520
521 struct dentry * d_find_alias(struct inode *inode)
522 {
523 struct dentry *de = NULL;
524
525 if (!list_empty(&inode->i_dentry)) {
526 spin_lock(&dcache_lock);
527 de = __d_find_alias(inode, 0);
528 spin_unlock(&dcache_lock);
529 }
530 return de;
531 }
532 EXPORT_SYMBOL(d_find_alias);
533
534 /*
535 * Try to kill dentries associated with this inode.
536 * WARNING: you must own a reference to inode.
537 */
538 void d_prune_aliases(struct inode *inode)
539 {
540 struct dentry *dentry;
541 restart:
542 spin_lock(&dcache_lock);
543 list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
544 spin_lock(&dentry->d_lock);
545 if (!dentry->d_count) {
546 __dget_locked_dlock(dentry);
547 __d_drop(dentry);
548 spin_unlock(&dentry->d_lock);
549 spin_unlock(&dcache_lock);
550 dput(dentry);
551 goto restart;
552 }
553 spin_unlock(&dentry->d_lock);
554 }
555 spin_unlock(&dcache_lock);
556 }
557 EXPORT_SYMBOL(d_prune_aliases);
558
559 /*
560 * Throw away a dentry - free the inode, dput the parent. This requires that
561 * the LRU list has already been removed.
562 *
563 * Try to prune ancestors as well. This is necessary to prevent
564 * quadratic behavior of shrink_dcache_parent(), but is also expected
565 * to be beneficial in reducing dentry cache fragmentation.
566 */
567 static void prune_one_dentry(struct dentry * dentry)
568 __releases(dentry->d_lock)
569 __releases(dcache_lock)
570 {
571 __d_drop(dentry);
572 dentry = d_kill(dentry);
573
574 /*
575 * Prune ancestors. Locking is simpler than in dput(),
576 * because dcache_lock needs to be taken anyway.
577 */
578 while (dentry) {
579 spin_lock(&dcache_lock);
580 spin_lock(&dentry->d_lock);
581 dentry->d_count--;
582 if (dentry->d_count) {
583 spin_unlock(&dentry->d_lock);
584 spin_unlock(&dcache_lock);
585 return;
586 }
587
588 dentry_lru_del(dentry);
589 __d_drop(dentry);
590 dentry = d_kill(dentry);
591 }
592 }
593
594 static void shrink_dentry_list(struct list_head *list)
595 {
596 struct dentry *dentry;
597
598 while (!list_empty(list)) {
599 dentry = list_entry(list->prev, struct dentry, d_lru);
600
601 if (!spin_trylock(&dentry->d_lock)) {
602 spin_unlock(&dcache_lru_lock);
603 cpu_relax();
604 spin_lock(&dcache_lru_lock);
605 continue;
606 }
607
608 __dentry_lru_del(dentry);
609
610 /*
611 * We found an inuse dentry which was not removed from
612 * the LRU because of laziness during lookup. Do not free
613 * it - just keep it off the LRU list.
614 */
615 if (dentry->d_count) {
616 spin_unlock(&dentry->d_lock);
617 continue;
618 }
619 spin_unlock(&dcache_lru_lock);
620
621 prune_one_dentry(dentry);
622 /* dcache_lock and dentry->d_lock dropped */
623 spin_lock(&dcache_lock);
624 spin_lock(&dcache_lru_lock);
625 }
626 }
627
628 /**
629 * __shrink_dcache_sb - shrink the dentry LRU on a given superblock
630 * @sb: superblock to shrink dentry LRU.
631 * @count: number of entries to prune
632 * @flags: flags to control the dentry processing
633 *
634 * If flags contains DCACHE_REFERENCED reference dentries will not be pruned.
635 */
636 static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags)
637 {
638 /* called from prune_dcache() and shrink_dcache_parent() */
639 struct dentry *dentry;
640 LIST_HEAD(referenced);
641 LIST_HEAD(tmp);
642 int cnt = *count;
643
644 spin_lock(&dcache_lock);
645 relock:
646 spin_lock(&dcache_lru_lock);
647 while (!list_empty(&sb->s_dentry_lru)) {
648 dentry = list_entry(sb->s_dentry_lru.prev,
649 struct dentry, d_lru);
650 BUG_ON(dentry->d_sb != sb);
651
652 if (!spin_trylock(&dentry->d_lock)) {
653 spin_unlock(&dcache_lru_lock);
654 cpu_relax();
655 goto relock;
656 }
657
658 /*
659 * If we are honouring the DCACHE_REFERENCED flag and the
660 * dentry has this flag set, don't free it. Clear the flag
661 * and put it back on the LRU.
662 */
663 if (flags & DCACHE_REFERENCED &&
664 dentry->d_flags & DCACHE_REFERENCED) {
665 dentry->d_flags &= ~DCACHE_REFERENCED;
666 list_move(&dentry->d_lru, &referenced);
667 spin_unlock(&dentry->d_lock);
668 } else {
669 list_move_tail(&dentry->d_lru, &tmp);
670 spin_unlock(&dentry->d_lock);
671 if (!--cnt)
672 break;
673 }
674 /* XXX: re-add cond_resched_lock when dcache_lock goes away */
675 }
676
677 *count = cnt;
678 shrink_dentry_list(&tmp);
679
680 if (!list_empty(&referenced))
681 list_splice(&referenced, &sb->s_dentry_lru);
682 spin_unlock(&dcache_lru_lock);
683 spin_unlock(&dcache_lock);
684
685 }
686
687 /**
688 * prune_dcache - shrink the dcache
689 * @count: number of entries to try to free
690 *
691 * Shrink the dcache. This is done when we need more memory, or simply when we
692 * need to unmount something (at which point we need to unuse all dentries).
693 *
694 * This function may fail to free any resources if all the dentries are in use.
695 */
696 static void prune_dcache(int count)
697 {
698 struct super_block *sb, *p = NULL;
699 int w_count;
700 int unused = dentry_stat.nr_unused;
701 int prune_ratio;
702 int pruned;
703
704 if (unused == 0 || count == 0)
705 return;
706 spin_lock(&dcache_lock);
707 if (count >= unused)
708 prune_ratio = 1;
709 else
710 prune_ratio = unused / count;
711 spin_lock(&sb_lock);
712 list_for_each_entry(sb, &super_blocks, s_list) {
713 if (list_empty(&sb->s_instances))
714 continue;
715 if (sb->s_nr_dentry_unused == 0)
716 continue;
717 sb->s_count++;
718 /* Now, we reclaim unused dentrins with fairness.
719 * We reclaim them same percentage from each superblock.
720 * We calculate number of dentries to scan on this sb
721 * as follows, but the implementation is arranged to avoid
722 * overflows:
723 * number of dentries to scan on this sb =
724 * count * (number of dentries on this sb /
725 * number of dentries in the machine)
726 */
727 spin_unlock(&sb_lock);
728 if (prune_ratio != 1)
729 w_count = (sb->s_nr_dentry_unused / prune_ratio) + 1;
730 else
731 w_count = sb->s_nr_dentry_unused;
732 pruned = w_count;
733 /*
734 * We need to be sure this filesystem isn't being unmounted,
735 * otherwise we could race with generic_shutdown_super(), and
736 * end up holding a reference to an inode while the filesystem
737 * is unmounted. So we try to get s_umount, and make sure
738 * s_root isn't NULL.
739 */
740 if (down_read_trylock(&sb->s_umount)) {
741 if ((sb->s_root != NULL) &&
742 (!list_empty(&sb->s_dentry_lru))) {
743 spin_unlock(&dcache_lock);
744 __shrink_dcache_sb(sb, &w_count,
745 DCACHE_REFERENCED);
746 pruned -= w_count;
747 spin_lock(&dcache_lock);
748 }
749 up_read(&sb->s_umount);
750 }
751 spin_lock(&sb_lock);
752 if (p)
753 __put_super(p);
754 count -= pruned;
755 p = sb;
756 /* more work left to do? */
757 if (count <= 0)
758 break;
759 }
760 if (p)
761 __put_super(p);
762 spin_unlock(&sb_lock);
763 spin_unlock(&dcache_lock);
764 }
765
766 /**
767 * shrink_dcache_sb - shrink dcache for a superblock
768 * @sb: superblock
769 *
770 * Shrink the dcache for the specified super block. This is used to free
771 * the dcache before unmounting a file system.
772 */
773 void shrink_dcache_sb(struct super_block *sb)
774 {
775 LIST_HEAD(tmp);
776
777 spin_lock(&dcache_lock);
778 spin_lock(&dcache_lru_lock);
779 while (!list_empty(&sb->s_dentry_lru)) {
780 list_splice_init(&sb->s_dentry_lru, &tmp);
781 shrink_dentry_list(&tmp);
782 }
783 spin_unlock(&dcache_lru_lock);
784 spin_unlock(&dcache_lock);
785 }
786 EXPORT_SYMBOL(shrink_dcache_sb);
787
788 /*
789 * destroy a single subtree of dentries for unmount
790 * - see the comments on shrink_dcache_for_umount() for a description of the
791 * locking
792 */
793 static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
794 {
795 struct dentry *parent;
796 unsigned detached = 0;
797
798 BUG_ON(!IS_ROOT(dentry));
799
800 /* detach this root from the system */
801 spin_lock(&dcache_lock);
802 spin_lock(&dentry->d_lock);
803 dentry_lru_del(dentry);
804 spin_unlock(&dentry->d_lock);
805 __d_drop(dentry);
806 spin_unlock(&dcache_lock);
807
808 for (;;) {
809 /* descend to the first leaf in the current subtree */
810 while (!list_empty(&dentry->d_subdirs)) {
811 struct dentry *loop;
812
813 /* this is a branch with children - detach all of them
814 * from the system in one go */
815 spin_lock(&dcache_lock);
816 list_for_each_entry(loop, &dentry->d_subdirs,
817 d_u.d_child) {
818 spin_lock(&loop->d_lock);
819 dentry_lru_del(loop);
820 spin_unlock(&loop->d_lock);
821 __d_drop(loop);
822 cond_resched_lock(&dcache_lock);
823 }
824 spin_unlock(&dcache_lock);
825
826 /* move to the first child */
827 dentry = list_entry(dentry->d_subdirs.next,
828 struct dentry, d_u.d_child);
829 }
830
831 /* consume the dentries from this leaf up through its parents
832 * until we find one with children or run out altogether */
833 do {
834 struct inode *inode;
835
836 if (dentry->d_count != 0) {
837 printk(KERN_ERR
838 "BUG: Dentry %p{i=%lx,n=%s}"
839 " still in use (%d)"
840 " [unmount of %s %s]\n",
841 dentry,
842 dentry->d_inode ?
843 dentry->d_inode->i_ino : 0UL,
844 dentry->d_name.name,
845 dentry->d_count,
846 dentry->d_sb->s_type->name,
847 dentry->d_sb->s_id);
848 BUG();
849 }
850
851 if (IS_ROOT(dentry))
852 parent = NULL;
853 else {
854 parent = dentry->d_parent;
855 spin_lock(&parent->d_lock);
856 parent->d_count--;
857 spin_unlock(&parent->d_lock);
858 }
859
860 list_del(&dentry->d_u.d_child);
861 detached++;
862
863 inode = dentry->d_inode;
864 if (inode) {
865 dentry->d_inode = NULL;
866 list_del_init(&dentry->d_alias);
867 if (dentry->d_op && dentry->d_op->d_iput)
868 dentry->d_op->d_iput(dentry, inode);
869 else
870 iput(inode);
871 }
872
873 d_free(dentry);
874
875 /* finished when we fall off the top of the tree,
876 * otherwise we ascend to the parent and move to the
877 * next sibling if there is one */
878 if (!parent)
879 return;
880 dentry = parent;
881 } while (list_empty(&dentry->d_subdirs));
882
883 dentry = list_entry(dentry->d_subdirs.next,
884 struct dentry, d_u.d_child);
885 }
886 }
887
888 /*
889 * destroy the dentries attached to a superblock on unmounting
890 * - we don't need to use dentry->d_lock, and only need dcache_lock when
891 * removing the dentry from the system lists and hashes because:
892 * - the superblock is detached from all mountings and open files, so the
893 * dentry trees will not be rearranged by the VFS
894 * - s_umount is write-locked, so the memory pressure shrinker will ignore
895 * any dentries belonging to this superblock that it comes across
896 * - the filesystem itself is no longer permitted to rearrange the dentries
897 * in this superblock
898 */
899 void shrink_dcache_for_umount(struct super_block *sb)
900 {
901 struct dentry *dentry;
902
903 if (down_read_trylock(&sb->s_umount))
904 BUG();
905
906 dentry = sb->s_root;
907 sb->s_root = NULL;
908 spin_lock(&dentry->d_lock);
909 dentry->d_count--;
910 spin_unlock(&dentry->d_lock);
911 shrink_dcache_for_umount_subtree(dentry);
912
913 while (!hlist_empty(&sb->s_anon)) {
914 dentry = hlist_entry(sb->s_anon.first, struct dentry, d_hash);
915 shrink_dcache_for_umount_subtree(dentry);
916 }
917 }
918
919 /*
920 * Search for at least 1 mount point in the dentry's subdirs.
921 * We descend to the next level whenever the d_subdirs
922 * list is non-empty and continue searching.
923 */
924
925 /**
926 * have_submounts - check for mounts over a dentry
927 * @parent: dentry to check.
928 *
929 * Return true if the parent or its subdirectories contain
930 * a mount point
931 */
932
933 int have_submounts(struct dentry *parent)
934 {
935 struct dentry *this_parent = parent;
936 struct list_head *next;
937
938 spin_lock(&dcache_lock);
939 if (d_mountpoint(parent))
940 goto positive;
941 repeat:
942 next = this_parent->d_subdirs.next;
943 resume:
944 while (next != &this_parent->d_subdirs) {
945 struct list_head *tmp = next;
946 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
947 next = tmp->next;
948 /* Have we found a mount point ? */
949 if (d_mountpoint(dentry))
950 goto positive;
951 if (!list_empty(&dentry->d_subdirs)) {
952 this_parent = dentry;
953 goto repeat;
954 }
955 }
956 /*
957 * All done at this level ... ascend and resume the search.
958 */
959 if (this_parent != parent) {
960 next = this_parent->d_u.d_child.next;
961 this_parent = this_parent->d_parent;
962 goto resume;
963 }
964 spin_unlock(&dcache_lock);
965 return 0; /* No mount points found in tree */
966 positive:
967 spin_unlock(&dcache_lock);
968 return 1;
969 }
970 EXPORT_SYMBOL(have_submounts);
971
972 /*
973 * Search the dentry child list for the specified parent,
974 * and move any unused dentries to the end of the unused
975 * list for prune_dcache(). We descend to the next level
976 * whenever the d_subdirs list is non-empty and continue
977 * searching.
978 *
979 * It returns zero iff there are no unused children,
980 * otherwise it returns the number of children moved to
981 * the end of the unused list. This may not be the total
982 * number of unused children, because select_parent can
983 * drop the lock and return early due to latency
984 * constraints.
985 */
986 static int select_parent(struct dentry * parent)
987 {
988 struct dentry *this_parent = parent;
989 struct list_head *next;
990 int found = 0;
991
992 spin_lock(&dcache_lock);
993 repeat:
994 next = this_parent->d_subdirs.next;
995 resume:
996 while (next != &this_parent->d_subdirs) {
997 struct list_head *tmp = next;
998 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
999 next = tmp->next;
1000
1001 spin_lock(&dentry->d_lock);
1002
1003 /*
1004 * move only zero ref count dentries to the end
1005 * of the unused list for prune_dcache
1006 */
1007 if (!dentry->d_count) {
1008 dentry_lru_move_tail(dentry);
1009 found++;
1010 } else {
1011 dentry_lru_del(dentry);
1012 }
1013
1014 spin_unlock(&dentry->d_lock);
1015
1016 /*
1017 * We can return to the caller if we have found some (this
1018 * ensures forward progress). We'll be coming back to find
1019 * the rest.
1020 */
1021 if (found && need_resched())
1022 goto out;
1023
1024 /*
1025 * Descend a level if the d_subdirs list is non-empty.
1026 */
1027 if (!list_empty(&dentry->d_subdirs)) {
1028 this_parent = dentry;
1029 goto repeat;
1030 }
1031 }
1032 /*
1033 * All done at this level ... ascend and resume the search.
1034 */
1035 if (this_parent != parent) {
1036 next = this_parent->d_u.d_child.next;
1037 this_parent = this_parent->d_parent;
1038 goto resume;
1039 }
1040 out:
1041 spin_unlock(&dcache_lock);
1042 return found;
1043 }
1044
1045 /**
1046 * shrink_dcache_parent - prune dcache
1047 * @parent: parent of entries to prune
1048 *
1049 * Prune the dcache to remove unused children of the parent dentry.
1050 */
1051
1052 void shrink_dcache_parent(struct dentry * parent)
1053 {
1054 struct super_block *sb = parent->d_sb;
1055 int found;
1056
1057 while ((found = select_parent(parent)) != 0)
1058 __shrink_dcache_sb(sb, &found, 0);
1059 }
1060 EXPORT_SYMBOL(shrink_dcache_parent);
1061
1062 /*
1063 * Scan `nr' dentries and return the number which remain.
1064 *
1065 * We need to avoid reentering the filesystem if the caller is performing a
1066 * GFP_NOFS allocation attempt. One example deadlock is:
1067 *
1068 * ext2_new_block->getblk->GFP->shrink_dcache_memory->prune_dcache->
1069 * prune_one_dentry->dput->dentry_iput->iput->inode->i_sb->s_op->put_inode->
1070 * ext2_discard_prealloc->ext2_free_blocks->lock_super->DEADLOCK.
1071 *
1072 * In this case we return -1 to tell the caller that we baled.
1073 */
1074 static int shrink_dcache_memory(struct shrinker *shrink, int nr, gfp_t gfp_mask)
1075 {
1076 if (nr) {
1077 if (!(gfp_mask & __GFP_FS))
1078 return -1;
1079 prune_dcache(nr);
1080 }
1081
1082 return (dentry_stat.nr_unused / 100) * sysctl_vfs_cache_pressure;
1083 }
1084
1085 static struct shrinker dcache_shrinker = {
1086 .shrink = shrink_dcache_memory,
1087 .seeks = DEFAULT_SEEKS,
1088 };
1089
1090 /**
1091 * d_alloc - allocate a dcache entry
1092 * @parent: parent of entry to allocate
1093 * @name: qstr of the name
1094 *
1095 * Allocates a dentry. It returns %NULL if there is insufficient memory
1096 * available. On a success the dentry is returned. The name passed in is
1097 * copied and the copy passed in may be reused after this call.
1098 */
1099
1100 struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
1101 {
1102 struct dentry *dentry;
1103 char *dname;
1104
1105 dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
1106 if (!dentry)
1107 return NULL;
1108
1109 if (name->len > DNAME_INLINE_LEN-1) {
1110 dname = kmalloc(name->len + 1, GFP_KERNEL);
1111 if (!dname) {
1112 kmem_cache_free(dentry_cache, dentry);
1113 return NULL;
1114 }
1115 } else {
1116 dname = dentry->d_iname;
1117 }
1118 dentry->d_name.name = dname;
1119
1120 dentry->d_name.len = name->len;
1121 dentry->d_name.hash = name->hash;
1122 memcpy(dname, name->name, name->len);
1123 dname[name->len] = 0;
1124
1125 dentry->d_count = 1;
1126 dentry->d_flags = DCACHE_UNHASHED;
1127 spin_lock_init(&dentry->d_lock);
1128 dentry->d_inode = NULL;
1129 dentry->d_parent = NULL;
1130 dentry->d_sb = NULL;
1131 dentry->d_op = NULL;
1132 dentry->d_fsdata = NULL;
1133 dentry->d_mounted = 0;
1134 INIT_HLIST_NODE(&dentry->d_hash);
1135 INIT_LIST_HEAD(&dentry->d_lru);
1136 INIT_LIST_HEAD(&dentry->d_subdirs);
1137 INIT_LIST_HEAD(&dentry->d_alias);
1138
1139 if (parent) {
1140 dentry->d_parent = dget(parent);
1141 dentry->d_sb = parent->d_sb;
1142 } else {
1143 INIT_LIST_HEAD(&dentry->d_u.d_child);
1144 }
1145
1146 spin_lock(&dcache_lock);
1147 if (parent)
1148 list_add(&dentry->d_u.d_child, &parent->d_subdirs);
1149 spin_unlock(&dcache_lock);
1150
1151 this_cpu_inc(nr_dentry);
1152
1153 return dentry;
1154 }
1155 EXPORT_SYMBOL(d_alloc);
1156
1157 struct dentry *d_alloc_name(struct dentry *parent, const char *name)
1158 {
1159 struct qstr q;
1160
1161 q.name = name;
1162 q.len = strlen(name);
1163 q.hash = full_name_hash(q.name, q.len);
1164 return d_alloc(parent, &q);
1165 }
1166 EXPORT_SYMBOL(d_alloc_name);
1167
1168 /* the caller must hold dcache_lock */
1169 static void __d_instantiate(struct dentry *dentry, struct inode *inode)
1170 {
1171 if (inode)
1172 list_add(&dentry->d_alias, &inode->i_dentry);
1173 dentry->d_inode = inode;
1174 fsnotify_d_instantiate(dentry, inode);
1175 }
1176
1177 /**
1178 * d_instantiate - fill in inode information for a dentry
1179 * @entry: dentry to complete
1180 * @inode: inode to attach to this dentry
1181 *
1182 * Fill in inode information in the entry.
1183 *
1184 * This turns negative dentries into productive full members
1185 * of society.
1186 *
1187 * NOTE! This assumes that the inode count has been incremented
1188 * (or otherwise set) by the caller to indicate that it is now
1189 * in use by the dcache.
1190 */
1191
1192 void d_instantiate(struct dentry *entry, struct inode * inode)
1193 {
1194 BUG_ON(!list_empty(&entry->d_alias));
1195 spin_lock(&dcache_lock);
1196 __d_instantiate(entry, inode);
1197 spin_unlock(&dcache_lock);
1198 security_d_instantiate(entry, inode);
1199 }
1200 EXPORT_SYMBOL(d_instantiate);
1201
1202 /**
1203 * d_instantiate_unique - instantiate a non-aliased dentry
1204 * @entry: dentry to instantiate
1205 * @inode: inode to attach to this dentry
1206 *
1207 * Fill in inode information in the entry. On success, it returns NULL.
1208 * If an unhashed alias of "entry" already exists, then we return the
1209 * aliased dentry instead and drop one reference to inode.
1210 *
1211 * Note that in order to avoid conflicts with rename() etc, the caller
1212 * had better be holding the parent directory semaphore.
1213 *
1214 * This also assumes that the inode count has been incremented
1215 * (or otherwise set) by the caller to indicate that it is now
1216 * in use by the dcache.
1217 */
1218 static struct dentry *__d_instantiate_unique(struct dentry *entry,
1219 struct inode *inode)
1220 {
1221 struct dentry *alias;
1222 int len = entry->d_name.len;
1223 const char *name = entry->d_name.name;
1224 unsigned int hash = entry->d_name.hash;
1225
1226 if (!inode) {
1227 __d_instantiate(entry, NULL);
1228 return NULL;
1229 }
1230
1231 list_for_each_entry(alias, &inode->i_dentry, d_alias) {
1232 struct qstr *qstr = &alias->d_name;
1233
1234 if (qstr->hash != hash)
1235 continue;
1236 if (alias->d_parent != entry->d_parent)
1237 continue;
1238 if (qstr->len != len)
1239 continue;
1240 if (memcmp(qstr->name, name, len))
1241 continue;
1242 dget_locked(alias);
1243 return alias;
1244 }
1245
1246 __d_instantiate(entry, inode);
1247 return NULL;
1248 }
1249
1250 struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode)
1251 {
1252 struct dentry *result;
1253
1254 BUG_ON(!list_empty(&entry->d_alias));
1255
1256 spin_lock(&dcache_lock);
1257 result = __d_instantiate_unique(entry, inode);
1258 spin_unlock(&dcache_lock);
1259
1260 if (!result) {
1261 security_d_instantiate(entry, inode);
1262 return NULL;
1263 }
1264
1265 BUG_ON(!d_unhashed(result));
1266 iput(inode);
1267 return result;
1268 }
1269
1270 EXPORT_SYMBOL(d_instantiate_unique);
1271
1272 /**
1273 * d_alloc_root - allocate root dentry
1274 * @root_inode: inode to allocate the root for
1275 *
1276 * Allocate a root ("/") dentry for the inode given. The inode is
1277 * instantiated and returned. %NULL is returned if there is insufficient
1278 * memory or the inode passed is %NULL.
1279 */
1280
1281 struct dentry * d_alloc_root(struct inode * root_inode)
1282 {
1283 struct dentry *res = NULL;
1284
1285 if (root_inode) {
1286 static const struct qstr name = { .name = "/", .len = 1 };
1287
1288 res = d_alloc(NULL, &name);
1289 if (res) {
1290 res->d_sb = root_inode->i_sb;
1291 res->d_parent = res;
1292 d_instantiate(res, root_inode);
1293 }
1294 }
1295 return res;
1296 }
1297 EXPORT_SYMBOL(d_alloc_root);
1298
1299 static inline struct hlist_head *d_hash(struct dentry *parent,
1300 unsigned long hash)
1301 {
1302 hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES;
1303 hash = hash ^ ((hash ^ GOLDEN_RATIO_PRIME) >> D_HASHBITS);
1304 return dentry_hashtable + (hash & D_HASHMASK);
1305 }
1306
1307 /**
1308 * d_obtain_alias - find or allocate a dentry for a given inode
1309 * @inode: inode to allocate the dentry for
1310 *
1311 * Obtain a dentry for an inode resulting from NFS filehandle conversion or
1312 * similar open by handle operations. The returned dentry may be anonymous,
1313 * or may have a full name (if the inode was already in the cache).
1314 *
1315 * When called on a directory inode, we must ensure that the inode only ever
1316 * has one dentry. If a dentry is found, that is returned instead of
1317 * allocating a new one.
1318 *
1319 * On successful return, the reference to the inode has been transferred
1320 * to the dentry. In case of an error the reference on the inode is released.
1321 * To make it easier to use in export operations a %NULL or IS_ERR inode may
1322 * be passed in and will be the error will be propagate to the return value,
1323 * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
1324 */
1325 struct dentry *d_obtain_alias(struct inode *inode)
1326 {
1327 static const struct qstr anonstring = { .name = "" };
1328 struct dentry *tmp;
1329 struct dentry *res;
1330
1331 if (!inode)
1332 return ERR_PTR(-ESTALE);
1333 if (IS_ERR(inode))
1334 return ERR_CAST(inode);
1335
1336 res = d_find_alias(inode);
1337 if (res)
1338 goto out_iput;
1339
1340 tmp = d_alloc(NULL, &anonstring);
1341 if (!tmp) {
1342 res = ERR_PTR(-ENOMEM);
1343 goto out_iput;
1344 }
1345 tmp->d_parent = tmp; /* make sure dput doesn't croak */
1346
1347 spin_lock(&dcache_lock);
1348 res = __d_find_alias(inode, 0);
1349 if (res) {
1350 spin_unlock(&dcache_lock);
1351 dput(tmp);
1352 goto out_iput;
1353 }
1354
1355 /* attach a disconnected dentry */
1356 spin_lock(&tmp->d_lock);
1357 tmp->d_sb = inode->i_sb;
1358 tmp->d_inode = inode;
1359 tmp->d_flags |= DCACHE_DISCONNECTED;
1360 tmp->d_flags &= ~DCACHE_UNHASHED;
1361 list_add(&tmp->d_alias, &inode->i_dentry);
1362 spin_lock(&dcache_hash_lock);
1363 hlist_add_head(&tmp->d_hash, &inode->i_sb->s_anon);
1364 spin_unlock(&dcache_hash_lock);
1365 spin_unlock(&tmp->d_lock);
1366
1367 spin_unlock(&dcache_lock);
1368 return tmp;
1369
1370 out_iput:
1371 iput(inode);
1372 return res;
1373 }
1374 EXPORT_SYMBOL(d_obtain_alias);
1375
1376 /**
1377 * d_splice_alias - splice a disconnected dentry into the tree if one exists
1378 * @inode: the inode which may have a disconnected dentry
1379 * @dentry: a negative dentry which we want to point to the inode.
1380 *
1381 * If inode is a directory and has a 'disconnected' dentry (i.e. IS_ROOT and
1382 * DCACHE_DISCONNECTED), then d_move that in place of the given dentry
1383 * and return it, else simply d_add the inode to the dentry and return NULL.
1384 *
1385 * This is needed in the lookup routine of any filesystem that is exportable
1386 * (via knfsd) so that we can build dcache paths to directories effectively.
1387 *
1388 * If a dentry was found and moved, then it is returned. Otherwise NULL
1389 * is returned. This matches the expected return value of ->lookup.
1390 *
1391 */
1392 struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
1393 {
1394 struct dentry *new = NULL;
1395
1396 if (inode && S_ISDIR(inode->i_mode)) {
1397 spin_lock(&dcache_lock);
1398 new = __d_find_alias(inode, 1);
1399 if (new) {
1400 BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));
1401 spin_unlock(&dcache_lock);
1402 security_d_instantiate(new, inode);
1403 d_move(new, dentry);
1404 iput(inode);
1405 } else {
1406 /* already taking dcache_lock, so d_add() by hand */
1407 __d_instantiate(dentry, inode);
1408 spin_unlock(&dcache_lock);
1409 security_d_instantiate(dentry, inode);
1410 d_rehash(dentry);
1411 }
1412 } else
1413 d_add(dentry, inode);
1414 return new;
1415 }
1416 EXPORT_SYMBOL(d_splice_alias);
1417
1418 /**
1419 * d_add_ci - lookup or allocate new dentry with case-exact name
1420 * @inode: the inode case-insensitive lookup has found
1421 * @dentry: the negative dentry that was passed to the parent's lookup func
1422 * @name: the case-exact name to be associated with the returned dentry
1423 *
1424 * This is to avoid filling the dcache with case-insensitive names to the
1425 * same inode, only the actual correct case is stored in the dcache for
1426 * case-insensitive filesystems.
1427 *
1428 * For a case-insensitive lookup match and if the the case-exact dentry
1429 * already exists in in the dcache, use it and return it.
1430 *
1431 * If no entry exists with the exact case name, allocate new dentry with
1432 * the exact case, and return the spliced entry.
1433 */
1434 struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
1435 struct qstr *name)
1436 {
1437 int error;
1438 struct dentry *found;
1439 struct dentry *new;
1440
1441 /*
1442 * First check if a dentry matching the name already exists,
1443 * if not go ahead and create it now.
1444 */
1445 found = d_hash_and_lookup(dentry->d_parent, name);
1446 if (!found) {
1447 new = d_alloc(dentry->d_parent, name);
1448 if (!new) {
1449 error = -ENOMEM;
1450 goto err_out;
1451 }
1452
1453 found = d_splice_alias(inode, new);
1454 if (found) {
1455 dput(new);
1456 return found;
1457 }
1458 return new;
1459 }
1460
1461 /*
1462 * If a matching dentry exists, and it's not negative use it.
1463 *
1464 * Decrement the reference count to balance the iget() done
1465 * earlier on.
1466 */
1467 if (found->d_inode) {
1468 if (unlikely(found->d_inode != inode)) {
1469 /* This can't happen because bad inodes are unhashed. */
1470 BUG_ON(!is_bad_inode(inode));
1471 BUG_ON(!is_bad_inode(found->d_inode));
1472 }
1473 iput(inode);
1474 return found;
1475 }
1476
1477 /*
1478 * Negative dentry: instantiate it unless the inode is a directory and
1479 * already has a dentry.
1480 */
1481 spin_lock(&dcache_lock);
1482 if (!S_ISDIR(inode->i_mode) || list_empty(&inode->i_dentry)) {
1483 __d_instantiate(found, inode);
1484 spin_unlock(&dcache_lock);
1485 security_d_instantiate(found, inode);
1486 return found;
1487 }
1488
1489 /*
1490 * In case a directory already has a (disconnected) entry grab a
1491 * reference to it, move it in place and use it.
1492 */
1493 new = list_entry(inode->i_dentry.next, struct dentry, d_alias);
1494 dget_locked(new);
1495 spin_unlock(&dcache_lock);
1496 security_d_instantiate(found, inode);
1497 d_move(new, found);
1498 iput(inode);
1499 dput(found);
1500 return new;
1501
1502 err_out:
1503 iput(inode);
1504 return ERR_PTR(error);
1505 }
1506 EXPORT_SYMBOL(d_add_ci);
1507
1508 /**
1509 * d_lookup - search for a dentry
1510 * @parent: parent dentry
1511 * @name: qstr of name we wish to find
1512 * Returns: dentry, or NULL
1513 *
1514 * d_lookup searches the children of the parent dentry for the name in
1515 * question. If the dentry is found its reference count is incremented and the
1516 * dentry is returned. The caller must use dput to free the entry when it has
1517 * finished using it. %NULL is returned if the dentry does not exist.
1518 */
1519 struct dentry * d_lookup(struct dentry * parent, struct qstr * name)
1520 {
1521 struct dentry * dentry = NULL;
1522 unsigned long seq;
1523
1524 do {
1525 seq = read_seqbegin(&rename_lock);
1526 dentry = __d_lookup(parent, name);
1527 if (dentry)
1528 break;
1529 } while (read_seqretry(&rename_lock, seq));
1530 return dentry;
1531 }
1532 EXPORT_SYMBOL(d_lookup);
1533
1534 /*
1535 * __d_lookup - search for a dentry (racy)
1536 * @parent: parent dentry
1537 * @name: qstr of name we wish to find
1538 * Returns: dentry, or NULL
1539 *
1540 * __d_lookup is like d_lookup, however it may (rarely) return a
1541 * false-negative result due to unrelated rename activity.
1542 *
1543 * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
1544 * however it must be used carefully, eg. with a following d_lookup in
1545 * the case of failure.
1546 *
1547 * __d_lookup callers must be commented.
1548 */
1549 struct dentry * __d_lookup(struct dentry * parent, struct qstr * name)
1550 {
1551 unsigned int len = name->len;
1552 unsigned int hash = name->hash;
1553 const unsigned char *str = name->name;
1554 struct hlist_head *head = d_hash(parent,hash);
1555 struct dentry *found = NULL;
1556 struct hlist_node *node;
1557 struct dentry *dentry;
1558
1559 /*
1560 * The hash list is protected using RCU.
1561 *
1562 * Take d_lock when comparing a candidate dentry, to avoid races
1563 * with d_move().
1564 *
1565 * It is possible that concurrent renames can mess up our list
1566 * walk here and result in missing our dentry, resulting in the
1567 * false-negative result. d_lookup() protects against concurrent
1568 * renames using rename_lock seqlock.
1569 *
1570 * See Documentation/vfs/dcache-locking.txt for more details.
1571 */
1572 rcu_read_lock();
1573
1574 hlist_for_each_entry_rcu(dentry, node, head, d_hash) {
1575 struct qstr *qstr;
1576
1577 if (dentry->d_name.hash != hash)
1578 continue;
1579 if (dentry->d_parent != parent)
1580 continue;
1581
1582 spin_lock(&dentry->d_lock);
1583
1584 /*
1585 * Recheck the dentry after taking the lock - d_move may have
1586 * changed things. Don't bother checking the hash because
1587 * we're about to compare the whole name anyway.
1588 */
1589 if (dentry->d_parent != parent)
1590 goto next;
1591
1592 /* non-existing due to RCU? */
1593 if (d_unhashed(dentry))
1594 goto next;
1595
1596 /*
1597 * It is safe to compare names since d_move() cannot
1598 * change the qstr (protected by d_lock).
1599 */
1600 qstr = &dentry->d_name;
1601 if (parent->d_op && parent->d_op->d_compare) {
1602 if (parent->d_op->d_compare(parent, parent->d_inode,
1603 dentry, dentry->d_inode,
1604 qstr->len, qstr->name, name))
1605 goto next;
1606 } else {
1607 if (qstr->len != len)
1608 goto next;
1609 if (memcmp(qstr->name, str, len))
1610 goto next;
1611 }
1612
1613 dentry->d_count++;
1614 found = dentry;
1615 spin_unlock(&dentry->d_lock);
1616 break;
1617 next:
1618 spin_unlock(&dentry->d_lock);
1619 }
1620 rcu_read_unlock();
1621
1622 return found;
1623 }
1624
1625 /**
1626 * d_hash_and_lookup - hash the qstr then search for a dentry
1627 * @dir: Directory to search in
1628 * @name: qstr of name we wish to find
1629 *
1630 * On hash failure or on lookup failure NULL is returned.
1631 */
1632 struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
1633 {
1634 struct dentry *dentry = NULL;
1635
1636 /*
1637 * Check for a fs-specific hash function. Note that we must
1638 * calculate the standard hash first, as the d_op->d_hash()
1639 * routine may choose to leave the hash value unchanged.
1640 */
1641 name->hash = full_name_hash(name->name, name->len);
1642 if (dir->d_op && dir->d_op->d_hash) {
1643 if (dir->d_op->d_hash(dir, dir->d_inode, name) < 0)
1644 goto out;
1645 }
1646 dentry = d_lookup(dir, name);
1647 out:
1648 return dentry;
1649 }
1650
1651 /**
1652 * d_validate - verify dentry provided from insecure source (deprecated)
1653 * @dentry: The dentry alleged to be valid child of @dparent
1654 * @dparent: The parent dentry (known to be valid)
1655 *
1656 * An insecure source has sent us a dentry, here we verify it and dget() it.
1657 * This is used by ncpfs in its readdir implementation.
1658 * Zero is returned in the dentry is invalid.
1659 *
1660 * This function is slow for big directories, and deprecated, do not use it.
1661 */
1662 int d_validate(struct dentry *dentry, struct dentry *dparent)
1663 {
1664 struct dentry *child;
1665
1666 spin_lock(&dcache_lock);
1667 list_for_each_entry(child, &dparent->d_subdirs, d_u.d_child) {
1668 if (dentry == child) {
1669 __dget_locked(dentry);
1670 spin_unlock(&dcache_lock);
1671 return 1;
1672 }
1673 }
1674 spin_unlock(&dcache_lock);
1675
1676 return 0;
1677 }
1678 EXPORT_SYMBOL(d_validate);
1679
1680 /*
1681 * When a file is deleted, we have two options:
1682 * - turn this dentry into a negative dentry
1683 * - unhash this dentry and free it.
1684 *
1685 * Usually, we want to just turn this into
1686 * a negative dentry, but if anybody else is
1687 * currently using the dentry or the inode
1688 * we can't do that and we fall back on removing
1689 * it from the hash queues and waiting for
1690 * it to be deleted later when it has no users
1691 */
1692
1693 /**
1694 * d_delete - delete a dentry
1695 * @dentry: The dentry to delete
1696 *
1697 * Turn the dentry into a negative dentry if possible, otherwise
1698 * remove it from the hash queues so it can be deleted later
1699 */
1700
1701 void d_delete(struct dentry * dentry)
1702 {
1703 int isdir = 0;
1704 /*
1705 * Are we the only user?
1706 */
1707 spin_lock(&dcache_lock);
1708 spin_lock(&dentry->d_lock);
1709 isdir = S_ISDIR(dentry->d_inode->i_mode);
1710 if (dentry->d_count == 1) {
1711 dentry->d_flags &= ~DCACHE_CANT_MOUNT;
1712 dentry_iput(dentry);
1713 fsnotify_nameremove(dentry, isdir);
1714 return;
1715 }
1716
1717 if (!d_unhashed(dentry))
1718 __d_drop(dentry);
1719
1720 spin_unlock(&dentry->d_lock);
1721 spin_unlock(&dcache_lock);
1722
1723 fsnotify_nameremove(dentry, isdir);
1724 }
1725 EXPORT_SYMBOL(d_delete);
1726
1727 static void __d_rehash(struct dentry * entry, struct hlist_head *list)
1728 {
1729
1730 entry->d_flags &= ~DCACHE_UNHASHED;
1731 hlist_add_head_rcu(&entry->d_hash, list);
1732 }
1733
1734 static void _d_rehash(struct dentry * entry)
1735 {
1736 __d_rehash(entry, d_hash(entry->d_parent, entry->d_name.hash));
1737 }
1738
1739 /**
1740 * d_rehash - add an entry back to the hash
1741 * @entry: dentry to add to the hash
1742 *
1743 * Adds a dentry to the hash according to its name.
1744 */
1745
1746 void d_rehash(struct dentry * entry)
1747 {
1748 spin_lock(&dcache_lock);
1749 spin_lock(&entry->d_lock);
1750 spin_lock(&dcache_hash_lock);
1751 _d_rehash(entry);
1752 spin_unlock(&dcache_hash_lock);
1753 spin_unlock(&entry->d_lock);
1754 spin_unlock(&dcache_lock);
1755 }
1756 EXPORT_SYMBOL(d_rehash);
1757
1758 /**
1759 * dentry_update_name_case - update case insensitive dentry with a new name
1760 * @dentry: dentry to be updated
1761 * @name: new name
1762 *
1763 * Update a case insensitive dentry with new case of name.
1764 *
1765 * dentry must have been returned by d_lookup with name @name. Old and new
1766 * name lengths must match (ie. no d_compare which allows mismatched name
1767 * lengths).
1768 *
1769 * Parent inode i_mutex must be held over d_lookup and into this call (to
1770 * keep renames and concurrent inserts, and readdir(2) away).
1771 */
1772 void dentry_update_name_case(struct dentry *dentry, struct qstr *name)
1773 {
1774 BUG_ON(!mutex_is_locked(&dentry->d_inode->i_mutex));
1775 BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */
1776
1777 spin_lock(&dcache_lock);
1778 spin_lock(&dentry->d_lock);
1779 memcpy((unsigned char *)dentry->d_name.name, name->name, name->len);
1780 spin_unlock(&dentry->d_lock);
1781 spin_unlock(&dcache_lock);
1782 }
1783 EXPORT_SYMBOL(dentry_update_name_case);
1784
1785 /*
1786 * When switching names, the actual string doesn't strictly have to
1787 * be preserved in the target - because we're dropping the target
1788 * anyway. As such, we can just do a simple memcpy() to copy over
1789 * the new name before we switch.
1790 *
1791 * Note that we have to be a lot more careful about getting the hash
1792 * switched - we have to switch the hash value properly even if it
1793 * then no longer matches the actual (corrupted) string of the target.
1794 * The hash value has to match the hash queue that the dentry is on..
1795 */
1796 static void switch_names(struct dentry *dentry, struct dentry *target)
1797 {
1798 if (dname_external(target)) {
1799 if (dname_external(dentry)) {
1800 /*
1801 * Both external: swap the pointers
1802 */
1803 swap(target->d_name.name, dentry->d_name.name);
1804 } else {
1805 /*
1806 * dentry:internal, target:external. Steal target's
1807 * storage and make target internal.
1808 */
1809 memcpy(target->d_iname, dentry->d_name.name,
1810 dentry->d_name.len + 1);
1811 dentry->d_name.name = target->d_name.name;
1812 target->d_name.name = target->d_iname;
1813 }
1814 } else {
1815 if (dname_external(dentry)) {
1816 /*
1817 * dentry:external, target:internal. Give dentry's
1818 * storage to target and make dentry internal
1819 */
1820 memcpy(dentry->d_iname, target->d_name.name,
1821 target->d_name.len + 1);
1822 target->d_name.name = dentry->d_name.name;
1823 dentry->d_name.name = dentry->d_iname;
1824 } else {
1825 /*
1826 * Both are internal. Just copy target to dentry
1827 */
1828 memcpy(dentry->d_iname, target->d_name.name,
1829 target->d_name.len + 1);
1830 dentry->d_name.len = target->d_name.len;
1831 return;
1832 }
1833 }
1834 swap(dentry->d_name.len, target->d_name.len);
1835 }
1836
1837 /*
1838 * We cannibalize "target" when moving dentry on top of it,
1839 * because it's going to be thrown away anyway. We could be more
1840 * polite about it, though.
1841 *
1842 * This forceful removal will result in ugly /proc output if
1843 * somebody holds a file open that got deleted due to a rename.
1844 * We could be nicer about the deleted file, and let it show
1845 * up under the name it had before it was deleted rather than
1846 * under the original name of the file that was moved on top of it.
1847 */
1848
1849 /*
1850 * d_move_locked - move a dentry
1851 * @dentry: entry to move
1852 * @target: new dentry
1853 *
1854 * Update the dcache to reflect the move of a file name. Negative
1855 * dcache entries should not be moved in this way.
1856 */
1857 static void d_move_locked(struct dentry * dentry, struct dentry * target)
1858 {
1859 if (!dentry->d_inode)
1860 printk(KERN_WARNING "VFS: moving negative dcache entry\n");
1861
1862 write_seqlock(&rename_lock);
1863 /*
1864 * XXXX: do we really need to take target->d_lock?
1865 */
1866 if (target < dentry) {
1867 spin_lock(&target->d_lock);
1868 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1869 } else {
1870 spin_lock(&dentry->d_lock);
1871 spin_lock_nested(&target->d_lock, DENTRY_D_LOCK_NESTED);
1872 }
1873
1874 /* Move the dentry to the target hash queue, if on different bucket */
1875 spin_lock(&dcache_hash_lock);
1876 if (!d_unhashed(dentry))
1877 hlist_del_rcu(&dentry->d_hash);
1878 __d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
1879 spin_unlock(&dcache_hash_lock);
1880
1881 /* Unhash the target: dput() will then get rid of it */
1882 __d_drop(target);
1883
1884 list_del(&dentry->d_u.d_child);
1885 list_del(&target->d_u.d_child);
1886
1887 /* Switch the names.. */
1888 switch_names(dentry, target);
1889 swap(dentry->d_name.hash, target->d_name.hash);
1890
1891 /* ... and switch the parents */
1892 if (IS_ROOT(dentry)) {
1893 dentry->d_parent = target->d_parent;
1894 target->d_parent = target;
1895 INIT_LIST_HEAD(&target->d_u.d_child);
1896 } else {
1897 swap(dentry->d_parent, target->d_parent);
1898
1899 /* And add them back to the (new) parent lists */
1900 list_add(&target->d_u.d_child, &target->d_parent->d_subdirs);
1901 }
1902
1903 list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
1904 spin_unlock(&target->d_lock);
1905 fsnotify_d_move(dentry);
1906 spin_unlock(&dentry->d_lock);
1907 write_sequnlock(&rename_lock);
1908 }
1909
1910 /**
1911 * d_move - move a dentry
1912 * @dentry: entry to move
1913 * @target: new dentry
1914 *
1915 * Update the dcache to reflect the move of a file name. Negative
1916 * dcache entries should not be moved in this way.
1917 */
1918
1919 void d_move(struct dentry * dentry, struct dentry * target)
1920 {
1921 spin_lock(&dcache_lock);
1922 d_move_locked(dentry, target);
1923 spin_unlock(&dcache_lock);
1924 }
1925 EXPORT_SYMBOL(d_move);
1926
1927 /**
1928 * d_ancestor - search for an ancestor
1929 * @p1: ancestor dentry
1930 * @p2: child dentry
1931 *
1932 * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
1933 * an ancestor of p2, else NULL.
1934 */
1935 struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
1936 {
1937 struct dentry *p;
1938
1939 for (p = p2; !IS_ROOT(p); p = p->d_parent) {
1940 if (p->d_parent == p1)
1941 return p;
1942 }
1943 return NULL;
1944 }
1945
1946 /*
1947 * This helper attempts to cope with remotely renamed directories
1948 *
1949 * It assumes that the caller is already holding
1950 * dentry->d_parent->d_inode->i_mutex and the dcache_lock
1951 *
1952 * Note: If ever the locking in lock_rename() changes, then please
1953 * remember to update this too...
1954 */
1955 static struct dentry *__d_unalias(struct dentry *dentry, struct dentry *alias)
1956 __releases(dcache_lock)
1957 {
1958 struct mutex *m1 = NULL, *m2 = NULL;
1959 struct dentry *ret;
1960
1961 /* If alias and dentry share a parent, then no extra locks required */
1962 if (alias->d_parent == dentry->d_parent)
1963 goto out_unalias;
1964
1965 /* Check for loops */
1966 ret = ERR_PTR(-ELOOP);
1967 if (d_ancestor(alias, dentry))
1968 goto out_err;
1969
1970 /* See lock_rename() */
1971 ret = ERR_PTR(-EBUSY);
1972 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
1973 goto out_err;
1974 m1 = &dentry->d_sb->s_vfs_rename_mutex;
1975 if (!mutex_trylock(&alias->d_parent->d_inode->i_mutex))
1976 goto out_err;
1977 m2 = &alias->d_parent->d_inode->i_mutex;
1978 out_unalias:
1979 d_move_locked(alias, dentry);
1980 ret = alias;
1981 out_err:
1982 spin_unlock(&dcache_lock);
1983 if (m2)
1984 mutex_unlock(m2);
1985 if (m1)
1986 mutex_unlock(m1);
1987 return ret;
1988 }
1989
1990 /*
1991 * Prepare an anonymous dentry for life in the superblock's dentry tree as a
1992 * named dentry in place of the dentry to be replaced.
1993 */
1994 static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
1995 {
1996 struct dentry *dparent, *aparent;
1997
1998 switch_names(dentry, anon);
1999 swap(dentry->d_name.hash, anon->d_name.hash);
2000
2001 dparent = dentry->d_parent;
2002 aparent = anon->d_parent;
2003
2004 dentry->d_parent = (aparent == anon) ? dentry : aparent;
2005 list_del(&dentry->d_u.d_child);
2006 if (!IS_ROOT(dentry))
2007 list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
2008 else
2009 INIT_LIST_HEAD(&dentry->d_u.d_child);
2010
2011 anon->d_parent = (dparent == dentry) ? anon : dparent;
2012 list_del(&anon->d_u.d_child);
2013 if (!IS_ROOT(anon))
2014 list_add(&anon->d_u.d_child, &anon->d_parent->d_subdirs);
2015 else
2016 INIT_LIST_HEAD(&anon->d_u.d_child);
2017
2018 anon->d_flags &= ~DCACHE_DISCONNECTED;
2019 }
2020
2021 /**
2022 * d_materialise_unique - introduce an inode into the tree
2023 * @dentry: candidate dentry
2024 * @inode: inode to bind to the dentry, to which aliases may be attached
2025 *
2026 * Introduces an dentry into the tree, substituting an extant disconnected
2027 * root directory alias in its place if there is one
2028 */
2029 struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
2030 {
2031 struct dentry *actual;
2032
2033 BUG_ON(!d_unhashed(dentry));
2034
2035 spin_lock(&dcache_lock);
2036
2037 if (!inode) {
2038 actual = dentry;
2039 __d_instantiate(dentry, NULL);
2040 goto found_lock;
2041 }
2042
2043 if (S_ISDIR(inode->i_mode)) {
2044 struct dentry *alias;
2045
2046 /* Does an aliased dentry already exist? */
2047 alias = __d_find_alias(inode, 0);
2048 if (alias) {
2049 actual = alias;
2050 /* Is this an anonymous mountpoint that we could splice
2051 * into our tree? */
2052 if (IS_ROOT(alias)) {
2053 spin_lock(&alias->d_lock);
2054 __d_materialise_dentry(dentry, alias);
2055 __d_drop(alias);
2056 goto found;
2057 }
2058 /* Nope, but we must(!) avoid directory aliasing */
2059 actual = __d_unalias(dentry, alias);
2060 if (IS_ERR(actual))
2061 dput(alias);
2062 goto out_nolock;
2063 }
2064 }
2065
2066 /* Add a unique reference */
2067 actual = __d_instantiate_unique(dentry, inode);
2068 if (!actual)
2069 actual = dentry;
2070 else if (unlikely(!d_unhashed(actual)))
2071 goto shouldnt_be_hashed;
2072
2073 found_lock:
2074 spin_lock(&actual->d_lock);
2075 found:
2076 spin_lock(&dcache_hash_lock);
2077 _d_rehash(actual);
2078 spin_unlock(&dcache_hash_lock);
2079 spin_unlock(&actual->d_lock);
2080 spin_unlock(&dcache_lock);
2081 out_nolock:
2082 if (actual == dentry) {
2083 security_d_instantiate(dentry, inode);
2084 return NULL;
2085 }
2086
2087 iput(inode);
2088 return actual;
2089
2090 shouldnt_be_hashed:
2091 spin_unlock(&dcache_lock);
2092 BUG();
2093 }
2094 EXPORT_SYMBOL_GPL(d_materialise_unique);
2095
2096 static int prepend(char **buffer, int *buflen, const char *str, int namelen)
2097 {
2098 *buflen -= namelen;
2099 if (*buflen < 0)
2100 return -ENAMETOOLONG;
2101 *buffer -= namelen;
2102 memcpy(*buffer, str, namelen);
2103 return 0;
2104 }
2105
2106 static int prepend_name(char **buffer, int *buflen, struct qstr *name)
2107 {
2108 return prepend(buffer, buflen, name->name, name->len);
2109 }
2110
2111 /**
2112 * Prepend path string to a buffer
2113 *
2114 * @path: the dentry/vfsmount to report
2115 * @root: root vfsmnt/dentry (may be modified by this function)
2116 * @buffer: pointer to the end of the buffer
2117 * @buflen: pointer to buffer length
2118 *
2119 * Caller holds the dcache_lock.
2120 *
2121 * If path is not reachable from the supplied root, then the value of
2122 * root is changed (without modifying refcounts).
2123 */
2124 static int prepend_path(const struct path *path, struct path *root,
2125 char **buffer, int *buflen)
2126 {
2127 struct dentry *dentry = path->dentry;
2128 struct vfsmount *vfsmnt = path->mnt;
2129 bool slash = false;
2130 int error = 0;
2131
2132 br_read_lock(vfsmount_lock);
2133 while (dentry != root->dentry || vfsmnt != root->mnt) {
2134 struct dentry * parent;
2135
2136 if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
2137 /* Global root? */
2138 if (vfsmnt->mnt_parent == vfsmnt) {
2139 goto global_root;
2140 }
2141 dentry = vfsmnt->mnt_mountpoint;
2142 vfsmnt = vfsmnt->mnt_parent;
2143 continue;
2144 }
2145 parent = dentry->d_parent;
2146 prefetch(parent);
2147 error = prepend_name(buffer, buflen, &dentry->d_name);
2148 if (!error)
2149 error = prepend(buffer, buflen, "/", 1);
2150 if (error)
2151 break;
2152
2153 slash = true;
2154 dentry = parent;
2155 }
2156
2157 out:
2158 if (!error && !slash)
2159 error = prepend(buffer, buflen, "/", 1);
2160
2161 br_read_unlock(vfsmount_lock);
2162 return error;
2163
2164 global_root:
2165 /*
2166 * Filesystems needing to implement special "root names"
2167 * should do so with ->d_dname()
2168 */
2169 if (IS_ROOT(dentry) &&
2170 (dentry->d_name.len != 1 || dentry->d_name.name[0] != '/')) {
2171 WARN(1, "Root dentry has weird name <%.*s>\n",
2172 (int) dentry->d_name.len, dentry->d_name.name);
2173 }
2174 root->mnt = vfsmnt;
2175 root->dentry = dentry;
2176 goto out;
2177 }
2178
2179 /**
2180 * __d_path - return the path of a dentry
2181 * @path: the dentry/vfsmount to report
2182 * @root: root vfsmnt/dentry (may be modified by this function)
2183 * @buf: buffer to return value in
2184 * @buflen: buffer length
2185 *
2186 * Convert a dentry into an ASCII path name.
2187 *
2188 * Returns a pointer into the buffer or an error code if the
2189 * path was too long.
2190 *
2191 * "buflen" should be positive.
2192 *
2193 * If path is not reachable from the supplied root, then the value of
2194 * root is changed (without modifying refcounts).
2195 */
2196 char *__d_path(const struct path *path, struct path *root,
2197 char *buf, int buflen)
2198 {
2199 char *res = buf + buflen;
2200 int error;
2201
2202 prepend(&res, &buflen, "\0", 1);
2203 spin_lock(&dcache_lock);
2204 error = prepend_path(path, root, &res, &buflen);
2205 spin_unlock(&dcache_lock);
2206
2207 if (error)
2208 return ERR_PTR(error);
2209 return res;
2210 }
2211
2212 /*
2213 * same as __d_path but appends "(deleted)" for unlinked files.
2214 */
2215 static int path_with_deleted(const struct path *path, struct path *root,
2216 char **buf, int *buflen)
2217 {
2218 prepend(buf, buflen, "\0", 1);
2219 if (d_unlinked(path->dentry)) {
2220 int error = prepend(buf, buflen, " (deleted)", 10);
2221 if (error)
2222 return error;
2223 }
2224
2225 return prepend_path(path, root, buf, buflen);
2226 }
2227
2228 static int prepend_unreachable(char **buffer, int *buflen)
2229 {
2230 return prepend(buffer, buflen, "(unreachable)", 13);
2231 }
2232
2233 /**
2234 * d_path - return the path of a dentry
2235 * @path: path to report
2236 * @buf: buffer to return value in
2237 * @buflen: buffer length
2238 *
2239 * Convert a dentry into an ASCII path name. If the entry has been deleted
2240 * the string " (deleted)" is appended. Note that this is ambiguous.
2241 *
2242 * Returns a pointer into the buffer or an error code if the path was
2243 * too long. Note: Callers should use the returned pointer, not the passed
2244 * in buffer, to use the name! The implementation often starts at an offset
2245 * into the buffer, and may leave 0 bytes at the start.
2246 *
2247 * "buflen" should be positive.
2248 */
2249 char *d_path(const struct path *path, char *buf, int buflen)
2250 {
2251 char *res = buf + buflen;
2252 struct path root;
2253 struct path tmp;
2254 int error;
2255
2256 /*
2257 * We have various synthetic filesystems that never get mounted. On
2258 * these filesystems dentries are never used for lookup purposes, and
2259 * thus don't need to be hashed. They also don't need a name until a
2260 * user wants to identify the object in /proc/pid/fd/. The little hack
2261 * below allows us to generate a name for these objects on demand:
2262 */
2263 if (path->dentry->d_op && path->dentry->d_op->d_dname)
2264 return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
2265
2266 get_fs_root(current->fs, &root);
2267 spin_lock(&dcache_lock);
2268 tmp = root;
2269 error = path_with_deleted(path, &tmp, &res, &buflen);
2270 if (error)
2271 res = ERR_PTR(error);
2272 spin_unlock(&dcache_lock);
2273 path_put(&root);
2274 return res;
2275 }
2276 EXPORT_SYMBOL(d_path);
2277
2278 /**
2279 * d_path_with_unreachable - return the path of a dentry
2280 * @path: path to report
2281 * @buf: buffer to return value in
2282 * @buflen: buffer length
2283 *
2284 * The difference from d_path() is that this prepends "(unreachable)"
2285 * to paths which are unreachable from the current process' root.
2286 */
2287 char *d_path_with_unreachable(const struct path *path, char *buf, int buflen)
2288 {
2289 char *res = buf + buflen;
2290 struct path root;
2291 struct path tmp;
2292 int error;
2293
2294 if (path->dentry->d_op && path->dentry->d_op->d_dname)
2295 return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
2296
2297 get_fs_root(current->fs, &root);
2298 spin_lock(&dcache_lock);
2299 tmp = root;
2300 error = path_with_deleted(path, &tmp, &res, &buflen);
2301 if (!error && !path_equal(&tmp, &root))
2302 error = prepend_unreachable(&res, &buflen);
2303 spin_unlock(&dcache_lock);
2304 path_put(&root);
2305 if (error)
2306 res = ERR_PTR(error);
2307
2308 return res;
2309 }
2310
2311 /*
2312 * Helper function for dentry_operations.d_dname() members
2313 */
2314 char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen,
2315 const char *fmt, ...)
2316 {
2317 va_list args;
2318 char temp[64];
2319 int sz;
2320
2321 va_start(args, fmt);
2322 sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1;
2323 va_end(args);
2324
2325 if (sz > sizeof(temp) || sz > buflen)
2326 return ERR_PTR(-ENAMETOOLONG);
2327
2328 buffer += buflen - sz;
2329 return memcpy(buffer, temp, sz);
2330 }
2331
2332 /*
2333 * Write full pathname from the root of the filesystem into the buffer.
2334 */
2335 static char *__dentry_path(struct dentry *dentry, char *buf, int buflen)
2336 {
2337 char *end = buf + buflen;
2338 char *retval;
2339
2340 prepend(&end, &buflen, "\0", 1);
2341 if (buflen < 1)
2342 goto Elong;
2343 /* Get '/' right */
2344 retval = end-1;
2345 *retval = '/';
2346
2347 while (!IS_ROOT(dentry)) {
2348 struct dentry *parent = dentry->d_parent;
2349
2350 prefetch(parent);
2351 if ((prepend_name(&end, &buflen, &dentry->d_name) != 0) ||
2352 (prepend(&end, &buflen, "/", 1) != 0))
2353 goto Elong;
2354
2355 retval = end;
2356 dentry = parent;
2357 }
2358 return retval;
2359 Elong:
2360 return ERR_PTR(-ENAMETOOLONG);
2361 }
2362
2363 char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen)
2364 {
2365 char *retval;
2366
2367 spin_lock(&dcache_lock);
2368 retval = __dentry_path(dentry, buf, buflen);
2369 spin_unlock(&dcache_lock);
2370
2371 return retval;
2372 }
2373 EXPORT_SYMBOL(dentry_path_raw);
2374
2375 char *dentry_path(struct dentry *dentry, char *buf, int buflen)
2376 {
2377 char *p = NULL;
2378 char *retval;
2379
2380 spin_lock(&dcache_lock);
2381 if (d_unlinked(dentry)) {
2382 p = buf + buflen;
2383 if (prepend(&p, &buflen, "//deleted", 10) != 0)
2384 goto Elong;
2385 buflen++;
2386 }
2387 retval = __dentry_path(dentry, buf, buflen);
2388 spin_unlock(&dcache_lock);
2389 if (!IS_ERR(retval) && p)
2390 *p = '/'; /* restore '/' overriden with '\0' */
2391 return retval;
2392 Elong:
2393 spin_unlock(&dcache_lock);
2394 return ERR_PTR(-ENAMETOOLONG);
2395 }
2396
2397 /*
2398 * NOTE! The user-level library version returns a
2399 * character pointer. The kernel system call just
2400 * returns the length of the buffer filled (which
2401 * includes the ending '\0' character), or a negative
2402 * error value. So libc would do something like
2403 *
2404 * char *getcwd(char * buf, size_t size)
2405 * {
2406 * int retval;
2407 *
2408 * retval = sys_getcwd(buf, size);
2409 * if (retval >= 0)
2410 * return buf;
2411 * errno = -retval;
2412 * return NULL;
2413 * }
2414 */
2415 SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
2416 {
2417 int error;
2418 struct path pwd, root;
2419 char *page = (char *) __get_free_page(GFP_USER);
2420
2421 if (!page)
2422 return -ENOMEM;
2423
2424 get_fs_root_and_pwd(current->fs, &root, &pwd);
2425
2426 error = -ENOENT;
2427 spin_lock(&dcache_lock);
2428 if (!d_unlinked(pwd.dentry)) {
2429 unsigned long len;
2430 struct path tmp = root;
2431 char *cwd = page + PAGE_SIZE;
2432 int buflen = PAGE_SIZE;
2433
2434 prepend(&cwd, &buflen, "\0", 1);
2435 error = prepend_path(&pwd, &tmp, &cwd, &buflen);
2436 spin_unlock(&dcache_lock);
2437
2438 if (error)
2439 goto out;
2440
2441 /* Unreachable from current root */
2442 if (!path_equal(&tmp, &root)) {
2443 error = prepend_unreachable(&cwd, &buflen);
2444 if (error)
2445 goto out;
2446 }
2447
2448 error = -ERANGE;
2449 len = PAGE_SIZE + page - cwd;
2450 if (len <= size) {
2451 error = len;
2452 if (copy_to_user(buf, cwd, len))
2453 error = -EFAULT;
2454 }
2455 } else
2456 spin_unlock(&dcache_lock);
2457
2458 out:
2459 path_put(&pwd);
2460 path_put(&root);
2461 free_page((unsigned long) page);
2462 return error;
2463 }
2464
2465 /*
2466 * Test whether new_dentry is a subdirectory of old_dentry.
2467 *
2468 * Trivially implemented using the dcache structure
2469 */
2470
2471 /**
2472 * is_subdir - is new dentry a subdirectory of old_dentry
2473 * @new_dentry: new dentry
2474 * @old_dentry: old dentry
2475 *
2476 * Returns 1 if new_dentry is a subdirectory of the parent (at any depth).
2477 * Returns 0 otherwise.
2478 * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
2479 */
2480
2481 int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
2482 {
2483 int result;
2484 unsigned long seq;
2485
2486 if (new_dentry == old_dentry)
2487 return 1;
2488
2489 /*
2490 * Need rcu_readlock to protect against the d_parent trashing
2491 * due to d_move
2492 */
2493 rcu_read_lock();
2494 do {
2495 /* for restarting inner loop in case of seq retry */
2496 seq = read_seqbegin(&rename_lock);
2497 if (d_ancestor(old_dentry, new_dentry))
2498 result = 1;
2499 else
2500 result = 0;
2501 } while (read_seqretry(&rename_lock, seq));
2502 rcu_read_unlock();
2503
2504 return result;
2505 }
2506
2507 int path_is_under(struct path *path1, struct path *path2)
2508 {
2509 struct vfsmount *mnt = path1->mnt;
2510 struct dentry *dentry = path1->dentry;
2511 int res;
2512
2513 br_read_lock(vfsmount_lock);
2514 if (mnt != path2->mnt) {
2515 for (;;) {
2516 if (mnt->mnt_parent == mnt) {
2517 br_read_unlock(vfsmount_lock);
2518 return 0;
2519 }
2520 if (mnt->mnt_parent == path2->mnt)
2521 break;
2522 mnt = mnt->mnt_parent;
2523 }
2524 dentry = mnt->mnt_mountpoint;
2525 }
2526 res = is_subdir(dentry, path2->dentry);
2527 br_read_unlock(vfsmount_lock);
2528 return res;
2529 }
2530 EXPORT_SYMBOL(path_is_under);
2531
2532 void d_genocide(struct dentry *root)
2533 {
2534 struct dentry *this_parent = root;
2535 struct list_head *next;
2536
2537 spin_lock(&dcache_lock);
2538 repeat:
2539 next = this_parent->d_subdirs.next;
2540 resume:
2541 while (next != &this_parent->d_subdirs) {
2542 struct list_head *tmp = next;
2543 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
2544 next = tmp->next;
2545 if (d_unhashed(dentry)||!dentry->d_inode)
2546 continue;
2547 if (!list_empty(&dentry->d_subdirs)) {
2548 this_parent = dentry;
2549 goto repeat;
2550 }
2551 spin_lock(&dentry->d_lock);
2552 dentry->d_count--;
2553 spin_unlock(&dentry->d_lock);
2554 }
2555 if (this_parent != root) {
2556 next = this_parent->d_u.d_child.next;
2557 spin_lock(&this_parent->d_lock);
2558 this_parent->d_count--;
2559 spin_unlock(&this_parent->d_lock);
2560 this_parent = this_parent->d_parent;
2561 goto resume;
2562 }
2563 spin_unlock(&dcache_lock);
2564 }
2565
2566 /**
2567 * find_inode_number - check for dentry with name
2568 * @dir: directory to check
2569 * @name: Name to find.
2570 *
2571 * Check whether a dentry already exists for the given name,
2572 * and return the inode number if it has an inode. Otherwise
2573 * 0 is returned.
2574 *
2575 * This routine is used to post-process directory listings for
2576 * filesystems using synthetic inode numbers, and is necessary
2577 * to keep getcwd() working.
2578 */
2579
2580 ino_t find_inode_number(struct dentry *dir, struct qstr *name)
2581 {
2582 struct dentry * dentry;
2583 ino_t ino = 0;
2584
2585 dentry = d_hash_and_lookup(dir, name);
2586 if (dentry) {
2587 if (dentry->d_inode)
2588 ino = dentry->d_inode->i_ino;
2589 dput(dentry);
2590 }
2591 return ino;
2592 }
2593 EXPORT_SYMBOL(find_inode_number);
2594
2595 static __initdata unsigned long dhash_entries;
2596 static int __init set_dhash_entries(char *str)
2597 {
2598 if (!str)
2599 return 0;
2600 dhash_entries = simple_strtoul(str, &str, 0);
2601 return 1;
2602 }
2603 __setup("dhash_entries=", set_dhash_entries);
2604
2605 static void __init dcache_init_early(void)
2606 {
2607 int loop;
2608
2609 /* If hashes are distributed across NUMA nodes, defer
2610 * hash allocation until vmalloc space is available.
2611 */
2612 if (hashdist)
2613 return;
2614
2615 dentry_hashtable =
2616 alloc_large_system_hash("Dentry cache",
2617 sizeof(struct hlist_head),
2618 dhash_entries,
2619 13,
2620 HASH_EARLY,
2621 &d_hash_shift,
2622 &d_hash_mask,
2623 0);
2624
2625 for (loop = 0; loop < (1 << d_hash_shift); loop++)
2626 INIT_HLIST_HEAD(&dentry_hashtable[loop]);
2627 }
2628
2629 static void __init dcache_init(void)
2630 {
2631 int loop;
2632
2633 /*
2634 * A constructor could be added for stable state like the lists,
2635 * but it is probably not worth it because of the cache nature
2636 * of the dcache.
2637 */
2638 dentry_cache = KMEM_CACHE(dentry,
2639 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD);
2640
2641 register_shrinker(&dcache_shrinker);
2642
2643 /* Hash may have been set up in dcache_init_early */
2644 if (!hashdist)
2645 return;
2646
2647 dentry_hashtable =
2648 alloc_large_system_hash("Dentry cache",
2649 sizeof(struct hlist_head),
2650 dhash_entries,
2651 13,
2652 0,
2653 &d_hash_shift,
2654 &d_hash_mask,
2655 0);
2656
2657 for (loop = 0; loop < (1 << d_hash_shift); loop++)
2658 INIT_HLIST_HEAD(&dentry_hashtable[loop]);
2659 }
2660
2661 /* SLAB cache for __getname() consumers */
2662 struct kmem_cache *names_cachep __read_mostly;
2663 EXPORT_SYMBOL(names_cachep);
2664
2665 EXPORT_SYMBOL(d_genocide);
2666
2667 void __init vfs_caches_init_early(void)
2668 {
2669 dcache_init_early();
2670 inode_init_early();
2671 }
2672
2673 void __init vfs_caches_init(unsigned long mempages)
2674 {
2675 unsigned long reserve;
2676
2677 /* Base hash sizes on available memory, with a reserve equal to
2678 150% of current kernel size */
2679
2680 reserve = min((mempages - nr_free_pages()) * 3/2, mempages - 1);
2681 mempages -= reserve;
2682
2683 names_cachep = kmem_cache_create("names_cache", PATH_MAX, 0,
2684 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2685
2686 dcache_init();
2687 inode_init();
2688 files_init(mempages);
2689 mnt_init();
2690 bdev_cache_init();
2691 chrdev_init();
2692 }