]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - fs/dcache.c
UBUNTU: Ubuntu-5.11.0-22.23
[mirror_ubuntu-hirsute-kernel.git] / fs / dcache.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * fs/dcache.c
4 *
5 * Complete reimplementation
6 * (C) 1997 Thomas Schoebel-Theuer,
7 * with heavy changes by Linus Torvalds
8 */
9
10 /*
11 * Notes on the allocation strategy:
12 *
13 * The dcache is a master of the icache - whenever a dcache entry
14 * exists, the inode will always exist. "iput()" is done either when
15 * the dcache entry is deleted or garbage collected.
16 */
17
18 #include <linux/ratelimit.h>
19 #include <linux/string.h>
20 #include <linux/mm.h>
21 #include <linux/fs.h>
22 #include <linux/fscrypt.h>
23 #include <linux/fsnotify.h>
24 #include <linux/slab.h>
25 #include <linux/init.h>
26 #include <linux/hash.h>
27 #include <linux/cache.h>
28 #include <linux/export.h>
29 #include <linux/security.h>
30 #include <linux/seqlock.h>
31 #include <linux/memblock.h>
32 #include <linux/bit_spinlock.h>
33 #include <linux/rculist_bl.h>
34 #include <linux/list_lru.h>
35 #include "internal.h"
36 #include "mount.h"
37
38 /*
39 * Usage:
40 * dcache->d_inode->i_lock protects:
41 * - i_dentry, d_u.d_alias, d_inode of aliases
42 * dcache_hash_bucket lock protects:
43 * - the dcache hash table
44 * s_roots bl list spinlock protects:
45 * - the s_roots list (see __d_drop)
46 * dentry->d_sb->s_dentry_lru_lock protects:
47 * - the dcache lru lists and counters
48 * d_lock protects:
49 * - d_flags
50 * - d_name
51 * - d_lru
52 * - d_count
53 * - d_unhashed()
54 * - d_parent and d_subdirs
55 * - childrens' d_child and d_parent
56 * - d_u.d_alias, d_inode
57 *
58 * Ordering:
59 * dentry->d_inode->i_lock
60 * dentry->d_lock
61 * dentry->d_sb->s_dentry_lru_lock
62 * dcache_hash_bucket lock
63 * s_roots lock
64 *
65 * If there is an ancestor relationship:
66 * dentry->d_parent->...->d_parent->d_lock
67 * ...
68 * dentry->d_parent->d_lock
69 * dentry->d_lock
70 *
71 * If no ancestor relationship:
72 * arbitrary, since it's serialized on rename_lock
73 */
74 int sysctl_vfs_cache_pressure __read_mostly = 100;
75 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
76
77 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
78
79 EXPORT_SYMBOL(rename_lock);
80
81 static struct kmem_cache *dentry_cache __read_mostly;
82
83 const struct qstr empty_name = QSTR_INIT("", 0);
84 EXPORT_SYMBOL(empty_name);
85 const struct qstr slash_name = QSTR_INIT("/", 1);
86 EXPORT_SYMBOL(slash_name);
87
88 /*
89 * This is the single most critical data structure when it comes
90 * to the dcache: the hashtable for lookups. Somebody should try
91 * to make this good - I've just made it work.
92 *
93 * This hash-function tries to avoid losing too many bits of hash
94 * information, yet avoid using a prime hash-size or similar.
95 */
96
97 static unsigned int d_hash_shift __read_mostly;
98
99 static struct hlist_bl_head *dentry_hashtable __read_mostly;
100
101 static inline struct hlist_bl_head *d_hash(unsigned int hash)
102 {
103 return dentry_hashtable + (hash >> d_hash_shift);
104 }
105
106 #define IN_LOOKUP_SHIFT 10
107 static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
108
109 static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
110 unsigned int hash)
111 {
112 hash += (unsigned long) parent / L1_CACHE_BYTES;
113 return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
114 }
115
116
117 /* Statistics gathering. */
118 struct dentry_stat_t dentry_stat = {
119 .age_limit = 45,
120 };
121
122 static DEFINE_PER_CPU(long, nr_dentry);
123 static DEFINE_PER_CPU(long, nr_dentry_unused);
124 static DEFINE_PER_CPU(long, nr_dentry_negative);
125
126 #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
127
128 /*
129 * Here we resort to our own counters instead of using generic per-cpu counters
130 * for consistency with what the vfs inode code does. We are expected to harvest
131 * better code and performance by having our own specialized counters.
132 *
133 * Please note that the loop is done over all possible CPUs, not over all online
134 * CPUs. The reason for this is that we don't want to play games with CPUs going
135 * on and off. If one of them goes off, we will just keep their counters.
136 *
137 * glommer: See cffbc8a for details, and if you ever intend to change this,
138 * please update all vfs counters to match.
139 */
140 static long get_nr_dentry(void)
141 {
142 int i;
143 long sum = 0;
144 for_each_possible_cpu(i)
145 sum += per_cpu(nr_dentry, i);
146 return sum < 0 ? 0 : sum;
147 }
148
149 static long get_nr_dentry_unused(void)
150 {
151 int i;
152 long sum = 0;
153 for_each_possible_cpu(i)
154 sum += per_cpu(nr_dentry_unused, i);
155 return sum < 0 ? 0 : sum;
156 }
157
158 static long get_nr_dentry_negative(void)
159 {
160 int i;
161 long sum = 0;
162
163 for_each_possible_cpu(i)
164 sum += per_cpu(nr_dentry_negative, i);
165 return sum < 0 ? 0 : sum;
166 }
167
168 int proc_nr_dentry(struct ctl_table *table, int write, void *buffer,
169 size_t *lenp, loff_t *ppos)
170 {
171 dentry_stat.nr_dentry = get_nr_dentry();
172 dentry_stat.nr_unused = get_nr_dentry_unused();
173 dentry_stat.nr_negative = get_nr_dentry_negative();
174 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
175 }
176 #endif
177
178 /*
179 * Compare 2 name strings, return 0 if they match, otherwise non-zero.
180 * The strings are both count bytes long, and count is non-zero.
181 */
182 #ifdef CONFIG_DCACHE_WORD_ACCESS
183
184 #include <asm/word-at-a-time.h>
185 /*
186 * NOTE! 'cs' and 'scount' come from a dentry, so it has a
187 * aligned allocation for this particular component. We don't
188 * strictly need the load_unaligned_zeropad() safety, but it
189 * doesn't hurt either.
190 *
191 * In contrast, 'ct' and 'tcount' can be from a pathname, and do
192 * need the careful unaligned handling.
193 */
194 static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
195 {
196 unsigned long a,b,mask;
197
198 for (;;) {
199 a = read_word_at_a_time(cs);
200 b = load_unaligned_zeropad(ct);
201 if (tcount < sizeof(unsigned long))
202 break;
203 if (unlikely(a != b))
204 return 1;
205 cs += sizeof(unsigned long);
206 ct += sizeof(unsigned long);
207 tcount -= sizeof(unsigned long);
208 if (!tcount)
209 return 0;
210 }
211 mask = bytemask_from_count(tcount);
212 return unlikely(!!((a ^ b) & mask));
213 }
214
215 #else
216
217 static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
218 {
219 do {
220 if (*cs != *ct)
221 return 1;
222 cs++;
223 ct++;
224 tcount--;
225 } while (tcount);
226 return 0;
227 }
228
229 #endif
230
231 static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
232 {
233 /*
234 * Be careful about RCU walk racing with rename:
235 * use 'READ_ONCE' to fetch the name pointer.
236 *
237 * NOTE! Even if a rename will mean that the length
238 * was not loaded atomically, we don't care. The
239 * RCU walk will check the sequence count eventually,
240 * and catch it. And we won't overrun the buffer,
241 * because we're reading the name pointer atomically,
242 * and a dentry name is guaranteed to be properly
243 * terminated with a NUL byte.
244 *
245 * End result: even if 'len' is wrong, we'll exit
246 * early because the data cannot match (there can
247 * be no NUL in the ct/tcount data)
248 */
249 const unsigned char *cs = READ_ONCE(dentry->d_name.name);
250
251 return dentry_string_cmp(cs, ct, tcount);
252 }
253
254 struct external_name {
255 union {
256 atomic_t count;
257 struct rcu_head head;
258 } u;
259 unsigned char name[];
260 };
261
262 static inline struct external_name *external_name(struct dentry *dentry)
263 {
264 return container_of(dentry->d_name.name, struct external_name, name[0]);
265 }
266
267 static void __d_free(struct rcu_head *head)
268 {
269 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
270
271 kmem_cache_free(dentry_cache, dentry);
272 }
273
274 static void __d_free_external(struct rcu_head *head)
275 {
276 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
277 kfree(external_name(dentry));
278 kmem_cache_free(dentry_cache, dentry);
279 }
280
281 static inline int dname_external(const struct dentry *dentry)
282 {
283 return dentry->d_name.name != dentry->d_iname;
284 }
285
286 void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
287 {
288 spin_lock(&dentry->d_lock);
289 name->name = dentry->d_name;
290 if (unlikely(dname_external(dentry))) {
291 atomic_inc(&external_name(dentry)->u.count);
292 } else {
293 memcpy(name->inline_name, dentry->d_iname,
294 dentry->d_name.len + 1);
295 name->name.name = name->inline_name;
296 }
297 spin_unlock(&dentry->d_lock);
298 }
299 EXPORT_SYMBOL(take_dentry_name_snapshot);
300
301 void release_dentry_name_snapshot(struct name_snapshot *name)
302 {
303 if (unlikely(name->name.name != name->inline_name)) {
304 struct external_name *p;
305 p = container_of(name->name.name, struct external_name, name[0]);
306 if (unlikely(atomic_dec_and_test(&p->u.count)))
307 kfree_rcu(p, u.head);
308 }
309 }
310 EXPORT_SYMBOL(release_dentry_name_snapshot);
311
312 static inline void __d_set_inode_and_type(struct dentry *dentry,
313 struct inode *inode,
314 unsigned type_flags)
315 {
316 unsigned flags;
317
318 dentry->d_inode = inode;
319 flags = READ_ONCE(dentry->d_flags);
320 flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
321 flags |= type_flags;
322 smp_store_release(&dentry->d_flags, flags);
323 }
324
325 static inline void __d_clear_type_and_inode(struct dentry *dentry)
326 {
327 unsigned flags = READ_ONCE(dentry->d_flags);
328
329 flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
330 WRITE_ONCE(dentry->d_flags, flags);
331 dentry->d_inode = NULL;
332 if (dentry->d_flags & DCACHE_LRU_LIST)
333 this_cpu_inc(nr_dentry_negative);
334 }
335
336 static void dentry_free(struct dentry *dentry)
337 {
338 WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias));
339 if (unlikely(dname_external(dentry))) {
340 struct external_name *p = external_name(dentry);
341 if (likely(atomic_dec_and_test(&p->u.count))) {
342 call_rcu(&dentry->d_u.d_rcu, __d_free_external);
343 return;
344 }
345 }
346 /* if dentry was never visible to RCU, immediate free is OK */
347 if (dentry->d_flags & DCACHE_NORCU)
348 __d_free(&dentry->d_u.d_rcu);
349 else
350 call_rcu(&dentry->d_u.d_rcu, __d_free);
351 }
352
353 /*
354 * Release the dentry's inode, using the filesystem
355 * d_iput() operation if defined.
356 */
357 static void dentry_unlink_inode(struct dentry * dentry)
358 __releases(dentry->d_lock)
359 __releases(dentry->d_inode->i_lock)
360 {
361 struct inode *inode = dentry->d_inode;
362
363 raw_write_seqcount_begin(&dentry->d_seq);
364 __d_clear_type_and_inode(dentry);
365 hlist_del_init(&dentry->d_u.d_alias);
366 raw_write_seqcount_end(&dentry->d_seq);
367 spin_unlock(&dentry->d_lock);
368 spin_unlock(&inode->i_lock);
369 if (!inode->i_nlink)
370 fsnotify_inoderemove(inode);
371 if (dentry->d_op && dentry->d_op->d_iput)
372 dentry->d_op->d_iput(dentry, inode);
373 else
374 iput(inode);
375 }
376
377 /*
378 * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
379 * is in use - which includes both the "real" per-superblock
380 * LRU list _and_ the DCACHE_SHRINK_LIST use.
381 *
382 * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
383 * on the shrink list (ie not on the superblock LRU list).
384 *
385 * The per-cpu "nr_dentry_unused" counters are updated with
386 * the DCACHE_LRU_LIST bit.
387 *
388 * The per-cpu "nr_dentry_negative" counters are only updated
389 * when deleted from or added to the per-superblock LRU list, not
390 * from/to the shrink list. That is to avoid an unneeded dec/inc
391 * pair when moving from LRU to shrink list in select_collect().
392 *
393 * These helper functions make sure we always follow the
394 * rules. d_lock must be held by the caller.
395 */
396 #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
397 static void d_lru_add(struct dentry *dentry)
398 {
399 D_FLAG_VERIFY(dentry, 0);
400 dentry->d_flags |= DCACHE_LRU_LIST;
401 this_cpu_inc(nr_dentry_unused);
402 if (d_is_negative(dentry))
403 this_cpu_inc(nr_dentry_negative);
404 WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
405 }
406
407 static void d_lru_del(struct dentry *dentry)
408 {
409 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
410 dentry->d_flags &= ~DCACHE_LRU_LIST;
411 this_cpu_dec(nr_dentry_unused);
412 if (d_is_negative(dentry))
413 this_cpu_dec(nr_dentry_negative);
414 WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
415 }
416
417 static void d_shrink_del(struct dentry *dentry)
418 {
419 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
420 list_del_init(&dentry->d_lru);
421 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
422 this_cpu_dec(nr_dentry_unused);
423 }
424
425 static void d_shrink_add(struct dentry *dentry, struct list_head *list)
426 {
427 D_FLAG_VERIFY(dentry, 0);
428 list_add(&dentry->d_lru, list);
429 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
430 this_cpu_inc(nr_dentry_unused);
431 }
432
433 /*
434 * These can only be called under the global LRU lock, ie during the
435 * callback for freeing the LRU list. "isolate" removes it from the
436 * LRU lists entirely, while shrink_move moves it to the indicated
437 * private list.
438 */
439 static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry)
440 {
441 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
442 dentry->d_flags &= ~DCACHE_LRU_LIST;
443 this_cpu_dec(nr_dentry_unused);
444 if (d_is_negative(dentry))
445 this_cpu_dec(nr_dentry_negative);
446 list_lru_isolate(lru, &dentry->d_lru);
447 }
448
449 static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry,
450 struct list_head *list)
451 {
452 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
453 dentry->d_flags |= DCACHE_SHRINK_LIST;
454 if (d_is_negative(dentry))
455 this_cpu_dec(nr_dentry_negative);
456 list_lru_isolate_move(lru, &dentry->d_lru, list);
457 }
458
459 /**
460 * d_drop - drop a dentry
461 * @dentry: dentry to drop
462 *
463 * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
464 * be found through a VFS lookup any more. Note that this is different from
465 * deleting the dentry - d_delete will try to mark the dentry negative if
466 * possible, giving a successful _negative_ lookup, while d_drop will
467 * just make the cache lookup fail.
468 *
469 * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
470 * reason (NFS timeouts or autofs deletes).
471 *
472 * __d_drop requires dentry->d_lock
473 * ___d_drop doesn't mark dentry as "unhashed"
474 * (dentry->d_hash.pprev will be LIST_POISON2, not NULL).
475 */
476 static void ___d_drop(struct dentry *dentry)
477 {
478 struct hlist_bl_head *b;
479 /*
480 * Hashed dentries are normally on the dentry hashtable,
481 * with the exception of those newly allocated by
482 * d_obtain_root, which are always IS_ROOT:
483 */
484 if (unlikely(IS_ROOT(dentry)))
485 b = &dentry->d_sb->s_roots;
486 else
487 b = d_hash(dentry->d_name.hash);
488
489 hlist_bl_lock(b);
490 __hlist_bl_del(&dentry->d_hash);
491 hlist_bl_unlock(b);
492 }
493
494 void __d_drop(struct dentry *dentry)
495 {
496 if (!d_unhashed(dentry)) {
497 ___d_drop(dentry);
498 dentry->d_hash.pprev = NULL;
499 write_seqcount_invalidate(&dentry->d_seq);
500 }
501 }
502 EXPORT_SYMBOL(__d_drop);
503
504 void d_drop(struct dentry *dentry)
505 {
506 spin_lock(&dentry->d_lock);
507 __d_drop(dentry);
508 spin_unlock(&dentry->d_lock);
509 }
510 EXPORT_SYMBOL(d_drop);
511
512 static inline void dentry_unlist(struct dentry *dentry, struct dentry *parent)
513 {
514 struct dentry *next;
515 /*
516 * Inform d_walk() and shrink_dentry_list() that we are no longer
517 * attached to the dentry tree
518 */
519 dentry->d_flags |= DCACHE_DENTRY_KILLED;
520 if (unlikely(list_empty(&dentry->d_child)))
521 return;
522 __list_del_entry(&dentry->d_child);
523 /*
524 * Cursors can move around the list of children. While we'd been
525 * a normal list member, it didn't matter - ->d_child.next would've
526 * been updated. However, from now on it won't be and for the
527 * things like d_walk() it might end up with a nasty surprise.
528 * Normally d_walk() doesn't care about cursors moving around -
529 * ->d_lock on parent prevents that and since a cursor has no children
530 * of its own, we get through it without ever unlocking the parent.
531 * There is one exception, though - if we ascend from a child that
532 * gets killed as soon as we unlock it, the next sibling is found
533 * using the value left in its ->d_child.next. And if _that_
534 * pointed to a cursor, and cursor got moved (e.g. by lseek())
535 * before d_walk() regains parent->d_lock, we'll end up skipping
536 * everything the cursor had been moved past.
537 *
538 * Solution: make sure that the pointer left behind in ->d_child.next
539 * points to something that won't be moving around. I.e. skip the
540 * cursors.
541 */
542 while (dentry->d_child.next != &parent->d_subdirs) {
543 next = list_entry(dentry->d_child.next, struct dentry, d_child);
544 if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR)))
545 break;
546 dentry->d_child.next = next->d_child.next;
547 }
548 }
549
550 static void __dentry_kill(struct dentry *dentry)
551 {
552 struct dentry *parent = NULL;
553 bool can_free = true;
554 if (!IS_ROOT(dentry))
555 parent = dentry->d_parent;
556
557 /*
558 * The dentry is now unrecoverably dead to the world.
559 */
560 lockref_mark_dead(&dentry->d_lockref);
561
562 /*
563 * inform the fs via d_prune that this dentry is about to be
564 * unhashed and destroyed.
565 */
566 if (dentry->d_flags & DCACHE_OP_PRUNE)
567 dentry->d_op->d_prune(dentry);
568
569 if (dentry->d_flags & DCACHE_LRU_LIST) {
570 if (!(dentry->d_flags & DCACHE_SHRINK_LIST))
571 d_lru_del(dentry);
572 }
573 /* if it was on the hash then remove it */
574 __d_drop(dentry);
575 dentry_unlist(dentry, parent);
576 if (parent)
577 spin_unlock(&parent->d_lock);
578 if (dentry->d_inode)
579 dentry_unlink_inode(dentry);
580 else
581 spin_unlock(&dentry->d_lock);
582 this_cpu_dec(nr_dentry);
583 if (dentry->d_op && dentry->d_op->d_release)
584 dentry->d_op->d_release(dentry);
585
586 spin_lock(&dentry->d_lock);
587 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
588 dentry->d_flags |= DCACHE_MAY_FREE;
589 can_free = false;
590 }
591 spin_unlock(&dentry->d_lock);
592 if (likely(can_free))
593 dentry_free(dentry);
594 cond_resched();
595 }
596
597 static struct dentry *__lock_parent(struct dentry *dentry)
598 {
599 struct dentry *parent;
600 rcu_read_lock();
601 spin_unlock(&dentry->d_lock);
602 again:
603 parent = READ_ONCE(dentry->d_parent);
604 spin_lock(&parent->d_lock);
605 /*
606 * We can't blindly lock dentry until we are sure
607 * that we won't violate the locking order.
608 * Any changes of dentry->d_parent must have
609 * been done with parent->d_lock held, so
610 * spin_lock() above is enough of a barrier
611 * for checking if it's still our child.
612 */
613 if (unlikely(parent != dentry->d_parent)) {
614 spin_unlock(&parent->d_lock);
615 goto again;
616 }
617 rcu_read_unlock();
618 if (parent != dentry)
619 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
620 else
621 parent = NULL;
622 return parent;
623 }
624
625 static inline struct dentry *lock_parent(struct dentry *dentry)
626 {
627 struct dentry *parent = dentry->d_parent;
628 if (IS_ROOT(dentry))
629 return NULL;
630 if (likely(spin_trylock(&parent->d_lock)))
631 return parent;
632 return __lock_parent(dentry);
633 }
634
635 static inline bool retain_dentry(struct dentry *dentry)
636 {
637 WARN_ON(d_in_lookup(dentry));
638
639 /* Unreachable? Get rid of it */
640 if (unlikely(d_unhashed(dentry)))
641 return false;
642
643 if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED))
644 return false;
645
646 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
647 if (dentry->d_op->d_delete(dentry))
648 return false;
649 }
650
651 if (unlikely(dentry->d_flags & DCACHE_DONTCACHE))
652 return false;
653
654 /* retain; LRU fodder */
655 dentry->d_lockref.count--;
656 if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
657 d_lru_add(dentry);
658 else if (unlikely(!(dentry->d_flags & DCACHE_REFERENCED)))
659 dentry->d_flags |= DCACHE_REFERENCED;
660 return true;
661 }
662
663 void d_mark_dontcache(struct inode *inode)
664 {
665 struct dentry *de;
666
667 spin_lock(&inode->i_lock);
668 hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) {
669 spin_lock(&de->d_lock);
670 de->d_flags |= DCACHE_DONTCACHE;
671 spin_unlock(&de->d_lock);
672 }
673 inode->i_state |= I_DONTCACHE;
674 spin_unlock(&inode->i_lock);
675 }
676 EXPORT_SYMBOL(d_mark_dontcache);
677
678 /*
679 * Finish off a dentry we've decided to kill.
680 * dentry->d_lock must be held, returns with it unlocked.
681 * Returns dentry requiring refcount drop, or NULL if we're done.
682 */
683 static struct dentry *dentry_kill(struct dentry *dentry)
684 __releases(dentry->d_lock)
685 {
686 struct inode *inode = dentry->d_inode;
687 struct dentry *parent = NULL;
688
689 if (inode && unlikely(!spin_trylock(&inode->i_lock)))
690 goto slow_positive;
691
692 if (!IS_ROOT(dentry)) {
693 parent = dentry->d_parent;
694 if (unlikely(!spin_trylock(&parent->d_lock))) {
695 parent = __lock_parent(dentry);
696 if (likely(inode || !dentry->d_inode))
697 goto got_locks;
698 /* negative that became positive */
699 if (parent)
700 spin_unlock(&parent->d_lock);
701 inode = dentry->d_inode;
702 goto slow_positive;
703 }
704 }
705 __dentry_kill(dentry);
706 return parent;
707
708 slow_positive:
709 spin_unlock(&dentry->d_lock);
710 spin_lock(&inode->i_lock);
711 spin_lock(&dentry->d_lock);
712 parent = lock_parent(dentry);
713 got_locks:
714 if (unlikely(dentry->d_lockref.count != 1)) {
715 dentry->d_lockref.count--;
716 } else if (likely(!retain_dentry(dentry))) {
717 __dentry_kill(dentry);
718 return parent;
719 }
720 /* we are keeping it, after all */
721 if (inode)
722 spin_unlock(&inode->i_lock);
723 if (parent)
724 spin_unlock(&parent->d_lock);
725 spin_unlock(&dentry->d_lock);
726 return NULL;
727 }
728
729 /*
730 * Try to do a lockless dput(), and return whether that was successful.
731 *
732 * If unsuccessful, we return false, having already taken the dentry lock.
733 *
734 * The caller needs to hold the RCU read lock, so that the dentry is
735 * guaranteed to stay around even if the refcount goes down to zero!
736 */
737 static inline bool fast_dput(struct dentry *dentry)
738 {
739 int ret;
740 unsigned int d_flags;
741
742 /*
743 * If we have a d_op->d_delete() operation, we sould not
744 * let the dentry count go to zero, so use "put_or_lock".
745 */
746 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE))
747 return lockref_put_or_lock(&dentry->d_lockref);
748
749 /*
750 * .. otherwise, we can try to just decrement the
751 * lockref optimistically.
752 */
753 ret = lockref_put_return(&dentry->d_lockref);
754
755 /*
756 * If the lockref_put_return() failed due to the lock being held
757 * by somebody else, the fast path has failed. We will need to
758 * get the lock, and then check the count again.
759 */
760 if (unlikely(ret < 0)) {
761 spin_lock(&dentry->d_lock);
762 if (dentry->d_lockref.count > 1) {
763 dentry->d_lockref.count--;
764 spin_unlock(&dentry->d_lock);
765 return true;
766 }
767 return false;
768 }
769
770 /*
771 * If we weren't the last ref, we're done.
772 */
773 if (ret)
774 return true;
775
776 /*
777 * Careful, careful. The reference count went down
778 * to zero, but we don't hold the dentry lock, so
779 * somebody else could get it again, and do another
780 * dput(), and we need to not race with that.
781 *
782 * However, there is a very special and common case
783 * where we don't care, because there is nothing to
784 * do: the dentry is still hashed, it does not have
785 * a 'delete' op, and it's referenced and already on
786 * the LRU list.
787 *
788 * NOTE! Since we aren't locked, these values are
789 * not "stable". However, it is sufficient that at
790 * some point after we dropped the reference the
791 * dentry was hashed and the flags had the proper
792 * value. Other dentry users may have re-gotten
793 * a reference to the dentry and change that, but
794 * our work is done - we can leave the dentry
795 * around with a zero refcount.
796 *
797 * Nevertheless, there are two cases that we should kill
798 * the dentry anyway.
799 * 1. free disconnected dentries as soon as their refcount
800 * reached zero.
801 * 2. free dentries if they should not be cached.
802 */
803 smp_rmb();
804 d_flags = READ_ONCE(dentry->d_flags);
805 d_flags &= DCACHE_REFERENCED | DCACHE_LRU_LIST |
806 DCACHE_DISCONNECTED | DCACHE_DONTCACHE;
807
808 /* Nothing to do? Dropping the reference was all we needed? */
809 if (d_flags == (DCACHE_REFERENCED | DCACHE_LRU_LIST) && !d_unhashed(dentry))
810 return true;
811
812 /*
813 * Not the fast normal case? Get the lock. We've already decremented
814 * the refcount, but we'll need to re-check the situation after
815 * getting the lock.
816 */
817 spin_lock(&dentry->d_lock);
818
819 /*
820 * Did somebody else grab a reference to it in the meantime, and
821 * we're no longer the last user after all? Alternatively, somebody
822 * else could have killed it and marked it dead. Either way, we
823 * don't need to do anything else.
824 */
825 if (dentry->d_lockref.count) {
826 spin_unlock(&dentry->d_lock);
827 return true;
828 }
829
830 /*
831 * Re-get the reference we optimistically dropped. We hold the
832 * lock, and we just tested that it was zero, so we can just
833 * set it to 1.
834 */
835 dentry->d_lockref.count = 1;
836 return false;
837 }
838
839
840 /*
841 * This is dput
842 *
843 * This is complicated by the fact that we do not want to put
844 * dentries that are no longer on any hash chain on the unused
845 * list: we'd much rather just get rid of them immediately.
846 *
847 * However, that implies that we have to traverse the dentry
848 * tree upwards to the parents which might _also_ now be
849 * scheduled for deletion (it may have been only waiting for
850 * its last child to go away).
851 *
852 * This tail recursion is done by hand as we don't want to depend
853 * on the compiler to always get this right (gcc generally doesn't).
854 * Real recursion would eat up our stack space.
855 */
856
857 /*
858 * dput - release a dentry
859 * @dentry: dentry to release
860 *
861 * Release a dentry. This will drop the usage count and if appropriate
862 * call the dentry unlink method as well as removing it from the queues and
863 * releasing its resources. If the parent dentries were scheduled for release
864 * they too may now get deleted.
865 */
866 void dput(struct dentry *dentry)
867 {
868 while (dentry) {
869 might_sleep();
870
871 rcu_read_lock();
872 if (likely(fast_dput(dentry))) {
873 rcu_read_unlock();
874 return;
875 }
876
877 /* Slow case: now with the dentry lock held */
878 rcu_read_unlock();
879
880 if (likely(retain_dentry(dentry))) {
881 spin_unlock(&dentry->d_lock);
882 return;
883 }
884
885 dentry = dentry_kill(dentry);
886 }
887 }
888 EXPORT_SYMBOL(dput);
889
890 static void __dput_to_list(struct dentry *dentry, struct list_head *list)
891 __must_hold(&dentry->d_lock)
892 {
893 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
894 /* let the owner of the list it's on deal with it */
895 --dentry->d_lockref.count;
896 } else {
897 if (dentry->d_flags & DCACHE_LRU_LIST)
898 d_lru_del(dentry);
899 if (!--dentry->d_lockref.count)
900 d_shrink_add(dentry, list);
901 }
902 }
903
904 void dput_to_list(struct dentry *dentry, struct list_head *list)
905 {
906 rcu_read_lock();
907 if (likely(fast_dput(dentry))) {
908 rcu_read_unlock();
909 return;
910 }
911 rcu_read_unlock();
912 if (!retain_dentry(dentry))
913 __dput_to_list(dentry, list);
914 spin_unlock(&dentry->d_lock);
915 }
916
917 /* This must be called with d_lock held */
918 static inline void __dget_dlock(struct dentry *dentry)
919 {
920 dentry->d_lockref.count++;
921 }
922
923 static inline void __dget(struct dentry *dentry)
924 {
925 lockref_get(&dentry->d_lockref);
926 }
927
928 struct dentry *dget_parent(struct dentry *dentry)
929 {
930 int gotref;
931 struct dentry *ret;
932 unsigned seq;
933
934 /*
935 * Do optimistic parent lookup without any
936 * locking.
937 */
938 rcu_read_lock();
939 seq = raw_seqcount_begin(&dentry->d_seq);
940 ret = READ_ONCE(dentry->d_parent);
941 gotref = lockref_get_not_zero(&ret->d_lockref);
942 rcu_read_unlock();
943 if (likely(gotref)) {
944 if (!read_seqcount_retry(&dentry->d_seq, seq))
945 return ret;
946 dput(ret);
947 }
948
949 repeat:
950 /*
951 * Don't need rcu_dereference because we re-check it was correct under
952 * the lock.
953 */
954 rcu_read_lock();
955 ret = dentry->d_parent;
956 spin_lock(&ret->d_lock);
957 if (unlikely(ret != dentry->d_parent)) {
958 spin_unlock(&ret->d_lock);
959 rcu_read_unlock();
960 goto repeat;
961 }
962 rcu_read_unlock();
963 BUG_ON(!ret->d_lockref.count);
964 ret->d_lockref.count++;
965 spin_unlock(&ret->d_lock);
966 return ret;
967 }
968 EXPORT_SYMBOL(dget_parent);
969
970 static struct dentry * __d_find_any_alias(struct inode *inode)
971 {
972 struct dentry *alias;
973
974 if (hlist_empty(&inode->i_dentry))
975 return NULL;
976 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
977 __dget(alias);
978 return alias;
979 }
980
981 /**
982 * d_find_any_alias - find any alias for a given inode
983 * @inode: inode to find an alias for
984 *
985 * If any aliases exist for the given inode, take and return a
986 * reference for one of them. If no aliases exist, return %NULL.
987 */
988 struct dentry *d_find_any_alias(struct inode *inode)
989 {
990 struct dentry *de;
991
992 spin_lock(&inode->i_lock);
993 de = __d_find_any_alias(inode);
994 spin_unlock(&inode->i_lock);
995 return de;
996 }
997 EXPORT_SYMBOL(d_find_any_alias);
998
999 /**
1000 * d_find_alias - grab a hashed alias of inode
1001 * @inode: inode in question
1002 *
1003 * If inode has a hashed alias, or is a directory and has any alias,
1004 * acquire the reference to alias and return it. Otherwise return NULL.
1005 * Notice that if inode is a directory there can be only one alias and
1006 * it can be unhashed only if it has no children, or if it is the root
1007 * of a filesystem, or if the directory was renamed and d_revalidate
1008 * was the first vfs operation to notice.
1009 *
1010 * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
1011 * any other hashed alias over that one.
1012 */
1013 static struct dentry *__d_find_alias(struct inode *inode)
1014 {
1015 struct dentry *alias;
1016
1017 if (S_ISDIR(inode->i_mode))
1018 return __d_find_any_alias(inode);
1019
1020 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
1021 spin_lock(&alias->d_lock);
1022 if (!d_unhashed(alias)) {
1023 __dget_dlock(alias);
1024 spin_unlock(&alias->d_lock);
1025 return alias;
1026 }
1027 spin_unlock(&alias->d_lock);
1028 }
1029 return NULL;
1030 }
1031
1032 struct dentry *d_find_alias(struct inode *inode)
1033 {
1034 struct dentry *de = NULL;
1035
1036 if (!hlist_empty(&inode->i_dentry)) {
1037 spin_lock(&inode->i_lock);
1038 de = __d_find_alias(inode);
1039 spin_unlock(&inode->i_lock);
1040 }
1041 return de;
1042 }
1043 EXPORT_SYMBOL(d_find_alias);
1044
1045 /*
1046 * Try to kill dentries associated with this inode.
1047 * WARNING: you must own a reference to inode.
1048 */
1049 void d_prune_aliases(struct inode *inode)
1050 {
1051 struct dentry *dentry;
1052 restart:
1053 spin_lock(&inode->i_lock);
1054 hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) {
1055 spin_lock(&dentry->d_lock);
1056 if (!dentry->d_lockref.count) {
1057 struct dentry *parent = lock_parent(dentry);
1058 if (likely(!dentry->d_lockref.count)) {
1059 __dentry_kill(dentry);
1060 dput(parent);
1061 goto restart;
1062 }
1063 if (parent)
1064 spin_unlock(&parent->d_lock);
1065 }
1066 spin_unlock(&dentry->d_lock);
1067 }
1068 spin_unlock(&inode->i_lock);
1069 }
1070 EXPORT_SYMBOL(d_prune_aliases);
1071
1072 /*
1073 * Lock a dentry from shrink list.
1074 * Called under rcu_read_lock() and dentry->d_lock; the former
1075 * guarantees that nothing we access will be freed under us.
1076 * Note that dentry is *not* protected from concurrent dentry_kill(),
1077 * d_delete(), etc.
1078 *
1079 * Return false if dentry has been disrupted or grabbed, leaving
1080 * the caller to kick it off-list. Otherwise, return true and have
1081 * that dentry's inode and parent both locked.
1082 */
1083 static bool shrink_lock_dentry(struct dentry *dentry)
1084 {
1085 struct inode *inode;
1086 struct dentry *parent;
1087
1088 if (dentry->d_lockref.count)
1089 return false;
1090
1091 inode = dentry->d_inode;
1092 if (inode && unlikely(!spin_trylock(&inode->i_lock))) {
1093 spin_unlock(&dentry->d_lock);
1094 spin_lock(&inode->i_lock);
1095 spin_lock(&dentry->d_lock);
1096 if (unlikely(dentry->d_lockref.count))
1097 goto out;
1098 /* changed inode means that somebody had grabbed it */
1099 if (unlikely(inode != dentry->d_inode))
1100 goto out;
1101 }
1102
1103 parent = dentry->d_parent;
1104 if (IS_ROOT(dentry) || likely(spin_trylock(&parent->d_lock)))
1105 return true;
1106
1107 spin_unlock(&dentry->d_lock);
1108 spin_lock(&parent->d_lock);
1109 if (unlikely(parent != dentry->d_parent)) {
1110 spin_unlock(&parent->d_lock);
1111 spin_lock(&dentry->d_lock);
1112 goto out;
1113 }
1114 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1115 if (likely(!dentry->d_lockref.count))
1116 return true;
1117 spin_unlock(&parent->d_lock);
1118 out:
1119 if (inode)
1120 spin_unlock(&inode->i_lock);
1121 return false;
1122 }
1123
1124 void shrink_dentry_list(struct list_head *list)
1125 {
1126 while (!list_empty(list)) {
1127 struct dentry *dentry, *parent;
1128
1129 dentry = list_entry(list->prev, struct dentry, d_lru);
1130 spin_lock(&dentry->d_lock);
1131 rcu_read_lock();
1132 if (!shrink_lock_dentry(dentry)) {
1133 bool can_free = false;
1134 rcu_read_unlock();
1135 d_shrink_del(dentry);
1136 if (dentry->d_lockref.count < 0)
1137 can_free = dentry->d_flags & DCACHE_MAY_FREE;
1138 spin_unlock(&dentry->d_lock);
1139 if (can_free)
1140 dentry_free(dentry);
1141 continue;
1142 }
1143 rcu_read_unlock();
1144 d_shrink_del(dentry);
1145 parent = dentry->d_parent;
1146 if (parent != dentry)
1147 __dput_to_list(parent, list);
1148 __dentry_kill(dentry);
1149 }
1150 }
1151
1152 static enum lru_status dentry_lru_isolate(struct list_head *item,
1153 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
1154 {
1155 struct list_head *freeable = arg;
1156 struct dentry *dentry = container_of(item, struct dentry, d_lru);
1157
1158
1159 /*
1160 * we are inverting the lru lock/dentry->d_lock here,
1161 * so use a trylock. If we fail to get the lock, just skip
1162 * it
1163 */
1164 if (!spin_trylock(&dentry->d_lock))
1165 return LRU_SKIP;
1166
1167 /*
1168 * Referenced dentries are still in use. If they have active
1169 * counts, just remove them from the LRU. Otherwise give them
1170 * another pass through the LRU.
1171 */
1172 if (dentry->d_lockref.count) {
1173 d_lru_isolate(lru, dentry);
1174 spin_unlock(&dentry->d_lock);
1175 return LRU_REMOVED;
1176 }
1177
1178 if (dentry->d_flags & DCACHE_REFERENCED) {
1179 dentry->d_flags &= ~DCACHE_REFERENCED;
1180 spin_unlock(&dentry->d_lock);
1181
1182 /*
1183 * The list move itself will be made by the common LRU code. At
1184 * this point, we've dropped the dentry->d_lock but keep the
1185 * lru lock. This is safe to do, since every list movement is
1186 * protected by the lru lock even if both locks are held.
1187 *
1188 * This is guaranteed by the fact that all LRU management
1189 * functions are intermediated by the LRU API calls like
1190 * list_lru_add and list_lru_del. List movement in this file
1191 * only ever occur through this functions or through callbacks
1192 * like this one, that are called from the LRU API.
1193 *
1194 * The only exceptions to this are functions like
1195 * shrink_dentry_list, and code that first checks for the
1196 * DCACHE_SHRINK_LIST flag. Those are guaranteed to be
1197 * operating only with stack provided lists after they are
1198 * properly isolated from the main list. It is thus, always a
1199 * local access.
1200 */
1201 return LRU_ROTATE;
1202 }
1203
1204 d_lru_shrink_move(lru, dentry, freeable);
1205 spin_unlock(&dentry->d_lock);
1206
1207 return LRU_REMOVED;
1208 }
1209
1210 /**
1211 * prune_dcache_sb - shrink the dcache
1212 * @sb: superblock
1213 * @sc: shrink control, passed to list_lru_shrink_walk()
1214 *
1215 * Attempt to shrink the superblock dcache LRU by @sc->nr_to_scan entries. This
1216 * is done when we need more memory and called from the superblock shrinker
1217 * function.
1218 *
1219 * This function may fail to free any resources if all the dentries are in
1220 * use.
1221 */
1222 long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc)
1223 {
1224 LIST_HEAD(dispose);
1225 long freed;
1226
1227 freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc,
1228 dentry_lru_isolate, &dispose);
1229 shrink_dentry_list(&dispose);
1230 return freed;
1231 }
1232
1233 static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
1234 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
1235 {
1236 struct list_head *freeable = arg;
1237 struct dentry *dentry = container_of(item, struct dentry, d_lru);
1238
1239 /*
1240 * we are inverting the lru lock/dentry->d_lock here,
1241 * so use a trylock. If we fail to get the lock, just skip
1242 * it
1243 */
1244 if (!spin_trylock(&dentry->d_lock))
1245 return LRU_SKIP;
1246
1247 d_lru_shrink_move(lru, dentry, freeable);
1248 spin_unlock(&dentry->d_lock);
1249
1250 return LRU_REMOVED;
1251 }
1252
1253
1254 /**
1255 * shrink_dcache_sb - shrink dcache for a superblock
1256 * @sb: superblock
1257 *
1258 * Shrink the dcache for the specified super block. This is used to free
1259 * the dcache before unmounting a file system.
1260 */
1261 void shrink_dcache_sb(struct super_block *sb)
1262 {
1263 do {
1264 LIST_HEAD(dispose);
1265
1266 list_lru_walk(&sb->s_dentry_lru,
1267 dentry_lru_isolate_shrink, &dispose, 1024);
1268 shrink_dentry_list(&dispose);
1269 } while (list_lru_count(&sb->s_dentry_lru) > 0);
1270 }
1271 EXPORT_SYMBOL(shrink_dcache_sb);
1272
1273 /**
1274 * enum d_walk_ret - action to talke during tree walk
1275 * @D_WALK_CONTINUE: contrinue walk
1276 * @D_WALK_QUIT: quit walk
1277 * @D_WALK_NORETRY: quit when retry is needed
1278 * @D_WALK_SKIP: skip this dentry and its children
1279 */
1280 enum d_walk_ret {
1281 D_WALK_CONTINUE,
1282 D_WALK_QUIT,
1283 D_WALK_NORETRY,
1284 D_WALK_SKIP,
1285 };
1286
1287 /**
1288 * d_walk - walk the dentry tree
1289 * @parent: start of walk
1290 * @data: data passed to @enter() and @finish()
1291 * @enter: callback when first entering the dentry
1292 *
1293 * The @enter() callbacks are called with d_lock held.
1294 */
1295 void d_walk(struct dentry *parent, void *data,
1296 enum d_walk_ret (*enter)(void *, struct dentry *))
1297 {
1298 struct dentry *this_parent;
1299 struct list_head *next;
1300 unsigned seq = 0;
1301 enum d_walk_ret ret;
1302 bool retry = true;
1303
1304 again:
1305 read_seqbegin_or_lock(&rename_lock, &seq);
1306 this_parent = parent;
1307 spin_lock(&this_parent->d_lock);
1308
1309 ret = enter(data, this_parent);
1310 switch (ret) {
1311 case D_WALK_CONTINUE:
1312 break;
1313 case D_WALK_QUIT:
1314 case D_WALK_SKIP:
1315 goto out_unlock;
1316 case D_WALK_NORETRY:
1317 retry = false;
1318 break;
1319 }
1320 repeat:
1321 next = this_parent->d_subdirs.next;
1322 resume:
1323 while (next != &this_parent->d_subdirs) {
1324 struct list_head *tmp = next;
1325 struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
1326 next = tmp->next;
1327
1328 if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR))
1329 continue;
1330
1331 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1332
1333 ret = enter(data, dentry);
1334 switch (ret) {
1335 case D_WALK_CONTINUE:
1336 break;
1337 case D_WALK_QUIT:
1338 spin_unlock(&dentry->d_lock);
1339 goto out_unlock;
1340 case D_WALK_NORETRY:
1341 retry = false;
1342 break;
1343 case D_WALK_SKIP:
1344 spin_unlock(&dentry->d_lock);
1345 continue;
1346 }
1347
1348 if (!list_empty(&dentry->d_subdirs)) {
1349 spin_unlock(&this_parent->d_lock);
1350 spin_release(&dentry->d_lock.dep_map, _RET_IP_);
1351 this_parent = dentry;
1352 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
1353 goto repeat;
1354 }
1355 spin_unlock(&dentry->d_lock);
1356 }
1357 /*
1358 * All done at this level ... ascend and resume the search.
1359 */
1360 rcu_read_lock();
1361 ascend:
1362 if (this_parent != parent) {
1363 struct dentry *child = this_parent;
1364 this_parent = child->d_parent;
1365
1366 spin_unlock(&child->d_lock);
1367 spin_lock(&this_parent->d_lock);
1368
1369 /* might go back up the wrong parent if we have had a rename. */
1370 if (need_seqretry(&rename_lock, seq))
1371 goto rename_retry;
1372 /* go into the first sibling still alive */
1373 do {
1374 next = child->d_child.next;
1375 if (next == &this_parent->d_subdirs)
1376 goto ascend;
1377 child = list_entry(next, struct dentry, d_child);
1378 } while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED));
1379 rcu_read_unlock();
1380 goto resume;
1381 }
1382 if (need_seqretry(&rename_lock, seq))
1383 goto rename_retry;
1384 rcu_read_unlock();
1385
1386 out_unlock:
1387 spin_unlock(&this_parent->d_lock);
1388 done_seqretry(&rename_lock, seq);
1389 return;
1390
1391 rename_retry:
1392 spin_unlock(&this_parent->d_lock);
1393 rcu_read_unlock();
1394 BUG_ON(seq & 1);
1395 if (!retry)
1396 return;
1397 seq = 1;
1398 goto again;
1399 }
1400 EXPORT_SYMBOL_GPL(d_walk);
1401
1402 struct check_mount {
1403 struct vfsmount *mnt;
1404 unsigned int mounted;
1405 };
1406
1407 static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry)
1408 {
1409 struct check_mount *info = data;
1410 struct path path = { .mnt = info->mnt, .dentry = dentry };
1411
1412 if (likely(!d_mountpoint(dentry)))
1413 return D_WALK_CONTINUE;
1414 if (__path_is_mountpoint(&path)) {
1415 info->mounted = 1;
1416 return D_WALK_QUIT;
1417 }
1418 return D_WALK_CONTINUE;
1419 }
1420
1421 /**
1422 * path_has_submounts - check for mounts over a dentry in the
1423 * current namespace.
1424 * @parent: path to check.
1425 *
1426 * Return true if the parent or its subdirectories contain
1427 * a mount point in the current namespace.
1428 */
1429 int path_has_submounts(const struct path *parent)
1430 {
1431 struct check_mount data = { .mnt = parent->mnt, .mounted = 0 };
1432
1433 read_seqlock_excl(&mount_lock);
1434 d_walk(parent->dentry, &data, path_check_mount);
1435 read_sequnlock_excl(&mount_lock);
1436
1437 return data.mounted;
1438 }
1439 EXPORT_SYMBOL(path_has_submounts);
1440
1441 /*
1442 * Called by mount code to set a mountpoint and check if the mountpoint is
1443 * reachable (e.g. NFS can unhash a directory dentry and then the complete
1444 * subtree can become unreachable).
1445 *
1446 * Only one of d_invalidate() and d_set_mounted() must succeed. For
1447 * this reason take rename_lock and d_lock on dentry and ancestors.
1448 */
1449 int d_set_mounted(struct dentry *dentry)
1450 {
1451 struct dentry *p;
1452 int ret = -ENOENT;
1453 write_seqlock(&rename_lock);
1454 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
1455 /* Need exclusion wrt. d_invalidate() */
1456 spin_lock(&p->d_lock);
1457 if (unlikely(d_unhashed(p))) {
1458 spin_unlock(&p->d_lock);
1459 goto out;
1460 }
1461 spin_unlock(&p->d_lock);
1462 }
1463 spin_lock(&dentry->d_lock);
1464 if (!d_unlinked(dentry)) {
1465 ret = -EBUSY;
1466 if (!d_mountpoint(dentry)) {
1467 dentry->d_flags |= DCACHE_MOUNTED;
1468 ret = 0;
1469 }
1470 }
1471 spin_unlock(&dentry->d_lock);
1472 out:
1473 write_sequnlock(&rename_lock);
1474 return ret;
1475 }
1476
1477 /*
1478 * Search the dentry child list of the specified parent,
1479 * and move any unused dentries to the end of the unused
1480 * list for prune_dcache(). We descend to the next level
1481 * whenever the d_subdirs list is non-empty and continue
1482 * searching.
1483 *
1484 * It returns zero iff there are no unused children,
1485 * otherwise it returns the number of children moved to
1486 * the end of the unused list. This may not be the total
1487 * number of unused children, because select_parent can
1488 * drop the lock and return early due to latency
1489 * constraints.
1490 */
1491
1492 struct select_data {
1493 struct dentry *start;
1494 union {
1495 long found;
1496 struct dentry *victim;
1497 };
1498 struct list_head dispose;
1499 };
1500
1501 static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
1502 {
1503 struct select_data *data = _data;
1504 enum d_walk_ret ret = D_WALK_CONTINUE;
1505
1506 if (data->start == dentry)
1507 goto out;
1508
1509 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1510 data->found++;
1511 } else {
1512 if (dentry->d_flags & DCACHE_LRU_LIST)
1513 d_lru_del(dentry);
1514 if (!dentry->d_lockref.count) {
1515 d_shrink_add(dentry, &data->dispose);
1516 data->found++;
1517 }
1518 }
1519 /*
1520 * We can return to the caller if we have found some (this
1521 * ensures forward progress). We'll be coming back to find
1522 * the rest.
1523 */
1524 if (!list_empty(&data->dispose))
1525 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1526 out:
1527 return ret;
1528 }
1529
1530 static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry)
1531 {
1532 struct select_data *data = _data;
1533 enum d_walk_ret ret = D_WALK_CONTINUE;
1534
1535 if (data->start == dentry)
1536 goto out;
1537
1538 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1539 if (!dentry->d_lockref.count) {
1540 rcu_read_lock();
1541 data->victim = dentry;
1542 return D_WALK_QUIT;
1543 }
1544 } else {
1545 if (dentry->d_flags & DCACHE_LRU_LIST)
1546 d_lru_del(dentry);
1547 if (!dentry->d_lockref.count)
1548 d_shrink_add(dentry, &data->dispose);
1549 }
1550 /*
1551 * We can return to the caller if we have found some (this
1552 * ensures forward progress). We'll be coming back to find
1553 * the rest.
1554 */
1555 if (!list_empty(&data->dispose))
1556 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1557 out:
1558 return ret;
1559 }
1560
1561 /**
1562 * shrink_dcache_parent - prune dcache
1563 * @parent: parent of entries to prune
1564 *
1565 * Prune the dcache to remove unused children of the parent dentry.
1566 */
1567 void shrink_dcache_parent(struct dentry *parent)
1568 {
1569 for (;;) {
1570 struct select_data data = {.start = parent};
1571
1572 INIT_LIST_HEAD(&data.dispose);
1573 d_walk(parent, &data, select_collect);
1574
1575 if (!list_empty(&data.dispose)) {
1576 shrink_dentry_list(&data.dispose);
1577 continue;
1578 }
1579
1580 cond_resched();
1581 if (!data.found)
1582 break;
1583 data.victim = NULL;
1584 d_walk(parent, &data, select_collect2);
1585 if (data.victim) {
1586 struct dentry *parent;
1587 spin_lock(&data.victim->d_lock);
1588 if (!shrink_lock_dentry(data.victim)) {
1589 spin_unlock(&data.victim->d_lock);
1590 rcu_read_unlock();
1591 } else {
1592 rcu_read_unlock();
1593 parent = data.victim->d_parent;
1594 if (parent != data.victim)
1595 __dput_to_list(parent, &data.dispose);
1596 __dentry_kill(data.victim);
1597 }
1598 }
1599 if (!list_empty(&data.dispose))
1600 shrink_dentry_list(&data.dispose);
1601 }
1602 }
1603 EXPORT_SYMBOL(shrink_dcache_parent);
1604
1605 static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
1606 {
1607 /* it has busy descendents; complain about those instead */
1608 if (!list_empty(&dentry->d_subdirs))
1609 return D_WALK_CONTINUE;
1610
1611 /* root with refcount 1 is fine */
1612 if (dentry == _data && dentry->d_lockref.count == 1)
1613 return D_WALK_CONTINUE;
1614
1615 printk(KERN_ERR "BUG: Dentry %p{i=%lx,n=%pd} "
1616 " still in use (%d) [unmount of %s %s]\n",
1617 dentry,
1618 dentry->d_inode ?
1619 dentry->d_inode->i_ino : 0UL,
1620 dentry,
1621 dentry->d_lockref.count,
1622 dentry->d_sb->s_type->name,
1623 dentry->d_sb->s_id);
1624 WARN_ON(1);
1625 return D_WALK_CONTINUE;
1626 }
1627
1628 static void do_one_tree(struct dentry *dentry)
1629 {
1630 shrink_dcache_parent(dentry);
1631 d_walk(dentry, dentry, umount_check);
1632 d_drop(dentry);
1633 dput(dentry);
1634 }
1635
1636 /*
1637 * destroy the dentries attached to a superblock on unmounting
1638 */
1639 void shrink_dcache_for_umount(struct super_block *sb)
1640 {
1641 struct dentry *dentry;
1642
1643 WARN(down_read_trylock(&sb->s_umount), "s_umount should've been locked");
1644
1645 dentry = sb->s_root;
1646 sb->s_root = NULL;
1647 do_one_tree(dentry);
1648
1649 while (!hlist_bl_empty(&sb->s_roots)) {
1650 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash));
1651 do_one_tree(dentry);
1652 }
1653 }
1654
1655 static enum d_walk_ret find_submount(void *_data, struct dentry *dentry)
1656 {
1657 struct dentry **victim = _data;
1658 if (d_mountpoint(dentry)) {
1659 __dget_dlock(dentry);
1660 *victim = dentry;
1661 return D_WALK_QUIT;
1662 }
1663 return D_WALK_CONTINUE;
1664 }
1665
1666 /**
1667 * d_invalidate - detach submounts, prune dcache, and drop
1668 * @dentry: dentry to invalidate (aka detach, prune and drop)
1669 */
1670 void d_invalidate(struct dentry *dentry)
1671 {
1672 bool had_submounts = false;
1673 spin_lock(&dentry->d_lock);
1674 if (d_unhashed(dentry)) {
1675 spin_unlock(&dentry->d_lock);
1676 return;
1677 }
1678 __d_drop(dentry);
1679 spin_unlock(&dentry->d_lock);
1680
1681 /* Negative dentries can be dropped without further checks */
1682 if (!dentry->d_inode)
1683 return;
1684
1685 shrink_dcache_parent(dentry);
1686 for (;;) {
1687 struct dentry *victim = NULL;
1688 d_walk(dentry, &victim, find_submount);
1689 if (!victim) {
1690 if (had_submounts)
1691 shrink_dcache_parent(dentry);
1692 return;
1693 }
1694 had_submounts = true;
1695 detach_mounts(victim);
1696 dput(victim);
1697 }
1698 }
1699 EXPORT_SYMBOL(d_invalidate);
1700
1701 /**
1702 * __d_alloc - allocate a dcache entry
1703 * @sb: filesystem it will belong to
1704 * @name: qstr of the name
1705 *
1706 * Allocates a dentry. It returns %NULL if there is insufficient memory
1707 * available. On a success the dentry is returned. The name passed in is
1708 * copied and the copy passed in may be reused after this call.
1709 */
1710
1711 static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
1712 {
1713 struct dentry *dentry;
1714 char *dname;
1715 int err;
1716
1717 dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
1718 if (!dentry)
1719 return NULL;
1720
1721 /*
1722 * We guarantee that the inline name is always NUL-terminated.
1723 * This way the memcpy() done by the name switching in rename
1724 * will still always have a NUL at the end, even if we might
1725 * be overwriting an internal NUL character
1726 */
1727 dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
1728 if (unlikely(!name)) {
1729 name = &slash_name;
1730 dname = dentry->d_iname;
1731 } else if (name->len > DNAME_INLINE_LEN-1) {
1732 size_t size = offsetof(struct external_name, name[1]);
1733 struct external_name *p = kmalloc(size + name->len,
1734 GFP_KERNEL_ACCOUNT |
1735 __GFP_RECLAIMABLE);
1736 if (!p) {
1737 kmem_cache_free(dentry_cache, dentry);
1738 return NULL;
1739 }
1740 atomic_set(&p->u.count, 1);
1741 dname = p->name;
1742 } else {
1743 dname = dentry->d_iname;
1744 }
1745
1746 dentry->d_name.len = name->len;
1747 dentry->d_name.hash = name->hash;
1748 memcpy(dname, name->name, name->len);
1749 dname[name->len] = 0;
1750
1751 /* Make sure we always see the terminating NUL character */
1752 smp_store_release(&dentry->d_name.name, dname); /* ^^^ */
1753
1754 dentry->d_lockref.count = 1;
1755 dentry->d_flags = 0;
1756 spin_lock_init(&dentry->d_lock);
1757 seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock);
1758 dentry->d_inode = NULL;
1759 dentry->d_parent = dentry;
1760 dentry->d_sb = sb;
1761 dentry->d_op = NULL;
1762 dentry->d_fsdata = NULL;
1763 INIT_HLIST_BL_NODE(&dentry->d_hash);
1764 INIT_LIST_HEAD(&dentry->d_lru);
1765 INIT_LIST_HEAD(&dentry->d_subdirs);
1766 INIT_HLIST_NODE(&dentry->d_u.d_alias);
1767 INIT_LIST_HEAD(&dentry->d_child);
1768 d_set_d_op(dentry, dentry->d_sb->s_d_op);
1769
1770 if (dentry->d_op && dentry->d_op->d_init) {
1771 err = dentry->d_op->d_init(dentry);
1772 if (err) {
1773 if (dname_external(dentry))
1774 kfree(external_name(dentry));
1775 kmem_cache_free(dentry_cache, dentry);
1776 return NULL;
1777 }
1778 }
1779
1780 this_cpu_inc(nr_dentry);
1781
1782 return dentry;
1783 }
1784
1785 /**
1786 * d_alloc - allocate a dcache entry
1787 * @parent: parent of entry to allocate
1788 * @name: qstr of the name
1789 *
1790 * Allocates a dentry. It returns %NULL if there is insufficient memory
1791 * available. On a success the dentry is returned. The name passed in is
1792 * copied and the copy passed in may be reused after this call.
1793 */
1794 struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
1795 {
1796 struct dentry *dentry = __d_alloc(parent->d_sb, name);
1797 if (!dentry)
1798 return NULL;
1799 spin_lock(&parent->d_lock);
1800 /*
1801 * don't need child lock because it is not subject
1802 * to concurrency here
1803 */
1804 __dget_dlock(parent);
1805 dentry->d_parent = parent;
1806 list_add(&dentry->d_child, &parent->d_subdirs);
1807 spin_unlock(&parent->d_lock);
1808
1809 return dentry;
1810 }
1811 EXPORT_SYMBOL(d_alloc);
1812
1813 struct dentry *d_alloc_anon(struct super_block *sb)
1814 {
1815 return __d_alloc(sb, NULL);
1816 }
1817 EXPORT_SYMBOL(d_alloc_anon);
1818
1819 struct dentry *d_alloc_cursor(struct dentry * parent)
1820 {
1821 struct dentry *dentry = d_alloc_anon(parent->d_sb);
1822 if (dentry) {
1823 dentry->d_flags |= DCACHE_DENTRY_CURSOR;
1824 dentry->d_parent = dget(parent);
1825 }
1826 return dentry;
1827 }
1828
1829 /**
1830 * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
1831 * @sb: the superblock
1832 * @name: qstr of the name
1833 *
1834 * For a filesystem that just pins its dentries in memory and never
1835 * performs lookups at all, return an unhashed IS_ROOT dentry.
1836 * This is used for pipes, sockets et.al. - the stuff that should
1837 * never be anyone's children or parents. Unlike all other
1838 * dentries, these will not have RCU delay between dropping the
1839 * last reference and freeing them.
1840 *
1841 * The only user is alloc_file_pseudo() and that's what should
1842 * be considered a public interface. Don't use directly.
1843 */
1844 struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
1845 {
1846 struct dentry *dentry = __d_alloc(sb, name);
1847 if (likely(dentry))
1848 dentry->d_flags |= DCACHE_NORCU;
1849 return dentry;
1850 }
1851
1852 struct dentry *d_alloc_name(struct dentry *parent, const char *name)
1853 {
1854 struct qstr q;
1855
1856 q.name = name;
1857 q.hash_len = hashlen_string(parent, name);
1858 return d_alloc(parent, &q);
1859 }
1860 EXPORT_SYMBOL(d_alloc_name);
1861
1862 void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
1863 {
1864 WARN_ON_ONCE(dentry->d_op);
1865 WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
1866 DCACHE_OP_COMPARE |
1867 DCACHE_OP_REVALIDATE |
1868 DCACHE_OP_WEAK_REVALIDATE |
1869 DCACHE_OP_DELETE |
1870 DCACHE_OP_REAL));
1871 dentry->d_op = op;
1872 if (!op)
1873 return;
1874 if (op->d_hash)
1875 dentry->d_flags |= DCACHE_OP_HASH;
1876 if (op->d_compare)
1877 dentry->d_flags |= DCACHE_OP_COMPARE;
1878 if (op->d_revalidate)
1879 dentry->d_flags |= DCACHE_OP_REVALIDATE;
1880 if (op->d_weak_revalidate)
1881 dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
1882 if (op->d_delete)
1883 dentry->d_flags |= DCACHE_OP_DELETE;
1884 if (op->d_prune)
1885 dentry->d_flags |= DCACHE_OP_PRUNE;
1886 if (op->d_real)
1887 dentry->d_flags |= DCACHE_OP_REAL;
1888
1889 }
1890 EXPORT_SYMBOL(d_set_d_op);
1891
1892
1893 /*
1894 * d_set_fallthru - Mark a dentry as falling through to a lower layer
1895 * @dentry - The dentry to mark
1896 *
1897 * Mark a dentry as falling through to the lower layer (as set with
1898 * d_pin_lower()). This flag may be recorded on the medium.
1899 */
1900 void d_set_fallthru(struct dentry *dentry)
1901 {
1902 spin_lock(&dentry->d_lock);
1903 dentry->d_flags |= DCACHE_FALLTHRU;
1904 spin_unlock(&dentry->d_lock);
1905 }
1906 EXPORT_SYMBOL(d_set_fallthru);
1907
1908 static unsigned d_flags_for_inode(struct inode *inode)
1909 {
1910 unsigned add_flags = DCACHE_REGULAR_TYPE;
1911
1912 if (!inode)
1913 return DCACHE_MISS_TYPE;
1914
1915 if (S_ISDIR(inode->i_mode)) {
1916 add_flags = DCACHE_DIRECTORY_TYPE;
1917 if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
1918 if (unlikely(!inode->i_op->lookup))
1919 add_flags = DCACHE_AUTODIR_TYPE;
1920 else
1921 inode->i_opflags |= IOP_LOOKUP;
1922 }
1923 goto type_determined;
1924 }
1925
1926 if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
1927 if (unlikely(inode->i_op->get_link)) {
1928 add_flags = DCACHE_SYMLINK_TYPE;
1929 goto type_determined;
1930 }
1931 inode->i_opflags |= IOP_NOFOLLOW;
1932 }
1933
1934 if (unlikely(!S_ISREG(inode->i_mode)))
1935 add_flags = DCACHE_SPECIAL_TYPE;
1936
1937 type_determined:
1938 if (unlikely(IS_AUTOMOUNT(inode)))
1939 add_flags |= DCACHE_NEED_AUTOMOUNT;
1940 return add_flags;
1941 }
1942
1943 static void __d_instantiate(struct dentry *dentry, struct inode *inode)
1944 {
1945 unsigned add_flags = d_flags_for_inode(inode);
1946 WARN_ON(d_in_lookup(dentry));
1947
1948 spin_lock(&dentry->d_lock);
1949 /*
1950 * Decrement negative dentry count if it was in the LRU list.
1951 */
1952 if (dentry->d_flags & DCACHE_LRU_LIST)
1953 this_cpu_dec(nr_dentry_negative);
1954 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
1955 raw_write_seqcount_begin(&dentry->d_seq);
1956 __d_set_inode_and_type(dentry, inode, add_flags);
1957 raw_write_seqcount_end(&dentry->d_seq);
1958 fsnotify_update_flags(dentry);
1959 spin_unlock(&dentry->d_lock);
1960 }
1961
1962 /**
1963 * d_instantiate - fill in inode information for a dentry
1964 * @entry: dentry to complete
1965 * @inode: inode to attach to this dentry
1966 *
1967 * Fill in inode information in the entry.
1968 *
1969 * This turns negative dentries into productive full members
1970 * of society.
1971 *
1972 * NOTE! This assumes that the inode count has been incremented
1973 * (or otherwise set) by the caller to indicate that it is now
1974 * in use by the dcache.
1975 */
1976
1977 void d_instantiate(struct dentry *entry, struct inode * inode)
1978 {
1979 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
1980 if (inode) {
1981 security_d_instantiate(entry, inode);
1982 spin_lock(&inode->i_lock);
1983 __d_instantiate(entry, inode);
1984 spin_unlock(&inode->i_lock);
1985 }
1986 }
1987 EXPORT_SYMBOL(d_instantiate);
1988
1989 /*
1990 * This should be equivalent to d_instantiate() + unlock_new_inode(),
1991 * with lockdep-related part of unlock_new_inode() done before
1992 * anything else. Use that instead of open-coding d_instantiate()/
1993 * unlock_new_inode() combinations.
1994 */
1995 void d_instantiate_new(struct dentry *entry, struct inode *inode)
1996 {
1997 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
1998 BUG_ON(!inode);
1999 lockdep_annotate_inode_mutex_key(inode);
2000 security_d_instantiate(entry, inode);
2001 spin_lock(&inode->i_lock);
2002 __d_instantiate(entry, inode);
2003 WARN_ON(!(inode->i_state & I_NEW));
2004 inode->i_state &= ~I_NEW & ~I_CREATING;
2005 smp_mb();
2006 wake_up_bit(&inode->i_state, __I_NEW);
2007 spin_unlock(&inode->i_lock);
2008 }
2009 EXPORT_SYMBOL(d_instantiate_new);
2010
2011 struct dentry *d_make_root(struct inode *root_inode)
2012 {
2013 struct dentry *res = NULL;
2014
2015 if (root_inode) {
2016 res = d_alloc_anon(root_inode->i_sb);
2017 if (res)
2018 d_instantiate(res, root_inode);
2019 else
2020 iput(root_inode);
2021 }
2022 return res;
2023 }
2024 EXPORT_SYMBOL(d_make_root);
2025
2026 static struct dentry *__d_instantiate_anon(struct dentry *dentry,
2027 struct inode *inode,
2028 bool disconnected)
2029 {
2030 struct dentry *res;
2031 unsigned add_flags;
2032
2033 security_d_instantiate(dentry, inode);
2034 spin_lock(&inode->i_lock);
2035 res = __d_find_any_alias(inode);
2036 if (res) {
2037 spin_unlock(&inode->i_lock);
2038 dput(dentry);
2039 goto out_iput;
2040 }
2041
2042 /* attach a disconnected dentry */
2043 add_flags = d_flags_for_inode(inode);
2044
2045 if (disconnected)
2046 add_flags |= DCACHE_DISCONNECTED;
2047
2048 spin_lock(&dentry->d_lock);
2049 __d_set_inode_and_type(dentry, inode, add_flags);
2050 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2051 if (!disconnected) {
2052 hlist_bl_lock(&dentry->d_sb->s_roots);
2053 hlist_bl_add_head(&dentry->d_hash, &dentry->d_sb->s_roots);
2054 hlist_bl_unlock(&dentry->d_sb->s_roots);
2055 }
2056 spin_unlock(&dentry->d_lock);
2057 spin_unlock(&inode->i_lock);
2058
2059 return dentry;
2060
2061 out_iput:
2062 iput(inode);
2063 return res;
2064 }
2065
2066 struct dentry *d_instantiate_anon(struct dentry *dentry, struct inode *inode)
2067 {
2068 return __d_instantiate_anon(dentry, inode, true);
2069 }
2070 EXPORT_SYMBOL(d_instantiate_anon);
2071
2072 static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
2073 {
2074 struct dentry *tmp;
2075 struct dentry *res;
2076
2077 if (!inode)
2078 return ERR_PTR(-ESTALE);
2079 if (IS_ERR(inode))
2080 return ERR_CAST(inode);
2081
2082 res = d_find_any_alias(inode);
2083 if (res)
2084 goto out_iput;
2085
2086 tmp = d_alloc_anon(inode->i_sb);
2087 if (!tmp) {
2088 res = ERR_PTR(-ENOMEM);
2089 goto out_iput;
2090 }
2091
2092 return __d_instantiate_anon(tmp, inode, disconnected);
2093
2094 out_iput:
2095 iput(inode);
2096 return res;
2097 }
2098
2099 /**
2100 * d_obtain_alias - find or allocate a DISCONNECTED dentry for a given inode
2101 * @inode: inode to allocate the dentry for
2102 *
2103 * Obtain a dentry for an inode resulting from NFS filehandle conversion or
2104 * similar open by handle operations. The returned dentry may be anonymous,
2105 * or may have a full name (if the inode was already in the cache).
2106 *
2107 * When called on a directory inode, we must ensure that the inode only ever
2108 * has one dentry. If a dentry is found, that is returned instead of
2109 * allocating a new one.
2110 *
2111 * On successful return, the reference to the inode has been transferred
2112 * to the dentry. In case of an error the reference on the inode is released.
2113 * To make it easier to use in export operations a %NULL or IS_ERR inode may
2114 * be passed in and the error will be propagated to the return value,
2115 * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
2116 */
2117 struct dentry *d_obtain_alias(struct inode *inode)
2118 {
2119 return __d_obtain_alias(inode, true);
2120 }
2121 EXPORT_SYMBOL(d_obtain_alias);
2122
2123 /**
2124 * d_obtain_root - find or allocate a dentry for a given inode
2125 * @inode: inode to allocate the dentry for
2126 *
2127 * Obtain an IS_ROOT dentry for the root of a filesystem.
2128 *
2129 * We must ensure that directory inodes only ever have one dentry. If a
2130 * dentry is found, that is returned instead of allocating a new one.
2131 *
2132 * On successful return, the reference to the inode has been transferred
2133 * to the dentry. In case of an error the reference on the inode is
2134 * released. A %NULL or IS_ERR inode may be passed in and will be the
2135 * error will be propagate to the return value, with a %NULL @inode
2136 * replaced by ERR_PTR(-ESTALE).
2137 */
2138 struct dentry *d_obtain_root(struct inode *inode)
2139 {
2140 return __d_obtain_alias(inode, false);
2141 }
2142 EXPORT_SYMBOL(d_obtain_root);
2143
2144 /**
2145 * d_add_ci - lookup or allocate new dentry with case-exact name
2146 * @inode: the inode case-insensitive lookup has found
2147 * @dentry: the negative dentry that was passed to the parent's lookup func
2148 * @name: the case-exact name to be associated with the returned dentry
2149 *
2150 * This is to avoid filling the dcache with case-insensitive names to the
2151 * same inode, only the actual correct case is stored in the dcache for
2152 * case-insensitive filesystems.
2153 *
2154 * For a case-insensitive lookup match and if the the case-exact dentry
2155 * already exists in in the dcache, use it and return it.
2156 *
2157 * If no entry exists with the exact case name, allocate new dentry with
2158 * the exact case, and return the spliced entry.
2159 */
2160 struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
2161 struct qstr *name)
2162 {
2163 struct dentry *found, *res;
2164
2165 /*
2166 * First check if a dentry matching the name already exists,
2167 * if not go ahead and create it now.
2168 */
2169 found = d_hash_and_lookup(dentry->d_parent, name);
2170 if (found) {
2171 iput(inode);
2172 return found;
2173 }
2174 if (d_in_lookup(dentry)) {
2175 found = d_alloc_parallel(dentry->d_parent, name,
2176 dentry->d_wait);
2177 if (IS_ERR(found) || !d_in_lookup(found)) {
2178 iput(inode);
2179 return found;
2180 }
2181 } else {
2182 found = d_alloc(dentry->d_parent, name);
2183 if (!found) {
2184 iput(inode);
2185 return ERR_PTR(-ENOMEM);
2186 }
2187 }
2188 res = d_splice_alias(inode, found);
2189 if (res) {
2190 dput(found);
2191 return res;
2192 }
2193 return found;
2194 }
2195 EXPORT_SYMBOL(d_add_ci);
2196
2197
2198 static inline bool d_same_name(const struct dentry *dentry,
2199 const struct dentry *parent,
2200 const struct qstr *name)
2201 {
2202 if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) {
2203 if (dentry->d_name.len != name->len)
2204 return false;
2205 return dentry_cmp(dentry, name->name, name->len) == 0;
2206 }
2207 return parent->d_op->d_compare(dentry,
2208 dentry->d_name.len, dentry->d_name.name,
2209 name) == 0;
2210 }
2211
2212 /**
2213 * __d_lookup_rcu - search for a dentry (racy, store-free)
2214 * @parent: parent dentry
2215 * @name: qstr of name we wish to find
2216 * @seqp: returns d_seq value at the point where the dentry was found
2217 * Returns: dentry, or NULL
2218 *
2219 * __d_lookup_rcu is the dcache lookup function for rcu-walk name
2220 * resolution (store-free path walking) design described in
2221 * Documentation/filesystems/path-lookup.txt.
2222 *
2223 * This is not to be used outside core vfs.
2224 *
2225 * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
2226 * held, and rcu_read_lock held. The returned dentry must not be stored into
2227 * without taking d_lock and checking d_seq sequence count against @seq
2228 * returned here.
2229 *
2230 * A refcount may be taken on the found dentry with the d_rcu_to_refcount
2231 * function.
2232 *
2233 * Alternatively, __d_lookup_rcu may be called again to look up the child of
2234 * the returned dentry, so long as its parent's seqlock is checked after the
2235 * child is looked up. Thus, an interlocking stepping of sequence lock checks
2236 * is formed, giving integrity down the path walk.
2237 *
2238 * NOTE! The caller *has* to check the resulting dentry against the sequence
2239 * number we've returned before using any of the resulting dentry state!
2240 */
2241 struct dentry *__d_lookup_rcu(const struct dentry *parent,
2242 const struct qstr *name,
2243 unsigned *seqp)
2244 {
2245 u64 hashlen = name->hash_len;
2246 const unsigned char *str = name->name;
2247 struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
2248 struct hlist_bl_node *node;
2249 struct dentry *dentry;
2250
2251 /*
2252 * Note: There is significant duplication with __d_lookup_rcu which is
2253 * required to prevent single threaded performance regressions
2254 * especially on architectures where smp_rmb (in seqcounts) are costly.
2255 * Keep the two functions in sync.
2256 */
2257
2258 /*
2259 * The hash list is protected using RCU.
2260 *
2261 * Carefully use d_seq when comparing a candidate dentry, to avoid
2262 * races with d_move().
2263 *
2264 * It is possible that concurrent renames can mess up our list
2265 * walk here and result in missing our dentry, resulting in the
2266 * false-negative result. d_lookup() protects against concurrent
2267 * renames using rename_lock seqlock.
2268 *
2269 * See Documentation/filesystems/path-lookup.txt for more details.
2270 */
2271 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2272 unsigned seq;
2273
2274 seqretry:
2275 /*
2276 * The dentry sequence count protects us from concurrent
2277 * renames, and thus protects parent and name fields.
2278 *
2279 * The caller must perform a seqcount check in order
2280 * to do anything useful with the returned dentry.
2281 *
2282 * NOTE! We do a "raw" seqcount_begin here. That means that
2283 * we don't wait for the sequence count to stabilize if it
2284 * is in the middle of a sequence change. If we do the slow
2285 * dentry compare, we will do seqretries until it is stable,
2286 * and if we end up with a successful lookup, we actually
2287 * want to exit RCU lookup anyway.
2288 *
2289 * Note that raw_seqcount_begin still *does* smp_rmb(), so
2290 * we are still guaranteed NUL-termination of ->d_name.name.
2291 */
2292 seq = raw_seqcount_begin(&dentry->d_seq);
2293 if (dentry->d_parent != parent)
2294 continue;
2295 if (d_unhashed(dentry))
2296 continue;
2297
2298 if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) {
2299 int tlen;
2300 const char *tname;
2301 if (dentry->d_name.hash != hashlen_hash(hashlen))
2302 continue;
2303 tlen = dentry->d_name.len;
2304 tname = dentry->d_name.name;
2305 /* we want a consistent (name,len) pair */
2306 if (read_seqcount_retry(&dentry->d_seq, seq)) {
2307 cpu_relax();
2308 goto seqretry;
2309 }
2310 if (parent->d_op->d_compare(dentry,
2311 tlen, tname, name) != 0)
2312 continue;
2313 } else {
2314 if (dentry->d_name.hash_len != hashlen)
2315 continue;
2316 if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)
2317 continue;
2318 }
2319 *seqp = seq;
2320 return dentry;
2321 }
2322 return NULL;
2323 }
2324
2325 /**
2326 * d_lookup - search for a dentry
2327 * @parent: parent dentry
2328 * @name: qstr of name we wish to find
2329 * Returns: dentry, or NULL
2330 *
2331 * d_lookup searches the children of the parent dentry for the name in
2332 * question. If the dentry is found its reference count is incremented and the
2333 * dentry is returned. The caller must use dput to free the entry when it has
2334 * finished using it. %NULL is returned if the dentry does not exist.
2335 */
2336 struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
2337 {
2338 struct dentry *dentry;
2339 unsigned seq;
2340
2341 do {
2342 seq = read_seqbegin(&rename_lock);
2343 dentry = __d_lookup(parent, name);
2344 if (dentry)
2345 break;
2346 } while (read_seqretry(&rename_lock, seq));
2347 return dentry;
2348 }
2349 EXPORT_SYMBOL(d_lookup);
2350
2351 /**
2352 * __d_lookup - search for a dentry (racy)
2353 * @parent: parent dentry
2354 * @name: qstr of name we wish to find
2355 * Returns: dentry, or NULL
2356 *
2357 * __d_lookup is like d_lookup, however it may (rarely) return a
2358 * false-negative result due to unrelated rename activity.
2359 *
2360 * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
2361 * however it must be used carefully, eg. with a following d_lookup in
2362 * the case of failure.
2363 *
2364 * __d_lookup callers must be commented.
2365 */
2366 struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
2367 {
2368 unsigned int hash = name->hash;
2369 struct hlist_bl_head *b = d_hash(hash);
2370 struct hlist_bl_node *node;
2371 struct dentry *found = NULL;
2372 struct dentry *dentry;
2373
2374 /*
2375 * Note: There is significant duplication with __d_lookup_rcu which is
2376 * required to prevent single threaded performance regressions
2377 * especially on architectures where smp_rmb (in seqcounts) are costly.
2378 * Keep the two functions in sync.
2379 */
2380
2381 /*
2382 * The hash list is protected using RCU.
2383 *
2384 * Take d_lock when comparing a candidate dentry, to avoid races
2385 * with d_move().
2386 *
2387 * It is possible that concurrent renames can mess up our list
2388 * walk here and result in missing our dentry, resulting in the
2389 * false-negative result. d_lookup() protects against concurrent
2390 * renames using rename_lock seqlock.
2391 *
2392 * See Documentation/filesystems/path-lookup.txt for more details.
2393 */
2394 rcu_read_lock();
2395
2396 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2397
2398 if (dentry->d_name.hash != hash)
2399 continue;
2400
2401 spin_lock(&dentry->d_lock);
2402 if (dentry->d_parent != parent)
2403 goto next;
2404 if (d_unhashed(dentry))
2405 goto next;
2406
2407 if (!d_same_name(dentry, parent, name))
2408 goto next;
2409
2410 dentry->d_lockref.count++;
2411 found = dentry;
2412 spin_unlock(&dentry->d_lock);
2413 break;
2414 next:
2415 spin_unlock(&dentry->d_lock);
2416 }
2417 rcu_read_unlock();
2418
2419 return found;
2420 }
2421
2422 /**
2423 * d_hash_and_lookup - hash the qstr then search for a dentry
2424 * @dir: Directory to search in
2425 * @name: qstr of name we wish to find
2426 *
2427 * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
2428 */
2429 struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
2430 {
2431 /*
2432 * Check for a fs-specific hash function. Note that we must
2433 * calculate the standard hash first, as the d_op->d_hash()
2434 * routine may choose to leave the hash value unchanged.
2435 */
2436 name->hash = full_name_hash(dir, name->name, name->len);
2437 if (dir->d_flags & DCACHE_OP_HASH) {
2438 int err = dir->d_op->d_hash(dir, name);
2439 if (unlikely(err < 0))
2440 return ERR_PTR(err);
2441 }
2442 return d_lookup(dir, name);
2443 }
2444 EXPORT_SYMBOL(d_hash_and_lookup);
2445
2446 /*
2447 * When a file is deleted, we have two options:
2448 * - turn this dentry into a negative dentry
2449 * - unhash this dentry and free it.
2450 *
2451 * Usually, we want to just turn this into
2452 * a negative dentry, but if anybody else is
2453 * currently using the dentry or the inode
2454 * we can't do that and we fall back on removing
2455 * it from the hash queues and waiting for
2456 * it to be deleted later when it has no users
2457 */
2458
2459 /**
2460 * d_delete - delete a dentry
2461 * @dentry: The dentry to delete
2462 *
2463 * Turn the dentry into a negative dentry if possible, otherwise
2464 * remove it from the hash queues so it can be deleted later
2465 */
2466
2467 void d_delete(struct dentry * dentry)
2468 {
2469 struct inode *inode = dentry->d_inode;
2470
2471 spin_lock(&inode->i_lock);
2472 spin_lock(&dentry->d_lock);
2473 /*
2474 * Are we the only user?
2475 */
2476 if (dentry->d_lockref.count == 1) {
2477 dentry->d_flags &= ~DCACHE_CANT_MOUNT;
2478 dentry_unlink_inode(dentry);
2479 } else {
2480 __d_drop(dentry);
2481 spin_unlock(&dentry->d_lock);
2482 spin_unlock(&inode->i_lock);
2483 }
2484 }
2485 EXPORT_SYMBOL(d_delete);
2486
2487 static void __d_rehash(struct dentry *entry)
2488 {
2489 struct hlist_bl_head *b = d_hash(entry->d_name.hash);
2490
2491 hlist_bl_lock(b);
2492 hlist_bl_add_head_rcu(&entry->d_hash, b);
2493 hlist_bl_unlock(b);
2494 }
2495
2496 /**
2497 * d_rehash - add an entry back to the hash
2498 * @entry: dentry to add to the hash
2499 *
2500 * Adds a dentry to the hash according to its name.
2501 */
2502
2503 void d_rehash(struct dentry * entry)
2504 {
2505 spin_lock(&entry->d_lock);
2506 __d_rehash(entry);
2507 spin_unlock(&entry->d_lock);
2508 }
2509 EXPORT_SYMBOL(d_rehash);
2510
2511 static inline unsigned start_dir_add(struct inode *dir)
2512 {
2513
2514 for (;;) {
2515 unsigned n = dir->i_dir_seq;
2516 if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
2517 return n;
2518 cpu_relax();
2519 }
2520 }
2521
2522 static inline void end_dir_add(struct inode *dir, unsigned n)
2523 {
2524 smp_store_release(&dir->i_dir_seq, n + 2);
2525 }
2526
2527 static void d_wait_lookup(struct dentry *dentry)
2528 {
2529 if (d_in_lookup(dentry)) {
2530 DECLARE_WAITQUEUE(wait, current);
2531 add_wait_queue(dentry->d_wait, &wait);
2532 do {
2533 set_current_state(TASK_UNINTERRUPTIBLE);
2534 spin_unlock(&dentry->d_lock);
2535 schedule();
2536 spin_lock(&dentry->d_lock);
2537 } while (d_in_lookup(dentry));
2538 }
2539 }
2540
2541 struct dentry *d_alloc_parallel(struct dentry *parent,
2542 const struct qstr *name,
2543 wait_queue_head_t *wq)
2544 {
2545 unsigned int hash = name->hash;
2546 struct hlist_bl_head *b = in_lookup_hash(parent, hash);
2547 struct hlist_bl_node *node;
2548 struct dentry *new = d_alloc(parent, name);
2549 struct dentry *dentry;
2550 unsigned seq, r_seq, d_seq;
2551
2552 if (unlikely(!new))
2553 return ERR_PTR(-ENOMEM);
2554
2555 retry:
2556 rcu_read_lock();
2557 seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
2558 r_seq = read_seqbegin(&rename_lock);
2559 dentry = __d_lookup_rcu(parent, name, &d_seq);
2560 if (unlikely(dentry)) {
2561 if (!lockref_get_not_dead(&dentry->d_lockref)) {
2562 rcu_read_unlock();
2563 goto retry;
2564 }
2565 if (read_seqcount_retry(&dentry->d_seq, d_seq)) {
2566 rcu_read_unlock();
2567 dput(dentry);
2568 goto retry;
2569 }
2570 rcu_read_unlock();
2571 dput(new);
2572 return dentry;
2573 }
2574 if (unlikely(read_seqretry(&rename_lock, r_seq))) {
2575 rcu_read_unlock();
2576 goto retry;
2577 }
2578
2579 if (unlikely(seq & 1)) {
2580 rcu_read_unlock();
2581 goto retry;
2582 }
2583
2584 hlist_bl_lock(b);
2585 if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
2586 hlist_bl_unlock(b);
2587 rcu_read_unlock();
2588 goto retry;
2589 }
2590 /*
2591 * No changes for the parent since the beginning of d_lookup().
2592 * Since all removals from the chain happen with hlist_bl_lock(),
2593 * any potential in-lookup matches are going to stay here until
2594 * we unlock the chain. All fields are stable in everything
2595 * we encounter.
2596 */
2597 hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) {
2598 if (dentry->d_name.hash != hash)
2599 continue;
2600 if (dentry->d_parent != parent)
2601 continue;
2602 if (!d_same_name(dentry, parent, name))
2603 continue;
2604 hlist_bl_unlock(b);
2605 /* now we can try to grab a reference */
2606 if (!lockref_get_not_dead(&dentry->d_lockref)) {
2607 rcu_read_unlock();
2608 goto retry;
2609 }
2610
2611 rcu_read_unlock();
2612 /*
2613 * somebody is likely to be still doing lookup for it;
2614 * wait for them to finish
2615 */
2616 spin_lock(&dentry->d_lock);
2617 d_wait_lookup(dentry);
2618 /*
2619 * it's not in-lookup anymore; in principle we should repeat
2620 * everything from dcache lookup, but it's likely to be what
2621 * d_lookup() would've found anyway. If it is, just return it;
2622 * otherwise we really have to repeat the whole thing.
2623 */
2624 if (unlikely(dentry->d_name.hash != hash))
2625 goto mismatch;
2626 if (unlikely(dentry->d_parent != parent))
2627 goto mismatch;
2628 if (unlikely(d_unhashed(dentry)))
2629 goto mismatch;
2630 if (unlikely(!d_same_name(dentry, parent, name)))
2631 goto mismatch;
2632 /* OK, it *is* a hashed match; return it */
2633 spin_unlock(&dentry->d_lock);
2634 dput(new);
2635 return dentry;
2636 }
2637 rcu_read_unlock();
2638 /* we can't take ->d_lock here; it's OK, though. */
2639 new->d_flags |= DCACHE_PAR_LOOKUP;
2640 new->d_wait = wq;
2641 hlist_bl_add_head_rcu(&new->d_u.d_in_lookup_hash, b);
2642 hlist_bl_unlock(b);
2643 return new;
2644 mismatch:
2645 spin_unlock(&dentry->d_lock);
2646 dput(dentry);
2647 goto retry;
2648 }
2649 EXPORT_SYMBOL(d_alloc_parallel);
2650
2651 void __d_lookup_done(struct dentry *dentry)
2652 {
2653 struct hlist_bl_head *b = in_lookup_hash(dentry->d_parent,
2654 dentry->d_name.hash);
2655 hlist_bl_lock(b);
2656 dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
2657 __hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
2658 wake_up_all(dentry->d_wait);
2659 dentry->d_wait = NULL;
2660 hlist_bl_unlock(b);
2661 INIT_HLIST_NODE(&dentry->d_u.d_alias);
2662 INIT_LIST_HEAD(&dentry->d_lru);
2663 }
2664 EXPORT_SYMBOL(__d_lookup_done);
2665
2666 /* inode->i_lock held if inode is non-NULL */
2667
2668 static inline void __d_add(struct dentry *dentry, struct inode *inode)
2669 {
2670 struct inode *dir = NULL;
2671 unsigned n;
2672 spin_lock(&dentry->d_lock);
2673 if (unlikely(d_in_lookup(dentry))) {
2674 dir = dentry->d_parent->d_inode;
2675 n = start_dir_add(dir);
2676 __d_lookup_done(dentry);
2677 }
2678 if (inode) {
2679 unsigned add_flags = d_flags_for_inode(inode);
2680 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2681 raw_write_seqcount_begin(&dentry->d_seq);
2682 __d_set_inode_and_type(dentry, inode, add_flags);
2683 raw_write_seqcount_end(&dentry->d_seq);
2684 fsnotify_update_flags(dentry);
2685 }
2686 __d_rehash(dentry);
2687 if (dir)
2688 end_dir_add(dir, n);
2689 spin_unlock(&dentry->d_lock);
2690 if (inode)
2691 spin_unlock(&inode->i_lock);
2692 }
2693
2694 /**
2695 * d_add - add dentry to hash queues
2696 * @entry: dentry to add
2697 * @inode: The inode to attach to this dentry
2698 *
2699 * This adds the entry to the hash queues and initializes @inode.
2700 * The entry was actually filled in earlier during d_alloc().
2701 */
2702
2703 void d_add(struct dentry *entry, struct inode *inode)
2704 {
2705 if (inode) {
2706 security_d_instantiate(entry, inode);
2707 spin_lock(&inode->i_lock);
2708 }
2709 __d_add(entry, inode);
2710 }
2711 EXPORT_SYMBOL(d_add);
2712
2713 /**
2714 * d_exact_alias - find and hash an exact unhashed alias
2715 * @entry: dentry to add
2716 * @inode: The inode to go with this dentry
2717 *
2718 * If an unhashed dentry with the same name/parent and desired
2719 * inode already exists, hash and return it. Otherwise, return
2720 * NULL.
2721 *
2722 * Parent directory should be locked.
2723 */
2724 struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode)
2725 {
2726 struct dentry *alias;
2727 unsigned int hash = entry->d_name.hash;
2728
2729 spin_lock(&inode->i_lock);
2730 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
2731 /*
2732 * Don't need alias->d_lock here, because aliases with
2733 * d_parent == entry->d_parent are not subject to name or
2734 * parent changes, because the parent inode i_mutex is held.
2735 */
2736 if (alias->d_name.hash != hash)
2737 continue;
2738 if (alias->d_parent != entry->d_parent)
2739 continue;
2740 if (!d_same_name(alias, entry->d_parent, &entry->d_name))
2741 continue;
2742 spin_lock(&alias->d_lock);
2743 if (!d_unhashed(alias)) {
2744 spin_unlock(&alias->d_lock);
2745 alias = NULL;
2746 } else {
2747 __dget_dlock(alias);
2748 __d_rehash(alias);
2749 spin_unlock(&alias->d_lock);
2750 }
2751 spin_unlock(&inode->i_lock);
2752 return alias;
2753 }
2754 spin_unlock(&inode->i_lock);
2755 return NULL;
2756 }
2757 EXPORT_SYMBOL(d_exact_alias);
2758
2759 static void swap_names(struct dentry *dentry, struct dentry *target)
2760 {
2761 if (unlikely(dname_external(target))) {
2762 if (unlikely(dname_external(dentry))) {
2763 /*
2764 * Both external: swap the pointers
2765 */
2766 swap(target->d_name.name, dentry->d_name.name);
2767 } else {
2768 /*
2769 * dentry:internal, target:external. Steal target's
2770 * storage and make target internal.
2771 */
2772 memcpy(target->d_iname, dentry->d_name.name,
2773 dentry->d_name.len + 1);
2774 dentry->d_name.name = target->d_name.name;
2775 target->d_name.name = target->d_iname;
2776 }
2777 } else {
2778 if (unlikely(dname_external(dentry))) {
2779 /*
2780 * dentry:external, target:internal. Give dentry's
2781 * storage to target and make dentry internal
2782 */
2783 memcpy(dentry->d_iname, target->d_name.name,
2784 target->d_name.len + 1);
2785 target->d_name.name = dentry->d_name.name;
2786 dentry->d_name.name = dentry->d_iname;
2787 } else {
2788 /*
2789 * Both are internal.
2790 */
2791 unsigned int i;
2792 BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long)));
2793 for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) {
2794 swap(((long *) &dentry->d_iname)[i],
2795 ((long *) &target->d_iname)[i]);
2796 }
2797 }
2798 }
2799 swap(dentry->d_name.hash_len, target->d_name.hash_len);
2800 }
2801
2802 static void copy_name(struct dentry *dentry, struct dentry *target)
2803 {
2804 struct external_name *old_name = NULL;
2805 if (unlikely(dname_external(dentry)))
2806 old_name = external_name(dentry);
2807 if (unlikely(dname_external(target))) {
2808 atomic_inc(&external_name(target)->u.count);
2809 dentry->d_name = target->d_name;
2810 } else {
2811 memcpy(dentry->d_iname, target->d_name.name,
2812 target->d_name.len + 1);
2813 dentry->d_name.name = dentry->d_iname;
2814 dentry->d_name.hash_len = target->d_name.hash_len;
2815 }
2816 if (old_name && likely(atomic_dec_and_test(&old_name->u.count)))
2817 kfree_rcu(old_name, u.head);
2818 }
2819
2820 /*
2821 * __d_move - move a dentry
2822 * @dentry: entry to move
2823 * @target: new dentry
2824 * @exchange: exchange the two dentries
2825 *
2826 * Update the dcache to reflect the move of a file name. Negative
2827 * dcache entries should not be moved in this way. Caller must hold
2828 * rename_lock, the i_mutex of the source and target directories,
2829 * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
2830 */
2831 static void __d_move(struct dentry *dentry, struct dentry *target,
2832 bool exchange)
2833 {
2834 struct dentry *old_parent, *p;
2835 struct inode *dir = NULL;
2836 unsigned n;
2837
2838 WARN_ON(!dentry->d_inode);
2839 if (WARN_ON(dentry == target))
2840 return;
2841
2842 BUG_ON(d_ancestor(target, dentry));
2843 old_parent = dentry->d_parent;
2844 p = d_ancestor(old_parent, target);
2845 if (IS_ROOT(dentry)) {
2846 BUG_ON(p);
2847 spin_lock(&target->d_parent->d_lock);
2848 } else if (!p) {
2849 /* target is not a descendent of dentry->d_parent */
2850 spin_lock(&target->d_parent->d_lock);
2851 spin_lock_nested(&old_parent->d_lock, DENTRY_D_LOCK_NESTED);
2852 } else {
2853 BUG_ON(p == dentry);
2854 spin_lock(&old_parent->d_lock);
2855 if (p != target)
2856 spin_lock_nested(&target->d_parent->d_lock,
2857 DENTRY_D_LOCK_NESTED);
2858 }
2859 spin_lock_nested(&dentry->d_lock, 2);
2860 spin_lock_nested(&target->d_lock, 3);
2861
2862 if (unlikely(d_in_lookup(target))) {
2863 dir = target->d_parent->d_inode;
2864 n = start_dir_add(dir);
2865 __d_lookup_done(target);
2866 }
2867
2868 write_seqcount_begin(&dentry->d_seq);
2869 write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
2870
2871 /* unhash both */
2872 if (!d_unhashed(dentry))
2873 ___d_drop(dentry);
2874 if (!d_unhashed(target))
2875 ___d_drop(target);
2876
2877 /* ... and switch them in the tree */
2878 dentry->d_parent = target->d_parent;
2879 if (!exchange) {
2880 copy_name(dentry, target);
2881 target->d_hash.pprev = NULL;
2882 dentry->d_parent->d_lockref.count++;
2883 if (dentry != old_parent) /* wasn't IS_ROOT */
2884 WARN_ON(!--old_parent->d_lockref.count);
2885 } else {
2886 target->d_parent = old_parent;
2887 swap_names(dentry, target);
2888 list_move(&target->d_child, &target->d_parent->d_subdirs);
2889 __d_rehash(target);
2890 fsnotify_update_flags(target);
2891 }
2892 list_move(&dentry->d_child, &dentry->d_parent->d_subdirs);
2893 __d_rehash(dentry);
2894 fsnotify_update_flags(dentry);
2895 fscrypt_handle_d_move(dentry);
2896
2897 write_seqcount_end(&target->d_seq);
2898 write_seqcount_end(&dentry->d_seq);
2899
2900 if (dir)
2901 end_dir_add(dir, n);
2902
2903 if (dentry->d_parent != old_parent)
2904 spin_unlock(&dentry->d_parent->d_lock);
2905 if (dentry != old_parent)
2906 spin_unlock(&old_parent->d_lock);
2907 spin_unlock(&target->d_lock);
2908 spin_unlock(&dentry->d_lock);
2909 }
2910
2911 /*
2912 * d_move - move a dentry
2913 * @dentry: entry to move
2914 * @target: new dentry
2915 *
2916 * Update the dcache to reflect the move of a file name. Negative
2917 * dcache entries should not be moved in this way. See the locking
2918 * requirements for __d_move.
2919 */
2920 void d_move(struct dentry *dentry, struct dentry *target)
2921 {
2922 write_seqlock(&rename_lock);
2923 __d_move(dentry, target, false);
2924 write_sequnlock(&rename_lock);
2925 }
2926 EXPORT_SYMBOL(d_move);
2927
2928 /*
2929 * d_exchange - exchange two dentries
2930 * @dentry1: first dentry
2931 * @dentry2: second dentry
2932 */
2933 void d_exchange(struct dentry *dentry1, struct dentry *dentry2)
2934 {
2935 write_seqlock(&rename_lock);
2936
2937 WARN_ON(!dentry1->d_inode);
2938 WARN_ON(!dentry2->d_inode);
2939 WARN_ON(IS_ROOT(dentry1));
2940 WARN_ON(IS_ROOT(dentry2));
2941
2942 __d_move(dentry1, dentry2, true);
2943
2944 write_sequnlock(&rename_lock);
2945 }
2946 EXPORT_SYMBOL_GPL(d_exchange);
2947
2948 /**
2949 * d_ancestor - search for an ancestor
2950 * @p1: ancestor dentry
2951 * @p2: child dentry
2952 *
2953 * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
2954 * an ancestor of p2, else NULL.
2955 */
2956 struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
2957 {
2958 struct dentry *p;
2959
2960 for (p = p2; !IS_ROOT(p); p = p->d_parent) {
2961 if (p->d_parent == p1)
2962 return p;
2963 }
2964 return NULL;
2965 }
2966
2967 /*
2968 * This helper attempts to cope with remotely renamed directories
2969 *
2970 * It assumes that the caller is already holding
2971 * dentry->d_parent->d_inode->i_mutex, and rename_lock
2972 *
2973 * Note: If ever the locking in lock_rename() changes, then please
2974 * remember to update this too...
2975 */
2976 static int __d_unalias(struct inode *inode,
2977 struct dentry *dentry, struct dentry *alias)
2978 {
2979 struct mutex *m1 = NULL;
2980 struct rw_semaphore *m2 = NULL;
2981 int ret = -ESTALE;
2982
2983 /* If alias and dentry share a parent, then no extra locks required */
2984 if (alias->d_parent == dentry->d_parent)
2985 goto out_unalias;
2986
2987 /* See lock_rename() */
2988 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
2989 goto out_err;
2990 m1 = &dentry->d_sb->s_vfs_rename_mutex;
2991 if (!inode_trylock_shared(alias->d_parent->d_inode))
2992 goto out_err;
2993 m2 = &alias->d_parent->d_inode->i_rwsem;
2994 out_unalias:
2995 __d_move(alias, dentry, false);
2996 ret = 0;
2997 out_err:
2998 if (m2)
2999 up_read(m2);
3000 if (m1)
3001 mutex_unlock(m1);
3002 return ret;
3003 }
3004
3005 /**
3006 * d_splice_alias - splice a disconnected dentry into the tree if one exists
3007 * @inode: the inode which may have a disconnected dentry
3008 * @dentry: a negative dentry which we want to point to the inode.
3009 *
3010 * If inode is a directory and has an IS_ROOT alias, then d_move that in
3011 * place of the given dentry and return it, else simply d_add the inode
3012 * to the dentry and return NULL.
3013 *
3014 * If a non-IS_ROOT directory is found, the filesystem is corrupt, and
3015 * we should error out: directories can't have multiple aliases.
3016 *
3017 * This is needed in the lookup routine of any filesystem that is exportable
3018 * (via knfsd) so that we can build dcache paths to directories effectively.
3019 *
3020 * If a dentry was found and moved, then it is returned. Otherwise NULL
3021 * is returned. This matches the expected return value of ->lookup.
3022 *
3023 * Cluster filesystems may call this function with a negative, hashed dentry.
3024 * In that case, we know that the inode will be a regular file, and also this
3025 * will only occur during atomic_open. So we need to check for the dentry
3026 * being already hashed only in the final case.
3027 */
3028 struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
3029 {
3030 if (IS_ERR(inode))
3031 return ERR_CAST(inode);
3032
3033 BUG_ON(!d_unhashed(dentry));
3034
3035 if (!inode)
3036 goto out;
3037
3038 security_d_instantiate(dentry, inode);
3039 spin_lock(&inode->i_lock);
3040 if (S_ISDIR(inode->i_mode)) {
3041 struct dentry *new = __d_find_any_alias(inode);
3042 if (unlikely(new)) {
3043 /* The reference to new ensures it remains an alias */
3044 spin_unlock(&inode->i_lock);
3045 write_seqlock(&rename_lock);
3046 if (unlikely(d_ancestor(new, dentry))) {
3047 write_sequnlock(&rename_lock);
3048 dput(new);
3049 new = ERR_PTR(-ELOOP);
3050 pr_warn_ratelimited(
3051 "VFS: Lookup of '%s' in %s %s"
3052 " would have caused loop\n",
3053 dentry->d_name.name,
3054 inode->i_sb->s_type->name,
3055 inode->i_sb->s_id);
3056 } else if (!IS_ROOT(new)) {
3057 struct dentry *old_parent = dget(new->d_parent);
3058 int err = __d_unalias(inode, dentry, new);
3059 write_sequnlock(&rename_lock);
3060 if (err) {
3061 dput(new);
3062 new = ERR_PTR(err);
3063 }
3064 dput(old_parent);
3065 } else {
3066 __d_move(new, dentry, false);
3067 write_sequnlock(&rename_lock);
3068 }
3069 iput(inode);
3070 return new;
3071 }
3072 }
3073 out:
3074 __d_add(dentry, inode);
3075 return NULL;
3076 }
3077 EXPORT_SYMBOL(d_splice_alias);
3078
3079 /*
3080 * Test whether new_dentry is a subdirectory of old_dentry.
3081 *
3082 * Trivially implemented using the dcache structure
3083 */
3084
3085 /**
3086 * is_subdir - is new dentry a subdirectory of old_dentry
3087 * @new_dentry: new dentry
3088 * @old_dentry: old dentry
3089 *
3090 * Returns true if new_dentry is a subdirectory of the parent (at any depth).
3091 * Returns false otherwise.
3092 * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
3093 */
3094
3095 bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
3096 {
3097 bool result;
3098 unsigned seq;
3099
3100 if (new_dentry == old_dentry)
3101 return true;
3102
3103 do {
3104 /* for restarting inner loop in case of seq retry */
3105 seq = read_seqbegin(&rename_lock);
3106 /*
3107 * Need rcu_readlock to protect against the d_parent trashing
3108 * due to d_move
3109 */
3110 rcu_read_lock();
3111 if (d_ancestor(old_dentry, new_dentry))
3112 result = true;
3113 else
3114 result = false;
3115 rcu_read_unlock();
3116 } while (read_seqretry(&rename_lock, seq));
3117
3118 return result;
3119 }
3120 EXPORT_SYMBOL(is_subdir);
3121
3122 static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
3123 {
3124 struct dentry *root = data;
3125 if (dentry != root) {
3126 if (d_unhashed(dentry) || !dentry->d_inode)
3127 return D_WALK_SKIP;
3128
3129 if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
3130 dentry->d_flags |= DCACHE_GENOCIDE;
3131 dentry->d_lockref.count--;
3132 }
3133 }
3134 return D_WALK_CONTINUE;
3135 }
3136
3137 void d_genocide(struct dentry *parent)
3138 {
3139 d_walk(parent, parent, d_genocide_kill);
3140 }
3141
3142 EXPORT_SYMBOL(d_genocide);
3143
3144 void d_tmpfile(struct dentry *dentry, struct inode *inode)
3145 {
3146 inode_dec_link_count(inode);
3147 BUG_ON(dentry->d_name.name != dentry->d_iname ||
3148 !hlist_unhashed(&dentry->d_u.d_alias) ||
3149 !d_unlinked(dentry));
3150 spin_lock(&dentry->d_parent->d_lock);
3151 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
3152 dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
3153 (unsigned long long)inode->i_ino);
3154 spin_unlock(&dentry->d_lock);
3155 spin_unlock(&dentry->d_parent->d_lock);
3156 d_instantiate(dentry, inode);
3157 }
3158 EXPORT_SYMBOL(d_tmpfile);
3159
3160 static __initdata unsigned long dhash_entries;
3161 static int __init set_dhash_entries(char *str)
3162 {
3163 if (!str)
3164 return 0;
3165 dhash_entries = simple_strtoul(str, &str, 0);
3166 return 1;
3167 }
3168 __setup("dhash_entries=", set_dhash_entries);
3169
3170 static void __init dcache_init_early(void)
3171 {
3172 /* If hashes are distributed across NUMA nodes, defer
3173 * hash allocation until vmalloc space is available.
3174 */
3175 if (hashdist)
3176 return;
3177
3178 dentry_hashtable =
3179 alloc_large_system_hash("Dentry cache",
3180 sizeof(struct hlist_bl_head),
3181 dhash_entries,
3182 13,
3183 HASH_EARLY | HASH_ZERO,
3184 &d_hash_shift,
3185 NULL,
3186 0,
3187 0);
3188 d_hash_shift = 32 - d_hash_shift;
3189 }
3190
3191 static void __init dcache_init(void)
3192 {
3193 /*
3194 * A constructor could be added for stable state like the lists,
3195 * but it is probably not worth it because of the cache nature
3196 * of the dcache.
3197 */
3198 dentry_cache = KMEM_CACHE_USERCOPY(dentry,
3199 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD|SLAB_ACCOUNT,
3200 d_iname);
3201
3202 /* Hash may have been set up in dcache_init_early */
3203 if (!hashdist)
3204 return;
3205
3206 dentry_hashtable =
3207 alloc_large_system_hash("Dentry cache",
3208 sizeof(struct hlist_bl_head),
3209 dhash_entries,
3210 13,
3211 HASH_ZERO,
3212 &d_hash_shift,
3213 NULL,
3214 0,
3215 0);
3216 d_hash_shift = 32 - d_hash_shift;
3217 }
3218
3219 /* SLAB cache for __getname() consumers */
3220 struct kmem_cache *names_cachep __read_mostly;
3221 EXPORT_SYMBOL(names_cachep);
3222
3223 void __init vfs_caches_init_early(void)
3224 {
3225 int i;
3226
3227 for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++)
3228 INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]);
3229
3230 dcache_init_early();
3231 inode_init_early();
3232 }
3233
3234 void __init vfs_caches_init(void)
3235 {
3236 names_cachep = kmem_cache_create_usercopy("names_cache", PATH_MAX, 0,
3237 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 0, PATH_MAX, NULL);
3238
3239 dcache_init();
3240 inode_init();
3241 files_init();
3242 files_maxfiles_init();
3243 mnt_init();
3244 bdev_cache_init();
3245 chrdev_init();
3246 }