1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 1991, 1992 Linus Torvalds
9 * Some corrections by tytso.
12 /* [Feb 1997 T. Schoebel-Theuer] Complete rewrite of the pathname
15 /* [Feb-Apr 2000, AV] Rewrite to the new namespace architecture.
18 #include <linux/init.h>
19 #include <linux/export.h>
20 #include <linux/kernel.h>
21 #include <linux/slab.h>
23 #include <linux/namei.h>
24 #include <linux/pagemap.h>
25 #include <linux/fsnotify.h>
26 #include <linux/personality.h>
27 #include <linux/security.h>
28 #include <linux/ima.h>
29 #include <linux/syscalls.h>
30 #include <linux/mount.h>
31 #include <linux/audit.h>
32 #include <linux/capability.h>
33 #include <linux/file.h>
34 #include <linux/fcntl.h>
35 #include <linux/device_cgroup.h>
36 #include <linux/fs_struct.h>
37 #include <linux/posix_acl.h>
38 #include <linux/hash.h>
39 #include <linux/bitops.h>
40 #include <linux/init_task.h>
41 #include <linux/uaccess.h>
46 /* [Feb-1997 T. Schoebel-Theuer]
47 * Fundamental changes in the pathname lookup mechanisms (namei)
48 * were necessary because of omirr. The reason is that omirr needs
49 * to know the _real_ pathname, not the user-supplied one, in case
50 * of symlinks (and also when transname replacements occur).
52 * The new code replaces the old recursive symlink resolution with
53 * an iterative one (in case of non-nested symlink chains). It does
54 * this with calls to <fs>_follow_link().
55 * As a side effect, dir_namei(), _namei() and follow_link() are now
56 * replaced with a single function lookup_dentry() that can handle all
57 * the special cases of the former code.
59 * With the new dcache, the pathname is stored at each inode, at least as
60 * long as the refcount of the inode is positive. As a side effect, the
61 * size of the dcache depends on the inode cache and thus is dynamic.
63 * [29-Apr-1998 C. Scott Ananian] Updated above description of symlink
64 * resolution to correspond with current state of the code.
66 * Note that the symlink resolution is not *completely* iterative.
67 * There is still a significant amount of tail- and mid- recursion in
68 * the algorithm. Also, note that <fs>_readlink() is not used in
69 * lookup_dentry(): lookup_dentry() on the result of <fs>_readlink()
70 * may return different results than <fs>_follow_link(). Many virtual
71 * filesystems (including /proc) exhibit this behavior.
74 /* [24-Feb-97 T. Schoebel-Theuer] Side effects caused by new implementation:
75 * New symlink semantics: when open() is called with flags O_CREAT | O_EXCL
76 * and the name already exists in form of a symlink, try to create the new
77 * name indicated by the symlink. The old code always complained that the
78 * name already exists, due to not following the symlink even if its target
79 * is nonexistent. The new semantics affects also mknod() and link() when
80 * the name is a symlink pointing to a non-existent name.
82 * I don't know which semantics is the right one, since I have no access
83 * to standards. But I found by trial that HP-UX 9.0 has the full "new"
84 * semantics implemented, while SunOS 4.1.1 and Solaris (SunOS 5.4) have the
85 * "old" one. Personally, I think the new semantics is much more logical.
86 * Note that "ln old new" where "new" is a symlink pointing to a non-existing
87 * file does succeed in both HP-UX and SunOs, but not in Solaris
88 * and in the old Linux semantics.
91 /* [16-Dec-97 Kevin Buhr] For security reasons, we change some symlink
92 * semantics. See the comments in "open_namei" and "do_link" below.
94 * [10-Sep-98 Alan Modra] Another symlink change.
97 /* [Feb-Apr 2000 AV] Complete rewrite. Rules for symlinks:
98 * inside the path - always follow.
99 * in the last component in creation/removal/renaming - never follow.
100 * if LOOKUP_FOLLOW passed - follow.
101 * if the pathname has trailing slashes - follow.
102 * otherwise - don't follow.
103 * (applied in that order).
105 * [Jun 2000 AV] Inconsistent behaviour of open() in case if flags==O_CREAT
106 * restored for 2.4. This is the last surviving part of old 4.2BSD bug.
107 * During the 2.4 we need to fix the userland stuff depending on it -
108 * hopefully we will be able to get rid of that wart in 2.5. So far only
109 * XEmacs seems to be relying on it...
112 * [Sep 2001 AV] Single-semaphore locking scheme (kudos to David Holland)
113 * implemented. Let's see if raised priority of ->s_vfs_rename_mutex gives
114 * any extra contention...
117 /* In order to reduce some races, while at the same time doing additional
118 * checking and hopefully speeding things up, we copy filenames to the
119 * kernel data space before using them..
121 * POSIX.1 2.4: an empty pathname is invalid (ENOENT).
122 * PATH_MAX includes the nul terminator --RR.
125 #define EMBEDDED_NAME_MAX (PATH_MAX - offsetof(struct filename, iname))
128 getname_flags(const char __user
*filename
, int flags
, int *empty
)
130 struct filename
*result
;
134 result
= audit_reusename(filename
);
138 result
= __getname();
139 if (unlikely(!result
))
140 return ERR_PTR(-ENOMEM
);
143 * First, try to embed the struct filename inside the names_cache
146 kname
= (char *)result
->iname
;
147 result
->name
= kname
;
149 len
= strncpy_from_user(kname
, filename
, EMBEDDED_NAME_MAX
);
150 if (unlikely(len
< 0)) {
156 * Uh-oh. We have a name that's approaching PATH_MAX. Allocate a
157 * separate struct filename so we can dedicate the entire
158 * names_cache allocation for the pathname, and re-do the copy from
161 if (unlikely(len
== EMBEDDED_NAME_MAX
)) {
162 const size_t size
= offsetof(struct filename
, iname
[1]);
163 kname
= (char *)result
;
166 * size is chosen that way we to guarantee that
167 * result->iname[0] is within the same object and that
168 * kname can't be equal to result->iname, no matter what.
170 result
= kzalloc(size
, GFP_KERNEL
);
171 if (unlikely(!result
)) {
173 return ERR_PTR(-ENOMEM
);
175 result
->name
= kname
;
176 len
= strncpy_from_user(kname
, filename
, PATH_MAX
);
177 if (unlikely(len
< 0)) {
182 if (unlikely(len
== PATH_MAX
)) {
185 return ERR_PTR(-ENAMETOOLONG
);
190 /* The empty path is special. */
191 if (unlikely(!len
)) {
194 if (!(flags
& LOOKUP_EMPTY
)) {
196 return ERR_PTR(-ENOENT
);
200 result
->uptr
= filename
;
201 result
->aname
= NULL
;
202 audit_getname(result
);
207 getname_uflags(const char __user
*filename
, int uflags
)
209 int flags
= (uflags
& AT_EMPTY_PATH
) ? LOOKUP_EMPTY
: 0;
211 return getname_flags(filename
, flags
, NULL
);
215 getname(const char __user
* filename
)
217 return getname_flags(filename
, 0, NULL
);
221 getname_kernel(const char * filename
)
223 struct filename
*result
;
224 int len
= strlen(filename
) + 1;
226 result
= __getname();
227 if (unlikely(!result
))
228 return ERR_PTR(-ENOMEM
);
230 if (len
<= EMBEDDED_NAME_MAX
) {
231 result
->name
= (char *)result
->iname
;
232 } else if (len
<= PATH_MAX
) {
233 const size_t size
= offsetof(struct filename
, iname
[1]);
234 struct filename
*tmp
;
236 tmp
= kmalloc(size
, GFP_KERNEL
);
237 if (unlikely(!tmp
)) {
239 return ERR_PTR(-ENOMEM
);
241 tmp
->name
= (char *)result
;
245 return ERR_PTR(-ENAMETOOLONG
);
247 memcpy((char *)result
->name
, filename
, len
);
249 result
->aname
= NULL
;
251 audit_getname(result
);
256 void putname(struct filename
*name
)
261 BUG_ON(name
->refcnt
<= 0);
263 if (--name
->refcnt
> 0)
266 if (name
->name
!= name
->iname
) {
267 __putname(name
->name
);
274 * check_acl - perform ACL permission checking
275 * @mnt_userns: user namespace of the mount the inode was found from
276 * @inode: inode to check permissions on
277 * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...)
279 * This function performs the ACL permission checking. Since this function
280 * retrieve POSIX acls it needs to know whether it is called from a blocking or
281 * non-blocking context and thus cares about the MAY_NOT_BLOCK bit.
283 * If the inode has been found through an idmapped mount the user namespace of
284 * the vfsmount must be passed through @mnt_userns. This function will then take
285 * care to map the inode according to @mnt_userns before checking permissions.
286 * On non-idmapped mounts or if permission checking is to be performed on the
287 * raw inode simply passs init_user_ns.
289 static int check_acl(struct user_namespace
*mnt_userns
,
290 struct inode
*inode
, int mask
)
292 #ifdef CONFIG_FS_POSIX_ACL
293 struct posix_acl
*acl
;
295 if (mask
& MAY_NOT_BLOCK
) {
296 acl
= get_cached_acl_rcu(inode
, ACL_TYPE_ACCESS
);
299 /* no ->get_acl() calls in RCU mode... */
300 if (is_uncached_acl(acl
))
302 return posix_acl_permission(mnt_userns
, inode
, acl
, mask
);
305 acl
= get_acl(inode
, ACL_TYPE_ACCESS
);
309 int error
= posix_acl_permission(mnt_userns
, inode
, acl
, mask
);
310 posix_acl_release(acl
);
319 * acl_permission_check - perform basic UNIX permission checking
320 * @mnt_userns: user namespace of the mount the inode was found from
321 * @inode: inode to check permissions on
322 * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...)
324 * This function performs the basic UNIX permission checking. Since this
325 * function may retrieve POSIX acls it needs to know whether it is called from a
326 * blocking or non-blocking context and thus cares about the MAY_NOT_BLOCK bit.
328 * If the inode has been found through an idmapped mount the user namespace of
329 * the vfsmount must be passed through @mnt_userns. This function will then take
330 * care to map the inode according to @mnt_userns before checking permissions.
331 * On non-idmapped mounts or if permission checking is to be performed on the
332 * raw inode simply passs init_user_ns.
334 static int acl_permission_check(struct user_namespace
*mnt_userns
,
335 struct inode
*inode
, int mask
)
337 unsigned int mode
= inode
->i_mode
;
340 /* Are we the owner? If so, ACL's don't matter */
341 i_uid
= i_uid_into_mnt(mnt_userns
, inode
);
342 if (likely(uid_eq(current_fsuid(), i_uid
))) {
345 return (mask
& ~mode
) ? -EACCES
: 0;
348 /* Do we have ACL's? */
349 if (IS_POSIXACL(inode
) && (mode
& S_IRWXG
)) {
350 int error
= check_acl(mnt_userns
, inode
, mask
);
351 if (error
!= -EAGAIN
)
355 /* Only RWX matters for group/other mode bits */
359 * Are the group permissions different from
360 * the other permissions in the bits we care
361 * about? Need to check group ownership if so.
363 if (mask
& (mode
^ (mode
>> 3))) {
364 kgid_t kgid
= i_gid_into_mnt(mnt_userns
, inode
);
365 if (in_group_p(kgid
))
369 /* Bits in 'mode' clear that we require? */
370 return (mask
& ~mode
) ? -EACCES
: 0;
374 * generic_permission - check for access rights on a Posix-like filesystem
375 * @mnt_userns: user namespace of the mount the inode was found from
376 * @inode: inode to check access rights for
377 * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC,
378 * %MAY_NOT_BLOCK ...)
380 * Used to check for read/write/execute permissions on a file.
381 * We use "fsuid" for this, letting us set arbitrary permissions
382 * for filesystem access without changing the "normal" uids which
383 * are used for other things.
385 * generic_permission is rcu-walk aware. It returns -ECHILD in case an rcu-walk
386 * request cannot be satisfied (eg. requires blocking or too much complexity).
387 * It would then be called again in ref-walk mode.
389 * If the inode has been found through an idmapped mount the user namespace of
390 * the vfsmount must be passed through @mnt_userns. This function will then take
391 * care to map the inode according to @mnt_userns before checking permissions.
392 * On non-idmapped mounts or if permission checking is to be performed on the
393 * raw inode simply passs init_user_ns.
395 int generic_permission(struct user_namespace
*mnt_userns
, struct inode
*inode
,
401 * Do the basic permission checks.
403 ret
= acl_permission_check(mnt_userns
, inode
, mask
);
407 if (S_ISDIR(inode
->i_mode
)) {
408 /* DACs are overridable for directories */
409 if (!(mask
& MAY_WRITE
))
410 if (capable_wrt_inode_uidgid(mnt_userns
, inode
,
411 CAP_DAC_READ_SEARCH
))
413 if (capable_wrt_inode_uidgid(mnt_userns
, inode
,
420 * Searching includes executable on directories, else just read.
422 mask
&= MAY_READ
| MAY_WRITE
| MAY_EXEC
;
423 if (mask
== MAY_READ
)
424 if (capable_wrt_inode_uidgid(mnt_userns
, inode
,
425 CAP_DAC_READ_SEARCH
))
428 * Read/write DACs are always overridable.
429 * Executable DACs are overridable when there is
430 * at least one exec bit set.
432 if (!(mask
& MAY_EXEC
) || (inode
->i_mode
& S_IXUGO
))
433 if (capable_wrt_inode_uidgid(mnt_userns
, inode
,
439 EXPORT_SYMBOL(generic_permission
);
442 * do_inode_permission - UNIX permission checking
443 * @mnt_userns: user namespace of the mount the inode was found from
444 * @inode: inode to check permissions on
445 * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...)
447 * We _really_ want to just do "generic_permission()" without
448 * even looking at the inode->i_op values. So we keep a cache
449 * flag in inode->i_opflags, that says "this has not special
450 * permission function, use the fast case".
452 static inline int do_inode_permission(struct user_namespace
*mnt_userns
,
453 struct inode
*inode
, int mask
)
455 if (unlikely(!(inode
->i_opflags
& IOP_FASTPERM
))) {
456 if (likely(inode
->i_op
->permission
))
457 return inode
->i_op
->permission(mnt_userns
, inode
, mask
);
459 /* This gets set once for the inode lifetime */
460 spin_lock(&inode
->i_lock
);
461 inode
->i_opflags
|= IOP_FASTPERM
;
462 spin_unlock(&inode
->i_lock
);
464 return generic_permission(mnt_userns
, inode
, mask
);
468 * sb_permission - Check superblock-level permissions
469 * @sb: Superblock of inode to check permission on
470 * @inode: Inode to check permission on
471 * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
473 * Separate out file-system wide checks from inode-specific permission checks.
475 static int sb_permission(struct super_block
*sb
, struct inode
*inode
, int mask
)
477 if (unlikely(mask
& MAY_WRITE
)) {
478 umode_t mode
= inode
->i_mode
;
480 /* Nobody gets write access to a read-only fs. */
481 if (sb_rdonly(sb
) && (S_ISREG(mode
) || S_ISDIR(mode
) || S_ISLNK(mode
)))
488 * inode_permission - Check for access rights to a given inode
489 * @mnt_userns: User namespace of the mount the inode was found from
490 * @inode: Inode to check permission on
491 * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
493 * Check for read/write/execute permissions on an inode. We use fs[ug]id for
494 * this, letting us set arbitrary permissions for filesystem access without
495 * changing the "normal" UIDs which are used for other things.
497 * When checking for MAY_APPEND, MAY_WRITE must also be set in @mask.
499 int inode_permission(struct user_namespace
*mnt_userns
,
500 struct inode
*inode
, int mask
)
504 retval
= sb_permission(inode
->i_sb
, inode
, mask
);
508 if (unlikely(mask
& MAY_WRITE
)) {
510 * Nobody gets write access to an immutable file.
512 if (IS_IMMUTABLE(inode
))
516 * Updating mtime will likely cause i_uid and i_gid to be
517 * written back improperly if their true value is unknown
520 if (HAS_UNMAPPED_ID(mnt_userns
, inode
))
524 retval
= do_inode_permission(mnt_userns
, inode
, mask
);
528 retval
= devcgroup_inode_permission(inode
, mask
);
532 return security_inode_permission(inode
, mask
);
534 EXPORT_SYMBOL(inode_permission
);
537 * path_get - get a reference to a path
538 * @path: path to get the reference to
540 * Given a path increment the reference count to the dentry and the vfsmount.
542 void path_get(const struct path
*path
)
547 EXPORT_SYMBOL(path_get
);
550 * path_put - put a reference to a path
551 * @path: path to put the reference to
553 * Given a path decrement the reference count to the dentry and the vfsmount.
555 void path_put(const struct path
*path
)
560 EXPORT_SYMBOL(path_put
);
562 #define EMBEDDED_LEVELS 2
567 struct inode
*inode
; /* path.dentry.d_inode */
568 unsigned int flags
, state
;
569 unsigned seq
, m_seq
, r_seq
;
572 int total_link_count
;
575 struct delayed_call done
;
578 } *stack
, internal
[EMBEDDED_LEVELS
];
579 struct filename
*name
;
580 struct nameidata
*saved
;
585 } __randomize_layout
;
587 #define ND_ROOT_PRESET 1
588 #define ND_ROOT_GRABBED 2
591 static void __set_nameidata(struct nameidata
*p
, int dfd
, struct filename
*name
)
593 struct nameidata
*old
= current
->nameidata
;
594 p
->stack
= p
->internal
;
599 p
->path
.dentry
= NULL
;
600 p
->total_link_count
= old
? old
->total_link_count
: 0;
602 current
->nameidata
= p
;
605 static inline void set_nameidata(struct nameidata
*p
, int dfd
, struct filename
*name
,
606 const struct path
*root
)
608 __set_nameidata(p
, dfd
, name
);
610 if (unlikely(root
)) {
611 p
->state
= ND_ROOT_PRESET
;
616 static void restore_nameidata(void)
618 struct nameidata
*now
= current
->nameidata
, *old
= now
->saved
;
620 current
->nameidata
= old
;
622 old
->total_link_count
= now
->total_link_count
;
623 if (now
->stack
!= now
->internal
)
627 static bool nd_alloc_stack(struct nameidata
*nd
)
631 p
= kmalloc_array(MAXSYMLINKS
, sizeof(struct saved
),
632 nd
->flags
& LOOKUP_RCU
? GFP_ATOMIC
: GFP_KERNEL
);
635 memcpy(p
, nd
->internal
, sizeof(nd
->internal
));
641 * path_connected - Verify that a dentry is below mnt.mnt_root
643 * Rename can sometimes move a file or directory outside of a bind
644 * mount, path_connected allows those cases to be detected.
646 static bool path_connected(struct vfsmount
*mnt
, struct dentry
*dentry
)
648 struct super_block
*sb
= mnt
->mnt_sb
;
650 /* Bind mounts can have disconnected paths */
651 if (mnt
->mnt_root
== sb
->s_root
)
654 return is_subdir(dentry
, mnt
->mnt_root
);
657 static void drop_links(struct nameidata
*nd
)
661 struct saved
*last
= nd
->stack
+ i
;
662 do_delayed_call(&last
->done
);
663 clear_delayed_call(&last
->done
);
667 static void terminate_walk(struct nameidata
*nd
)
670 if (!(nd
->flags
& LOOKUP_RCU
)) {
673 for (i
= 0; i
< nd
->depth
; i
++)
674 path_put(&nd
->stack
[i
].link
);
675 if (nd
->state
& ND_ROOT_GRABBED
) {
677 nd
->state
&= ~ND_ROOT_GRABBED
;
680 nd
->flags
&= ~LOOKUP_RCU
;
685 nd
->path
.dentry
= NULL
;
688 /* path_put is needed afterwards regardless of success or failure */
689 static bool __legitimize_path(struct path
*path
, unsigned seq
, unsigned mseq
)
691 int res
= __legitimize_mnt(path
->mnt
, mseq
);
698 if (unlikely(!lockref_get_not_dead(&path
->dentry
->d_lockref
))) {
702 return !read_seqcount_retry(&path
->dentry
->d_seq
, seq
);
705 static inline bool legitimize_path(struct nameidata
*nd
,
706 struct path
*path
, unsigned seq
)
708 return __legitimize_path(path
, seq
, nd
->m_seq
);
711 static bool legitimize_links(struct nameidata
*nd
)
714 if (unlikely(nd
->flags
& LOOKUP_CACHED
)) {
719 for (i
= 0; i
< nd
->depth
; i
++) {
720 struct saved
*last
= nd
->stack
+ i
;
721 if (unlikely(!legitimize_path(nd
, &last
->link
, last
->seq
))) {
730 static bool legitimize_root(struct nameidata
*nd
)
733 * For scoped-lookups (where nd->root has been zeroed), we need to
734 * restart the whole lookup from scratch -- because set_root() is wrong
735 * for these lookups (nd->dfd is the root, not the filesystem root).
737 if (!nd
->root
.mnt
&& (nd
->flags
& LOOKUP_IS_SCOPED
))
739 /* Nothing to do if nd->root is zero or is managed by the VFS user. */
740 if (!nd
->root
.mnt
|| (nd
->state
& ND_ROOT_PRESET
))
742 nd
->state
|= ND_ROOT_GRABBED
;
743 return legitimize_path(nd
, &nd
->root
, nd
->root_seq
);
747 * Path walking has 2 modes, rcu-walk and ref-walk (see
748 * Documentation/filesystems/path-lookup.txt). In situations when we can't
749 * continue in RCU mode, we attempt to drop out of rcu-walk mode and grab
750 * normal reference counts on dentries and vfsmounts to transition to ref-walk
751 * mode. Refcounts are grabbed at the last known good point before rcu-walk
752 * got stuck, so ref-walk may continue from there. If this is not successful
753 * (eg. a seqcount has changed), then failure is returned and it's up to caller
754 * to restart the path walk from the beginning in ref-walk mode.
758 * try_to_unlazy - try to switch to ref-walk mode.
759 * @nd: nameidata pathwalk data
760 * Returns: true on success, false on failure
762 * try_to_unlazy attempts to legitimize the current nd->path and nd->root
764 * Must be called from rcu-walk context.
765 * Nothing should touch nameidata between try_to_unlazy() failure and
768 static bool try_to_unlazy(struct nameidata
*nd
)
770 struct dentry
*parent
= nd
->path
.dentry
;
772 BUG_ON(!(nd
->flags
& LOOKUP_RCU
));
774 nd
->flags
&= ~LOOKUP_RCU
;
775 if (unlikely(!legitimize_links(nd
)))
777 if (unlikely(!legitimize_path(nd
, &nd
->path
, nd
->seq
)))
779 if (unlikely(!legitimize_root(nd
)))
782 BUG_ON(nd
->inode
!= parent
->d_inode
);
787 nd
->path
.dentry
= NULL
;
794 * try_to_unlazy_next - try to switch to ref-walk mode.
795 * @nd: nameidata pathwalk data
796 * @dentry: next dentry to step into
797 * @seq: seq number to check @dentry against
798 * Returns: true on success, false on failure
800 * Similar to to try_to_unlazy(), but here we have the next dentry already
801 * picked by rcu-walk and want to legitimize that in addition to the current
802 * nd->path and nd->root for ref-walk mode. Must be called from rcu-walk context.
803 * Nothing should touch nameidata between try_to_unlazy_next() failure and
806 static bool try_to_unlazy_next(struct nameidata
*nd
, struct dentry
*dentry
, unsigned seq
)
808 BUG_ON(!(nd
->flags
& LOOKUP_RCU
));
810 nd
->flags
&= ~LOOKUP_RCU
;
811 if (unlikely(!legitimize_links(nd
)))
813 if (unlikely(!legitimize_mnt(nd
->path
.mnt
, nd
->m_seq
)))
815 if (unlikely(!lockref_get_not_dead(&nd
->path
.dentry
->d_lockref
)))
819 * We need to move both the parent and the dentry from the RCU domain
820 * to be properly refcounted. And the sequence number in the dentry
821 * validates *both* dentry counters, since we checked the sequence
822 * number of the parent after we got the child sequence number. So we
823 * know the parent must still be valid if the child sequence number is
825 if (unlikely(!lockref_get_not_dead(&dentry
->d_lockref
)))
827 if (unlikely(read_seqcount_retry(&dentry
->d_seq
, seq
)))
830 * Sequence counts matched. Now make sure that the root is
831 * still valid and get it if required.
833 if (unlikely(!legitimize_root(nd
)))
841 nd
->path
.dentry
= NULL
;
851 static inline int d_revalidate(struct dentry
*dentry
, unsigned int flags
)
853 if (unlikely(dentry
->d_flags
& DCACHE_OP_REVALIDATE
))
854 return dentry
->d_op
->d_revalidate(dentry
, flags
);
860 * complete_walk - successful completion of path walk
861 * @nd: pointer nameidata
863 * If we had been in RCU mode, drop out of it and legitimize nd->path.
864 * Revalidate the final result, unless we'd already done that during
865 * the path walk or the filesystem doesn't ask for it. Return 0 on
866 * success, -error on failure. In case of failure caller does not
867 * need to drop nd->path.
869 static int complete_walk(struct nameidata
*nd
)
871 struct dentry
*dentry
= nd
->path
.dentry
;
874 if (nd
->flags
& LOOKUP_RCU
) {
876 * We don't want to zero nd->root for scoped-lookups or
877 * externally-managed nd->root.
879 if (!(nd
->state
& ND_ROOT_PRESET
))
880 if (!(nd
->flags
& LOOKUP_IS_SCOPED
))
882 nd
->flags
&= ~LOOKUP_CACHED
;
883 if (!try_to_unlazy(nd
))
887 if (unlikely(nd
->flags
& LOOKUP_IS_SCOPED
)) {
889 * While the guarantee of LOOKUP_IS_SCOPED is (roughly) "don't
890 * ever step outside the root during lookup" and should already
891 * be guaranteed by the rest of namei, we want to avoid a namei
892 * BUG resulting in userspace being given a path that was not
893 * scoped within the root at some point during the lookup.
895 * So, do a final sanity-check to make sure that in the
896 * worst-case scenario (a complete bypass of LOOKUP_IS_SCOPED)
897 * we won't silently return an fd completely outside of the
898 * requested root to userspace.
900 * Userspace could move the path outside the root after this
901 * check, but as discussed elsewhere this is not a concern (the
902 * resolved file was inside the root at some point).
904 if (!path_is_under(&nd
->path
, &nd
->root
))
908 if (likely(!(nd
->state
& ND_JUMPED
)))
911 if (likely(!(dentry
->d_flags
& DCACHE_OP_WEAK_REVALIDATE
)))
914 status
= dentry
->d_op
->d_weak_revalidate(dentry
, nd
->flags
);
924 static int set_root(struct nameidata
*nd
)
926 struct fs_struct
*fs
= current
->fs
;
929 * Jumping to the real root in a scoped-lookup is a BUG in namei, but we
930 * still have to ensure it doesn't happen because it will cause a breakout
933 if (WARN_ON(nd
->flags
& LOOKUP_IS_SCOPED
))
934 return -ENOTRECOVERABLE
;
936 if (nd
->flags
& LOOKUP_RCU
) {
940 seq
= read_seqcount_begin(&fs
->seq
);
942 nd
->root_seq
= __read_seqcount_begin(&nd
->root
.dentry
->d_seq
);
943 } while (read_seqcount_retry(&fs
->seq
, seq
));
945 get_fs_root(fs
, &nd
->root
);
946 nd
->state
|= ND_ROOT_GRABBED
;
951 static int nd_jump_root(struct nameidata
*nd
)
953 if (unlikely(nd
->flags
& LOOKUP_BENEATH
))
955 if (unlikely(nd
->flags
& LOOKUP_NO_XDEV
)) {
956 /* Absolute path arguments to path_init() are allowed. */
957 if (nd
->path
.mnt
!= NULL
&& nd
->path
.mnt
!= nd
->root
.mnt
)
961 int error
= set_root(nd
);
965 if (nd
->flags
& LOOKUP_RCU
) {
969 nd
->inode
= d
->d_inode
;
970 nd
->seq
= nd
->root_seq
;
971 if (unlikely(read_seqcount_retry(&d
->d_seq
, nd
->seq
)))
977 nd
->inode
= nd
->path
.dentry
->d_inode
;
979 nd
->state
|= ND_JUMPED
;
984 * Helper to directly jump to a known parsed path from ->get_link,
985 * caller must have taken a reference to path beforehand.
987 int nd_jump_link(struct path
*path
)
990 struct nameidata
*nd
= current
->nameidata
;
992 if (unlikely(nd
->flags
& LOOKUP_NO_MAGICLINKS
))
996 if (unlikely(nd
->flags
& LOOKUP_NO_XDEV
)) {
997 if (nd
->path
.mnt
!= path
->mnt
)
1000 /* Not currently safe for scoped-lookups. */
1001 if (unlikely(nd
->flags
& LOOKUP_IS_SCOPED
))
1004 path_put(&nd
->path
);
1006 nd
->inode
= nd
->path
.dentry
->d_inode
;
1007 nd
->state
|= ND_JUMPED
;
1015 static inline void put_link(struct nameidata
*nd
)
1017 struct saved
*last
= nd
->stack
+ --nd
->depth
;
1018 do_delayed_call(&last
->done
);
1019 if (!(nd
->flags
& LOOKUP_RCU
))
1020 path_put(&last
->link
);
1023 int sysctl_protected_symlinks __read_mostly
= 0;
1024 int sysctl_protected_hardlinks __read_mostly
= 0;
1025 int sysctl_protected_fifos __read_mostly
;
1026 int sysctl_protected_regular __read_mostly
;
1029 * may_follow_link - Check symlink following for unsafe situations
1030 * @nd: nameidata pathwalk data
1032 * In the case of the sysctl_protected_symlinks sysctl being enabled,
1033 * CAP_DAC_OVERRIDE needs to be specifically ignored if the symlink is
1034 * in a sticky world-writable directory. This is to protect privileged
1035 * processes from failing races against path names that may change out
1036 * from under them by way of other users creating malicious symlinks.
1037 * It will permit symlinks to be followed only when outside a sticky
1038 * world-writable directory, or when the uid of the symlink and follower
1039 * match, or when the directory owner matches the symlink's owner.
1041 * Returns 0 if following the symlink is allowed, -ve on error.
1043 static inline int may_follow_link(struct nameidata
*nd
, const struct inode
*inode
)
1045 struct user_namespace
*mnt_userns
;
1048 if (!sysctl_protected_symlinks
)
1051 mnt_userns
= mnt_user_ns(nd
->path
.mnt
);
1052 i_uid
= i_uid_into_mnt(mnt_userns
, inode
);
1053 /* Allowed if owner and follower match. */
1054 if (uid_eq(current_cred()->fsuid
, i_uid
))
1057 /* Allowed if parent directory not sticky and world-writable. */
1058 if ((nd
->dir_mode
& (S_ISVTX
|S_IWOTH
)) != (S_ISVTX
|S_IWOTH
))
1061 /* Allowed if parent directory and link owner match. */
1062 if (uid_valid(nd
->dir_uid
) && uid_eq(nd
->dir_uid
, i_uid
))
1065 if (nd
->flags
& LOOKUP_RCU
)
1068 audit_inode(nd
->name
, nd
->stack
[0].link
.dentry
, 0);
1069 audit_log_path_denied(AUDIT_ANOM_LINK
, "follow_link");
1074 * safe_hardlink_source - Check for safe hardlink conditions
1075 * @mnt_userns: user namespace of the mount the inode was found from
1076 * @inode: the source inode to hardlink from
1078 * Return false if at least one of the following conditions:
1079 * - inode is not a regular file
1081 * - inode is setgid and group-exec
1082 * - access failure for read and write
1084 * Otherwise returns true.
1086 static bool safe_hardlink_source(struct user_namespace
*mnt_userns
,
1087 struct inode
*inode
)
1089 umode_t mode
= inode
->i_mode
;
1091 /* Special files should not get pinned to the filesystem. */
1095 /* Setuid files should not get pinned to the filesystem. */
1099 /* Executable setgid files should not get pinned to the filesystem. */
1100 if ((mode
& (S_ISGID
| S_IXGRP
)) == (S_ISGID
| S_IXGRP
))
1103 /* Hardlinking to unreadable or unwritable sources is dangerous. */
1104 if (inode_permission(mnt_userns
, inode
, MAY_READ
| MAY_WRITE
))
1111 * may_linkat - Check permissions for creating a hardlink
1112 * @mnt_userns: user namespace of the mount the inode was found from
1113 * @link: the source to hardlink from
1115 * Block hardlink when all of:
1116 * - sysctl_protected_hardlinks enabled
1117 * - fsuid does not match inode
1118 * - hardlink source is unsafe (see safe_hardlink_source() above)
1119 * - not CAP_FOWNER in a namespace with the inode owner uid mapped
1121 * If the inode has been found through an idmapped mount the user namespace of
1122 * the vfsmount must be passed through @mnt_userns. This function will then take
1123 * care to map the inode according to @mnt_userns before checking permissions.
1124 * On non-idmapped mounts or if permission checking is to be performed on the
1125 * raw inode simply passs init_user_ns.
1127 * Returns 0 if successful, -ve on error.
1129 int may_linkat(struct user_namespace
*mnt_userns
, struct path
*link
)
1131 struct inode
*inode
= link
->dentry
->d_inode
;
1133 /* Inode writeback is not safe when the uid or gid are invalid. */
1134 if (!uid_valid(i_uid_into_mnt(mnt_userns
, inode
)) ||
1135 !gid_valid(i_gid_into_mnt(mnt_userns
, inode
)))
1138 if (!sysctl_protected_hardlinks
)
1141 /* Source inode owner (or CAP_FOWNER) can hardlink all they like,
1142 * otherwise, it must be a safe source.
1144 if (safe_hardlink_source(mnt_userns
, inode
) ||
1145 inode_owner_or_capable(mnt_userns
, inode
))
1148 audit_log_path_denied(AUDIT_ANOM_LINK
, "linkat");
1153 * may_create_in_sticky - Check whether an O_CREAT open in a sticky directory
1154 * should be allowed, or not, on files that already
1156 * @mnt_userns: user namespace of the mount the inode was found from
1157 * @nd: nameidata pathwalk data
1158 * @inode: the inode of the file to open
1160 * Block an O_CREAT open of a FIFO (or a regular file) when:
1161 * - sysctl_protected_fifos (or sysctl_protected_regular) is enabled
1162 * - the file already exists
1163 * - we are in a sticky directory
1164 * - we don't own the file
1165 * - the owner of the directory doesn't own the file
1166 * - the directory is world writable
1167 * If the sysctl_protected_fifos (or sysctl_protected_regular) is set to 2
1168 * the directory doesn't have to be world writable: being group writable will
1171 * If the inode has been found through an idmapped mount the user namespace of
1172 * the vfsmount must be passed through @mnt_userns. This function will then take
1173 * care to map the inode according to @mnt_userns before checking permissions.
1174 * On non-idmapped mounts or if permission checking is to be performed on the
1175 * raw inode simply passs init_user_ns.
1177 * Returns 0 if the open is allowed, -ve on error.
1179 static int may_create_in_sticky(struct user_namespace
*mnt_userns
,
1180 struct nameidata
*nd
, struct inode
*const inode
)
1182 umode_t dir_mode
= nd
->dir_mode
;
1183 kuid_t dir_uid
= nd
->dir_uid
;
1185 if ((!sysctl_protected_fifos
&& S_ISFIFO(inode
->i_mode
)) ||
1186 (!sysctl_protected_regular
&& S_ISREG(inode
->i_mode
)) ||
1187 likely(!(dir_mode
& S_ISVTX
)) ||
1188 uid_eq(i_uid_into_mnt(mnt_userns
, inode
), dir_uid
) ||
1189 uid_eq(current_fsuid(), i_uid_into_mnt(mnt_userns
, inode
)))
1192 if (likely(dir_mode
& 0002) ||
1194 ((sysctl_protected_fifos
>= 2 && S_ISFIFO(inode
->i_mode
)) ||
1195 (sysctl_protected_regular
>= 2 && S_ISREG(inode
->i_mode
))))) {
1196 const char *operation
= S_ISFIFO(inode
->i_mode
) ?
1197 "sticky_create_fifo" :
1198 "sticky_create_regular";
1199 audit_log_path_denied(AUDIT_ANOM_CREAT
, operation
);
1206 * follow_up - Find the mountpoint of path's vfsmount
1208 * Given a path, find the mountpoint of its source file system.
1209 * Replace @path with the path of the mountpoint in the parent mount.
1212 * Return 1 if we went up a level and 0 if we were already at the
1215 int follow_up(struct path
*path
)
1217 struct mount
*mnt
= real_mount(path
->mnt
);
1218 struct mount
*parent
;
1219 struct dentry
*mountpoint
;
1221 read_seqlock_excl(&mount_lock
);
1222 parent
= mnt
->mnt_parent
;
1223 if (parent
== mnt
) {
1224 read_sequnlock_excl(&mount_lock
);
1227 mntget(&parent
->mnt
);
1228 mountpoint
= dget(mnt
->mnt_mountpoint
);
1229 read_sequnlock_excl(&mount_lock
);
1231 path
->dentry
= mountpoint
;
1233 path
->mnt
= &parent
->mnt
;
1236 EXPORT_SYMBOL(follow_up
);
1238 static bool choose_mountpoint_rcu(struct mount
*m
, const struct path
*root
,
1239 struct path
*path
, unsigned *seqp
)
1241 while (mnt_has_parent(m
)) {
1242 struct dentry
*mountpoint
= m
->mnt_mountpoint
;
1245 if (unlikely(root
->dentry
== mountpoint
&&
1246 root
->mnt
== &m
->mnt
))
1248 if (mountpoint
!= m
->mnt
.mnt_root
) {
1249 path
->mnt
= &m
->mnt
;
1250 path
->dentry
= mountpoint
;
1251 *seqp
= read_seqcount_begin(&mountpoint
->d_seq
);
1258 static bool choose_mountpoint(struct mount
*m
, const struct path
*root
,
1265 unsigned seq
, mseq
= read_seqbegin(&mount_lock
);
1267 found
= choose_mountpoint_rcu(m
, root
, path
, &seq
);
1268 if (unlikely(!found
)) {
1269 if (!read_seqretry(&mount_lock
, mseq
))
1272 if (likely(__legitimize_path(path
, seq
, mseq
)))
1284 * Perform an automount
1285 * - return -EISDIR to tell follow_managed() to stop and return the path we
1288 static int follow_automount(struct path
*path
, int *count
, unsigned lookup_flags
)
1290 struct dentry
*dentry
= path
->dentry
;
1292 /* We don't want to mount if someone's just doing a stat -
1293 * unless they're stat'ing a directory and appended a '/' to
1296 * We do, however, want to mount if someone wants to open or
1297 * create a file of any type under the mountpoint, wants to
1298 * traverse through the mountpoint or wants to open the
1299 * mounted directory. Also, autofs may mark negative dentries
1300 * as being automount points. These will need the attentions
1301 * of the daemon to instantiate them before they can be used.
1303 if (!(lookup_flags
& (LOOKUP_PARENT
| LOOKUP_DIRECTORY
|
1304 LOOKUP_OPEN
| LOOKUP_CREATE
| LOOKUP_AUTOMOUNT
)) &&
1308 if (count
&& (*count
)++ >= MAXSYMLINKS
)
1311 return finish_automount(dentry
->d_op
->d_automount(path
), path
);
1315 * mount traversal - out-of-line part. One note on ->d_flags accesses -
1316 * dentries are pinned but not locked here, so negative dentry can go
1317 * positive right under us. Use of smp_load_acquire() provides a barrier
1318 * sufficient for ->d_inode and ->d_flags consistency.
1320 static int __traverse_mounts(struct path
*path
, unsigned flags
, bool *jumped
,
1321 int *count
, unsigned lookup_flags
)
1323 struct vfsmount
*mnt
= path
->mnt
;
1324 bool need_mntput
= false;
1327 while (flags
& DCACHE_MANAGED_DENTRY
) {
1328 /* Allow the filesystem to manage the transit without i_mutex
1330 if (flags
& DCACHE_MANAGE_TRANSIT
) {
1331 ret
= path
->dentry
->d_op
->d_manage(path
, false);
1332 flags
= smp_load_acquire(&path
->dentry
->d_flags
);
1337 if (flags
& DCACHE_MOUNTED
) { // something's mounted on it..
1338 struct vfsmount
*mounted
= lookup_mnt(path
);
1339 if (mounted
) { // ... in our namespace
1343 path
->mnt
= mounted
;
1344 path
->dentry
= dget(mounted
->mnt_root
);
1345 // here we know it's positive
1346 flags
= path
->dentry
->d_flags
;
1352 if (!(flags
& DCACHE_NEED_AUTOMOUNT
))
1355 // uncovered automount point
1356 ret
= follow_automount(path
, count
, lookup_flags
);
1357 flags
= smp_load_acquire(&path
->dentry
->d_flags
);
1364 // possible if you race with several mount --move
1365 if (need_mntput
&& path
->mnt
== mnt
)
1367 if (!ret
&& unlikely(d_flags_negative(flags
)))
1369 *jumped
= need_mntput
;
1373 static inline int traverse_mounts(struct path
*path
, bool *jumped
,
1374 int *count
, unsigned lookup_flags
)
1376 unsigned flags
= smp_load_acquire(&path
->dentry
->d_flags
);
1379 if (likely(!(flags
& DCACHE_MANAGED_DENTRY
))) {
1381 if (unlikely(d_flags_negative(flags
)))
1385 return __traverse_mounts(path
, flags
, jumped
, count
, lookup_flags
);
1388 int follow_down_one(struct path
*path
)
1390 struct vfsmount
*mounted
;
1392 mounted
= lookup_mnt(path
);
1396 path
->mnt
= mounted
;
1397 path
->dentry
= dget(mounted
->mnt_root
);
1402 EXPORT_SYMBOL(follow_down_one
);
1405 * Follow down to the covering mount currently visible to userspace. At each
1406 * point, the filesystem owning that dentry may be queried as to whether the
1407 * caller is permitted to proceed or not.
1409 int follow_down(struct path
*path
)
1411 struct vfsmount
*mnt
= path
->mnt
;
1413 int ret
= traverse_mounts(path
, &jumped
, NULL
, 0);
1415 if (path
->mnt
!= mnt
)
1419 EXPORT_SYMBOL(follow_down
);
1422 * Try to skip to top of mountpoint pile in rcuwalk mode. Fail if
1423 * we meet a managed dentry that would need blocking.
1425 static bool __follow_mount_rcu(struct nameidata
*nd
, struct path
*path
,
1426 struct inode
**inode
, unsigned *seqp
)
1428 struct dentry
*dentry
= path
->dentry
;
1429 unsigned int flags
= dentry
->d_flags
;
1431 if (likely(!(flags
& DCACHE_MANAGED_DENTRY
)))
1434 if (unlikely(nd
->flags
& LOOKUP_NO_XDEV
))
1439 * Don't forget we might have a non-mountpoint managed dentry
1440 * that wants to block transit.
1442 if (unlikely(flags
& DCACHE_MANAGE_TRANSIT
)) {
1443 int res
= dentry
->d_op
->d_manage(path
, true);
1445 return res
== -EISDIR
;
1446 flags
= dentry
->d_flags
;
1449 if (flags
& DCACHE_MOUNTED
) {
1450 struct mount
*mounted
= __lookup_mnt(path
->mnt
, dentry
);
1452 path
->mnt
= &mounted
->mnt
;
1453 dentry
= path
->dentry
= mounted
->mnt
.mnt_root
;
1454 nd
->state
|= ND_JUMPED
;
1455 *seqp
= read_seqcount_begin(&dentry
->d_seq
);
1456 *inode
= dentry
->d_inode
;
1458 * We don't need to re-check ->d_seq after this
1459 * ->d_inode read - there will be an RCU delay
1460 * between mount hash removal and ->mnt_root
1461 * becoming unpinned.
1463 flags
= dentry
->d_flags
;
1466 if (read_seqretry(&mount_lock
, nd
->m_seq
))
1469 return !(flags
& DCACHE_NEED_AUTOMOUNT
);
1473 static inline int handle_mounts(struct nameidata
*nd
, struct dentry
*dentry
,
1474 struct path
*path
, struct inode
**inode
,
1480 path
->mnt
= nd
->path
.mnt
;
1481 path
->dentry
= dentry
;
1482 if (nd
->flags
& LOOKUP_RCU
) {
1483 unsigned int seq
= *seqp
;
1484 if (unlikely(!*inode
))
1486 if (likely(__follow_mount_rcu(nd
, path
, inode
, seqp
)))
1488 if (!try_to_unlazy_next(nd
, dentry
, seq
))
1490 // *path might've been clobbered by __follow_mount_rcu()
1491 path
->mnt
= nd
->path
.mnt
;
1492 path
->dentry
= dentry
;
1494 ret
= traverse_mounts(path
, &jumped
, &nd
->total_link_count
, nd
->flags
);
1496 if (unlikely(nd
->flags
& LOOKUP_NO_XDEV
))
1499 nd
->state
|= ND_JUMPED
;
1501 if (unlikely(ret
)) {
1503 if (path
->mnt
!= nd
->path
.mnt
)
1506 *inode
= d_backing_inode(path
->dentry
);
1507 *seqp
= 0; /* out of RCU mode, so the value doesn't matter */
1513 * This looks up the name in dcache and possibly revalidates the found dentry.
1514 * NULL is returned if the dentry does not exist in the cache.
1516 static struct dentry
*lookup_dcache(const struct qstr
*name
,
1520 struct dentry
*dentry
= d_lookup(dir
, name
);
1522 int error
= d_revalidate(dentry
, flags
);
1523 if (unlikely(error
<= 0)) {
1525 d_invalidate(dentry
);
1527 return ERR_PTR(error
);
1534 * Parent directory has inode locked exclusive. This is one
1535 * and only case when ->lookup() gets called on non in-lookup
1536 * dentries - as the matter of fact, this only gets called
1537 * when directory is guaranteed to have no in-lookup children
1540 static struct dentry
*__lookup_hash(const struct qstr
*name
,
1541 struct dentry
*base
, unsigned int flags
)
1543 struct dentry
*dentry
= lookup_dcache(name
, base
, flags
);
1545 struct inode
*dir
= base
->d_inode
;
1550 /* Don't create child dentry for a dead directory. */
1551 if (unlikely(IS_DEADDIR(dir
)))
1552 return ERR_PTR(-ENOENT
);
1554 dentry
= d_alloc(base
, name
);
1555 if (unlikely(!dentry
))
1556 return ERR_PTR(-ENOMEM
);
1558 old
= dir
->i_op
->lookup(dir
, dentry
, flags
);
1559 if (unlikely(old
)) {
1566 static struct dentry
*lookup_fast(struct nameidata
*nd
,
1567 struct inode
**inode
,
1570 struct dentry
*dentry
, *parent
= nd
->path
.dentry
;
1574 * Rename seqlock is not required here because in the off chance
1575 * of a false negative due to a concurrent rename, the caller is
1576 * going to fall back to non-racy lookup.
1578 if (nd
->flags
& LOOKUP_RCU
) {
1580 dentry
= __d_lookup_rcu(parent
, &nd
->last
, &seq
);
1581 if (unlikely(!dentry
)) {
1582 if (!try_to_unlazy(nd
))
1583 return ERR_PTR(-ECHILD
);
1588 * This sequence count validates that the inode matches
1589 * the dentry name information from lookup.
1591 *inode
= d_backing_inode(dentry
);
1592 if (unlikely(read_seqcount_retry(&dentry
->d_seq
, seq
)))
1593 return ERR_PTR(-ECHILD
);
1596 * This sequence count validates that the parent had no
1597 * changes while we did the lookup of the dentry above.
1599 * The memory barrier in read_seqcount_begin of child is
1600 * enough, we can use __read_seqcount_retry here.
1602 if (unlikely(__read_seqcount_retry(&parent
->d_seq
, nd
->seq
)))
1603 return ERR_PTR(-ECHILD
);
1606 status
= d_revalidate(dentry
, nd
->flags
);
1607 if (likely(status
> 0))
1609 if (!try_to_unlazy_next(nd
, dentry
, seq
))
1610 return ERR_PTR(-ECHILD
);
1611 if (status
== -ECHILD
)
1612 /* we'd been told to redo it in non-rcu mode */
1613 status
= d_revalidate(dentry
, nd
->flags
);
1615 dentry
= __d_lookup(parent
, &nd
->last
);
1616 if (unlikely(!dentry
))
1618 status
= d_revalidate(dentry
, nd
->flags
);
1620 if (unlikely(status
<= 0)) {
1622 d_invalidate(dentry
);
1624 return ERR_PTR(status
);
1629 /* Fast lookup failed, do it the slow way */
1630 static struct dentry
*__lookup_slow(const struct qstr
*name
,
1634 struct dentry
*dentry
, *old
;
1635 struct inode
*inode
= dir
->d_inode
;
1636 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq
);
1638 /* Don't go there if it's already dead */
1639 if (unlikely(IS_DEADDIR(inode
)))
1640 return ERR_PTR(-ENOENT
);
1642 dentry
= d_alloc_parallel(dir
, name
, &wq
);
1645 if (unlikely(!d_in_lookup(dentry
))) {
1646 int error
= d_revalidate(dentry
, flags
);
1647 if (unlikely(error
<= 0)) {
1649 d_invalidate(dentry
);
1654 dentry
= ERR_PTR(error
);
1657 old
= inode
->i_op
->lookup(inode
, dentry
, flags
);
1658 d_lookup_done(dentry
);
1659 if (unlikely(old
)) {
1667 static struct dentry
*lookup_slow(const struct qstr
*name
,
1671 struct inode
*inode
= dir
->d_inode
;
1673 inode_lock_shared(inode
);
1674 res
= __lookup_slow(name
, dir
, flags
);
1675 inode_unlock_shared(inode
);
1679 static inline int may_lookup(struct user_namespace
*mnt_userns
,
1680 struct nameidata
*nd
)
1682 if (nd
->flags
& LOOKUP_RCU
) {
1683 int err
= inode_permission(mnt_userns
, nd
->inode
, MAY_EXEC
|MAY_NOT_BLOCK
);
1684 if (err
!= -ECHILD
|| !try_to_unlazy(nd
))
1687 return inode_permission(mnt_userns
, nd
->inode
, MAY_EXEC
);
1690 static int reserve_stack(struct nameidata
*nd
, struct path
*link
, unsigned seq
)
1692 if (unlikely(nd
->total_link_count
++ >= MAXSYMLINKS
))
1695 if (likely(nd
->depth
!= EMBEDDED_LEVELS
))
1697 if (likely(nd
->stack
!= nd
->internal
))
1699 if (likely(nd_alloc_stack(nd
)))
1702 if (nd
->flags
& LOOKUP_RCU
) {
1703 // we need to grab link before we do unlazy. And we can't skip
1704 // unlazy even if we fail to grab the link - cleanup needs it
1705 bool grabbed_link
= legitimize_path(nd
, link
, seq
);
1707 if (!try_to_unlazy(nd
) != 0 || !grabbed_link
)
1710 if (nd_alloc_stack(nd
))
1716 enum {WALK_TRAILING
= 1, WALK_MORE
= 2, WALK_NOFOLLOW
= 4};
1718 static const char *pick_link(struct nameidata
*nd
, struct path
*link
,
1719 struct inode
*inode
, unsigned seq
, int flags
)
1723 int error
= reserve_stack(nd
, link
, seq
);
1725 if (unlikely(error
)) {
1726 if (!(nd
->flags
& LOOKUP_RCU
))
1728 return ERR_PTR(error
);
1730 last
= nd
->stack
+ nd
->depth
++;
1732 clear_delayed_call(&last
->done
);
1735 if (flags
& WALK_TRAILING
) {
1736 error
= may_follow_link(nd
, inode
);
1737 if (unlikely(error
))
1738 return ERR_PTR(error
);
1741 if (unlikely(nd
->flags
& LOOKUP_NO_SYMLINKS
) ||
1742 unlikely(link
->mnt
->mnt_flags
& MNT_NOSYMFOLLOW
))
1743 return ERR_PTR(-ELOOP
);
1745 if (!(nd
->flags
& LOOKUP_RCU
)) {
1746 touch_atime(&last
->link
);
1748 } else if (atime_needs_update(&last
->link
, inode
)) {
1749 if (!try_to_unlazy(nd
))
1750 return ERR_PTR(-ECHILD
);
1751 touch_atime(&last
->link
);
1754 error
= security_inode_follow_link(link
->dentry
, inode
,
1755 nd
->flags
& LOOKUP_RCU
);
1756 if (unlikely(error
))
1757 return ERR_PTR(error
);
1759 res
= READ_ONCE(inode
->i_link
);
1761 const char * (*get
)(struct dentry
*, struct inode
*,
1762 struct delayed_call
*);
1763 get
= inode
->i_op
->get_link
;
1764 if (nd
->flags
& LOOKUP_RCU
) {
1765 res
= get(NULL
, inode
, &last
->done
);
1766 if (res
== ERR_PTR(-ECHILD
) && try_to_unlazy(nd
))
1767 res
= get(link
->dentry
, inode
, &last
->done
);
1769 res
= get(link
->dentry
, inode
, &last
->done
);
1777 error
= nd_jump_root(nd
);
1778 if (unlikely(error
))
1779 return ERR_PTR(error
);
1780 while (unlikely(*++res
== '/'))
1785 all_done
: // pure jump
1791 * Do we need to follow links? We _really_ want to be able
1792 * to do this check without having to look at inode->i_op,
1793 * so we keep a cache of "no, this doesn't need follow_link"
1794 * for the common case.
1796 static const char *step_into(struct nameidata
*nd
, int flags
,
1797 struct dentry
*dentry
, struct inode
*inode
, unsigned seq
)
1800 int err
= handle_mounts(nd
, dentry
, &path
, &inode
, &seq
);
1803 return ERR_PTR(err
);
1804 if (likely(!d_is_symlink(path
.dentry
)) ||
1805 ((flags
& WALK_TRAILING
) && !(nd
->flags
& LOOKUP_FOLLOW
)) ||
1806 (flags
& WALK_NOFOLLOW
)) {
1807 /* not a symlink or should not follow */
1808 if (!(nd
->flags
& LOOKUP_RCU
)) {
1809 dput(nd
->path
.dentry
);
1810 if (nd
->path
.mnt
!= path
.mnt
)
1811 mntput(nd
->path
.mnt
);
1818 if (nd
->flags
& LOOKUP_RCU
) {
1819 /* make sure that d_is_symlink above matches inode */
1820 if (read_seqcount_retry(&path
.dentry
->d_seq
, seq
))
1821 return ERR_PTR(-ECHILD
);
1823 if (path
.mnt
== nd
->path
.mnt
)
1826 return pick_link(nd
, &path
, inode
, seq
, flags
);
1829 static struct dentry
*follow_dotdot_rcu(struct nameidata
*nd
,
1830 struct inode
**inodep
,
1833 struct dentry
*parent
, *old
;
1835 if (path_equal(&nd
->path
, &nd
->root
))
1837 if (unlikely(nd
->path
.dentry
== nd
->path
.mnt
->mnt_root
)) {
1840 if (!choose_mountpoint_rcu(real_mount(nd
->path
.mnt
),
1841 &nd
->root
, &path
, &seq
))
1843 if (unlikely(nd
->flags
& LOOKUP_NO_XDEV
))
1844 return ERR_PTR(-ECHILD
);
1846 nd
->inode
= path
.dentry
->d_inode
;
1848 if (unlikely(read_seqretry(&mount_lock
, nd
->m_seq
)))
1849 return ERR_PTR(-ECHILD
);
1850 /* we know that mountpoint was pinned */
1852 old
= nd
->path
.dentry
;
1853 parent
= old
->d_parent
;
1854 *inodep
= parent
->d_inode
;
1855 *seqp
= read_seqcount_begin(&parent
->d_seq
);
1856 if (unlikely(read_seqcount_retry(&old
->d_seq
, nd
->seq
)))
1857 return ERR_PTR(-ECHILD
);
1858 if (unlikely(!path_connected(nd
->path
.mnt
, parent
)))
1859 return ERR_PTR(-ECHILD
);
1862 if (unlikely(read_seqretry(&mount_lock
, nd
->m_seq
)))
1863 return ERR_PTR(-ECHILD
);
1864 if (unlikely(nd
->flags
& LOOKUP_BENEATH
))
1865 return ERR_PTR(-ECHILD
);
1869 static struct dentry
*follow_dotdot(struct nameidata
*nd
,
1870 struct inode
**inodep
,
1873 struct dentry
*parent
;
1875 if (path_equal(&nd
->path
, &nd
->root
))
1877 if (unlikely(nd
->path
.dentry
== nd
->path
.mnt
->mnt_root
)) {
1880 if (!choose_mountpoint(real_mount(nd
->path
.mnt
),
1883 path_put(&nd
->path
);
1885 nd
->inode
= path
.dentry
->d_inode
;
1886 if (unlikely(nd
->flags
& LOOKUP_NO_XDEV
))
1887 return ERR_PTR(-EXDEV
);
1889 /* rare case of legitimate dget_parent()... */
1890 parent
= dget_parent(nd
->path
.dentry
);
1891 if (unlikely(!path_connected(nd
->path
.mnt
, parent
))) {
1893 return ERR_PTR(-ENOENT
);
1896 *inodep
= parent
->d_inode
;
1900 if (unlikely(nd
->flags
& LOOKUP_BENEATH
))
1901 return ERR_PTR(-EXDEV
);
1902 dget(nd
->path
.dentry
);
1906 static const char *handle_dots(struct nameidata
*nd
, int type
)
1908 if (type
== LAST_DOTDOT
) {
1909 const char *error
= NULL
;
1910 struct dentry
*parent
;
1911 struct inode
*inode
;
1914 if (!nd
->root
.mnt
) {
1915 error
= ERR_PTR(set_root(nd
));
1919 if (nd
->flags
& LOOKUP_RCU
)
1920 parent
= follow_dotdot_rcu(nd
, &inode
, &seq
);
1922 parent
= follow_dotdot(nd
, &inode
, &seq
);
1924 return ERR_CAST(parent
);
1925 if (unlikely(!parent
))
1926 error
= step_into(nd
, WALK_NOFOLLOW
,
1927 nd
->path
.dentry
, nd
->inode
, nd
->seq
);
1929 error
= step_into(nd
, WALK_NOFOLLOW
,
1930 parent
, inode
, seq
);
1931 if (unlikely(error
))
1934 if (unlikely(nd
->flags
& LOOKUP_IS_SCOPED
)) {
1936 * If there was a racing rename or mount along our
1937 * path, then we can't be sure that ".." hasn't jumped
1938 * above nd->root (and so userspace should retry or use
1942 if (unlikely(__read_seqcount_retry(&mount_lock
.seqcount
, nd
->m_seq
)))
1943 return ERR_PTR(-EAGAIN
);
1944 if (unlikely(__read_seqcount_retry(&rename_lock
.seqcount
, nd
->r_seq
)))
1945 return ERR_PTR(-EAGAIN
);
1951 static const char *walk_component(struct nameidata
*nd
, int flags
)
1953 struct dentry
*dentry
;
1954 struct inode
*inode
;
1957 * "." and ".." are special - ".." especially so because it has
1958 * to be able to know about the current root directory and
1959 * parent relationships.
1961 if (unlikely(nd
->last_type
!= LAST_NORM
)) {
1962 if (!(flags
& WALK_MORE
) && nd
->depth
)
1964 return handle_dots(nd
, nd
->last_type
);
1966 dentry
= lookup_fast(nd
, &inode
, &seq
);
1968 return ERR_CAST(dentry
);
1969 if (unlikely(!dentry
)) {
1970 dentry
= lookup_slow(&nd
->last
, nd
->path
.dentry
, nd
->flags
);
1972 return ERR_CAST(dentry
);
1974 if (!(flags
& WALK_MORE
) && nd
->depth
)
1976 return step_into(nd
, flags
, dentry
, inode
, seq
);
1980 * We can do the critical dentry name comparison and hashing
1981 * operations one word at a time, but we are limited to:
1983 * - Architectures with fast unaligned word accesses. We could
1984 * do a "get_unaligned()" if this helps and is sufficiently
1987 * - non-CONFIG_DEBUG_PAGEALLOC configurations (so that we
1988 * do not trap on the (extremely unlikely) case of a page
1989 * crossing operation.
1991 * - Furthermore, we need an efficient 64-bit compile for the
1992 * 64-bit case in order to generate the "number of bytes in
1993 * the final mask". Again, that could be replaced with a
1994 * efficient population count instruction or similar.
1996 #ifdef CONFIG_DCACHE_WORD_ACCESS
1998 #include <asm/word-at-a-time.h>
2002 /* Architecture provides HASH_MIX and fold_hash() in <asm/hash.h> */
2004 #elif defined(CONFIG_64BIT)
2006 * Register pressure in the mixing function is an issue, particularly
2007 * on 32-bit x86, but almost any function requires one state value and
2008 * one temporary. Instead, use a function designed for two state values
2009 * and no temporaries.
2011 * This function cannot create a collision in only two iterations, so
2012 * we have two iterations to achieve avalanche. In those two iterations,
2013 * we have six layers of mixing, which is enough to spread one bit's
2014 * influence out to 2^6 = 64 state bits.
2016 * Rotate constants are scored by considering either 64 one-bit input
2017 * deltas or 64*63/2 = 2016 two-bit input deltas, and finding the
2018 * probability of that delta causing a change to each of the 128 output
2019 * bits, using a sample of random initial states.
2021 * The Shannon entropy of the computed probabilities is then summed
2022 * to produce a score. Ideally, any input change has a 50% chance of
2023 * toggling any given output bit.
2025 * Mixing scores (in bits) for (12,45):
2026 * Input delta: 1-bit 2-bit
2027 * 1 round: 713.3 42542.6
2028 * 2 rounds: 2753.7 140389.8
2029 * 3 rounds: 5954.1 233458.2
2030 * 4 rounds: 7862.6 256672.2
2031 * Perfect: 8192 258048
2032 * (64*128) (64*63/2 * 128)
2034 #define HASH_MIX(x, y, a) \
2036 y ^= x, x = rol64(x,12),\
2037 x += y, y = rol64(y,45),\
2041 * Fold two longs into one 32-bit hash value. This must be fast, but
2042 * latency isn't quite as critical, as there is a fair bit of additional
2043 * work done before the hash value is used.
2045 static inline unsigned int fold_hash(unsigned long x
, unsigned long y
)
2047 y
^= x
* GOLDEN_RATIO_64
;
2048 y
*= GOLDEN_RATIO_64
;
2052 #else /* 32-bit case */
2055 * Mixing scores (in bits) for (7,20):
2056 * Input delta: 1-bit 2-bit
2057 * 1 round: 330.3 9201.6
2058 * 2 rounds: 1246.4 25475.4
2059 * 3 rounds: 1907.1 31295.1
2060 * 4 rounds: 2042.3 31718.6
2061 * Perfect: 2048 31744
2062 * (32*64) (32*31/2 * 64)
2064 #define HASH_MIX(x, y, a) \
2066 y ^= x, x = rol32(x, 7),\
2067 x += y, y = rol32(y,20),\
2070 static inline unsigned int fold_hash(unsigned long x
, unsigned long y
)
2072 /* Use arch-optimized multiply if one exists */
2073 return __hash_32(y
^ __hash_32(x
));
2079 * Return the hash of a string of known length. This is carfully
2080 * designed to match hash_name(), which is the more critical function.
2081 * In particular, we must end by hashing a final word containing 0..7
2082 * payload bytes, to match the way that hash_name() iterates until it
2083 * finds the delimiter after the name.
2085 unsigned int full_name_hash(const void *salt
, const char *name
, unsigned int len
)
2087 unsigned long a
, x
= 0, y
= (unsigned long)salt
;
2092 a
= load_unaligned_zeropad(name
);
2093 if (len
< sizeof(unsigned long))
2096 name
+= sizeof(unsigned long);
2097 len
-= sizeof(unsigned long);
2099 x
^= a
& bytemask_from_count(len
);
2101 return fold_hash(x
, y
);
2103 EXPORT_SYMBOL(full_name_hash
);
2105 /* Return the "hash_len" (hash and length) of a null-terminated string */
2106 u64
hashlen_string(const void *salt
, const char *name
)
2108 unsigned long a
= 0, x
= 0, y
= (unsigned long)salt
;
2109 unsigned long adata
, mask
, len
;
2110 const struct word_at_a_time constants
= WORD_AT_A_TIME_CONSTANTS
;
2117 len
+= sizeof(unsigned long);
2119 a
= load_unaligned_zeropad(name
+len
);
2120 } while (!has_zero(a
, &adata
, &constants
));
2122 adata
= prep_zero_mask(a
, adata
, &constants
);
2123 mask
= create_zero_mask(adata
);
2124 x
^= a
& zero_bytemask(mask
);
2126 return hashlen_create(fold_hash(x
, y
), len
+ find_zero(mask
));
2128 EXPORT_SYMBOL(hashlen_string
);
2131 * Calculate the length and hash of the path component, and
2132 * return the "hash_len" as the result.
2134 static inline u64
hash_name(const void *salt
, const char *name
)
2136 unsigned long a
= 0, b
, x
= 0, y
= (unsigned long)salt
;
2137 unsigned long adata
, bdata
, mask
, len
;
2138 const struct word_at_a_time constants
= WORD_AT_A_TIME_CONSTANTS
;
2145 len
+= sizeof(unsigned long);
2147 a
= load_unaligned_zeropad(name
+len
);
2148 b
= a
^ REPEAT_BYTE('/');
2149 } while (!(has_zero(a
, &adata
, &constants
) | has_zero(b
, &bdata
, &constants
)));
2151 adata
= prep_zero_mask(a
, adata
, &constants
);
2152 bdata
= prep_zero_mask(b
, bdata
, &constants
);
2153 mask
= create_zero_mask(adata
| bdata
);
2154 x
^= a
& zero_bytemask(mask
);
2156 return hashlen_create(fold_hash(x
, y
), len
+ find_zero(mask
));
2159 #else /* !CONFIG_DCACHE_WORD_ACCESS: Slow, byte-at-a-time version */
2161 /* Return the hash of a string of known length */
2162 unsigned int full_name_hash(const void *salt
, const char *name
, unsigned int len
)
2164 unsigned long hash
= init_name_hash(salt
);
2166 hash
= partial_name_hash((unsigned char)*name
++, hash
);
2167 return end_name_hash(hash
);
2169 EXPORT_SYMBOL(full_name_hash
);
2171 /* Return the "hash_len" (hash and length) of a null-terminated string */
2172 u64
hashlen_string(const void *salt
, const char *name
)
2174 unsigned long hash
= init_name_hash(salt
);
2175 unsigned long len
= 0, c
;
2177 c
= (unsigned char)*name
;
2180 hash
= partial_name_hash(c
, hash
);
2181 c
= (unsigned char)name
[len
];
2183 return hashlen_create(end_name_hash(hash
), len
);
2185 EXPORT_SYMBOL(hashlen_string
);
2188 * We know there's a real path component here of at least
2191 static inline u64
hash_name(const void *salt
, const char *name
)
2193 unsigned long hash
= init_name_hash(salt
);
2194 unsigned long len
= 0, c
;
2196 c
= (unsigned char)*name
;
2199 hash
= partial_name_hash(c
, hash
);
2200 c
= (unsigned char)name
[len
];
2201 } while (c
&& c
!= '/');
2202 return hashlen_create(end_name_hash(hash
), len
);
2209 * This is the basic name resolution function, turning a pathname into
2210 * the final dentry. We expect 'base' to be positive and a directory.
2212 * Returns 0 and nd will have valid dentry and mnt on success.
2213 * Returns error and drops reference to input namei data on failure.
2215 static int link_path_walk(const char *name
, struct nameidata
*nd
)
2217 int depth
= 0; // depth <= nd->depth
2220 nd
->last_type
= LAST_ROOT
;
2221 nd
->flags
|= LOOKUP_PARENT
;
2223 return PTR_ERR(name
);
2227 nd
->dir_mode
= 0; // short-circuit the 'hardening' idiocy
2231 /* At this point we know we have a real path component. */
2233 struct user_namespace
*mnt_userns
;
2238 mnt_userns
= mnt_user_ns(nd
->path
.mnt
);
2239 err
= may_lookup(mnt_userns
, nd
);
2243 hash_len
= hash_name(nd
->path
.dentry
, name
);
2246 if (name
[0] == '.') switch (hashlen_len(hash_len
)) {
2248 if (name
[1] == '.') {
2250 nd
->state
|= ND_JUMPED
;
2256 if (likely(type
== LAST_NORM
)) {
2257 struct dentry
*parent
= nd
->path
.dentry
;
2258 nd
->state
&= ~ND_JUMPED
;
2259 if (unlikely(parent
->d_flags
& DCACHE_OP_HASH
)) {
2260 struct qstr
this = { { .hash_len
= hash_len
}, .name
= name
};
2261 err
= parent
->d_op
->d_hash(parent
, &this);
2264 hash_len
= this.hash_len
;
2269 nd
->last
.hash_len
= hash_len
;
2270 nd
->last
.name
= name
;
2271 nd
->last_type
= type
;
2273 name
+= hashlen_len(hash_len
);
2277 * If it wasn't NUL, we know it was '/'. Skip that
2278 * slash, and continue until no more slashes.
2282 } while (unlikely(*name
== '/'));
2283 if (unlikely(!*name
)) {
2285 /* pathname or trailing symlink, done */
2287 nd
->dir_uid
= i_uid_into_mnt(mnt_userns
, nd
->inode
);
2288 nd
->dir_mode
= nd
->inode
->i_mode
;
2289 nd
->flags
&= ~LOOKUP_PARENT
;
2292 /* last component of nested symlink */
2293 name
= nd
->stack
[--depth
].name
;
2294 link
= walk_component(nd
, 0);
2296 /* not the last component */
2297 link
= walk_component(nd
, WALK_MORE
);
2299 if (unlikely(link
)) {
2301 return PTR_ERR(link
);
2302 /* a symlink to follow */
2303 nd
->stack
[depth
++].name
= name
;
2307 if (unlikely(!d_can_lookup(nd
->path
.dentry
))) {
2308 if (nd
->flags
& LOOKUP_RCU
) {
2309 if (!try_to_unlazy(nd
))
2317 /* must be paired with terminate_walk() */
2318 static const char *path_init(struct nameidata
*nd
, unsigned flags
)
2321 const char *s
= nd
->name
->name
;
2323 /* LOOKUP_CACHED requires RCU, ask caller to retry */
2324 if ((flags
& (LOOKUP_RCU
| LOOKUP_CACHED
)) == LOOKUP_CACHED
)
2325 return ERR_PTR(-EAGAIN
);
2328 flags
&= ~LOOKUP_RCU
;
2329 if (flags
& LOOKUP_RCU
)
2333 nd
->state
|= ND_JUMPED
;
2335 nd
->m_seq
= __read_seqcount_begin(&mount_lock
.seqcount
);
2336 nd
->r_seq
= __read_seqcount_begin(&rename_lock
.seqcount
);
2339 if (nd
->state
& ND_ROOT_PRESET
) {
2340 struct dentry
*root
= nd
->root
.dentry
;
2341 struct inode
*inode
= root
->d_inode
;
2342 if (*s
&& unlikely(!d_can_lookup(root
)))
2343 return ERR_PTR(-ENOTDIR
);
2344 nd
->path
= nd
->root
;
2346 if (flags
& LOOKUP_RCU
) {
2347 nd
->seq
= read_seqcount_begin(&nd
->path
.dentry
->d_seq
);
2348 nd
->root_seq
= nd
->seq
;
2350 path_get(&nd
->path
);
2355 nd
->root
.mnt
= NULL
;
2357 /* Absolute pathname -- fetch the root (LOOKUP_IN_ROOT uses nd->dfd). */
2358 if (*s
== '/' && !(flags
& LOOKUP_IN_ROOT
)) {
2359 error
= nd_jump_root(nd
);
2360 if (unlikely(error
))
2361 return ERR_PTR(error
);
2365 /* Relative pathname -- get the starting-point it is relative to. */
2366 if (nd
->dfd
== AT_FDCWD
) {
2367 if (flags
& LOOKUP_RCU
) {
2368 struct fs_struct
*fs
= current
->fs
;
2372 seq
= read_seqcount_begin(&fs
->seq
);
2374 nd
->inode
= nd
->path
.dentry
->d_inode
;
2375 nd
->seq
= __read_seqcount_begin(&nd
->path
.dentry
->d_seq
);
2376 } while (read_seqcount_retry(&fs
->seq
, seq
));
2378 get_fs_pwd(current
->fs
, &nd
->path
);
2379 nd
->inode
= nd
->path
.dentry
->d_inode
;
2382 /* Caller must check execute permissions on the starting path component */
2383 struct fd f
= fdget_raw(nd
->dfd
);
2384 struct dentry
*dentry
;
2387 return ERR_PTR(-EBADF
);
2389 dentry
= f
.file
->f_path
.dentry
;
2391 if (*s
&& unlikely(!d_can_lookup(dentry
))) {
2393 return ERR_PTR(-ENOTDIR
);
2396 nd
->path
= f
.file
->f_path
;
2397 if (flags
& LOOKUP_RCU
) {
2398 nd
->inode
= nd
->path
.dentry
->d_inode
;
2399 nd
->seq
= read_seqcount_begin(&nd
->path
.dentry
->d_seq
);
2401 path_get(&nd
->path
);
2402 nd
->inode
= nd
->path
.dentry
->d_inode
;
2407 /* For scoped-lookups we need to set the root to the dirfd as well. */
2408 if (flags
& LOOKUP_IS_SCOPED
) {
2409 nd
->root
= nd
->path
;
2410 if (flags
& LOOKUP_RCU
) {
2411 nd
->root_seq
= nd
->seq
;
2413 path_get(&nd
->root
);
2414 nd
->state
|= ND_ROOT_GRABBED
;
2420 static inline const char *lookup_last(struct nameidata
*nd
)
2422 if (nd
->last_type
== LAST_NORM
&& nd
->last
.name
[nd
->last
.len
])
2423 nd
->flags
|= LOOKUP_FOLLOW
| LOOKUP_DIRECTORY
;
2425 return walk_component(nd
, WALK_TRAILING
);
2428 static int handle_lookup_down(struct nameidata
*nd
)
2430 if (!(nd
->flags
& LOOKUP_RCU
))
2431 dget(nd
->path
.dentry
);
2432 return PTR_ERR(step_into(nd
, WALK_NOFOLLOW
,
2433 nd
->path
.dentry
, nd
->inode
, nd
->seq
));
2436 /* Returns 0 and nd will be valid on success; Retuns error, otherwise. */
2437 static int path_lookupat(struct nameidata
*nd
, unsigned flags
, struct path
*path
)
2439 const char *s
= path_init(nd
, flags
);
2442 if (unlikely(flags
& LOOKUP_DOWN
) && !IS_ERR(s
)) {
2443 err
= handle_lookup_down(nd
);
2444 if (unlikely(err
< 0))
2448 while (!(err
= link_path_walk(s
, nd
)) &&
2449 (s
= lookup_last(nd
)) != NULL
)
2451 if (!err
&& unlikely(nd
->flags
& LOOKUP_MOUNTPOINT
)) {
2452 err
= handle_lookup_down(nd
);
2453 nd
->state
&= ~ND_JUMPED
; // no d_weak_revalidate(), please...
2456 err
= complete_walk(nd
);
2458 if (!err
&& nd
->flags
& LOOKUP_DIRECTORY
)
2459 if (!d_can_lookup(nd
->path
.dentry
))
2463 nd
->path
.mnt
= NULL
;
2464 nd
->path
.dentry
= NULL
;
2470 int filename_lookup(int dfd
, struct filename
*name
, unsigned flags
,
2471 struct path
*path
, struct path
*root
)
2474 struct nameidata nd
;
2476 return PTR_ERR(name
);
2477 set_nameidata(&nd
, dfd
, name
, root
);
2478 retval
= path_lookupat(&nd
, flags
| LOOKUP_RCU
, path
);
2479 if (unlikely(retval
== -ECHILD
))
2480 retval
= path_lookupat(&nd
, flags
, path
);
2481 if (unlikely(retval
== -ESTALE
))
2482 retval
= path_lookupat(&nd
, flags
| LOOKUP_REVAL
, path
);
2484 if (likely(!retval
))
2485 audit_inode(name
, path
->dentry
,
2486 flags
& LOOKUP_MOUNTPOINT
? AUDIT_INODE_NOEVAL
: 0);
2487 restore_nameidata();
2491 /* Returns 0 and nd will be valid on success; Retuns error, otherwise. */
2492 static int path_parentat(struct nameidata
*nd
, unsigned flags
,
2493 struct path
*parent
)
2495 const char *s
= path_init(nd
, flags
);
2496 int err
= link_path_walk(s
, nd
);
2498 err
= complete_walk(nd
);
2501 nd
->path
.mnt
= NULL
;
2502 nd
->path
.dentry
= NULL
;
2508 /* Note: this does not consume "name" */
2509 static int filename_parentat(int dfd
, struct filename
*name
,
2510 unsigned int flags
, struct path
*parent
,
2511 struct qstr
*last
, int *type
)
2514 struct nameidata nd
;
2517 return PTR_ERR(name
);
2518 set_nameidata(&nd
, dfd
, name
, NULL
);
2519 retval
= path_parentat(&nd
, flags
| LOOKUP_RCU
, parent
);
2520 if (unlikely(retval
== -ECHILD
))
2521 retval
= path_parentat(&nd
, flags
, parent
);
2522 if (unlikely(retval
== -ESTALE
))
2523 retval
= path_parentat(&nd
, flags
| LOOKUP_REVAL
, parent
);
2524 if (likely(!retval
)) {
2526 *type
= nd
.last_type
;
2527 audit_inode(name
, parent
->dentry
, AUDIT_INODE_PARENT
);
2529 restore_nameidata();
2533 /* does lookup, returns the object with parent locked */
2534 static struct dentry
*__kern_path_locked(struct filename
*name
, struct path
*path
)
2540 error
= filename_parentat(AT_FDCWD
, name
, 0, path
, &last
, &type
);
2542 return ERR_PTR(error
);
2543 if (unlikely(type
!= LAST_NORM
)) {
2545 return ERR_PTR(-EINVAL
);
2547 inode_lock_nested(path
->dentry
->d_inode
, I_MUTEX_PARENT
);
2548 d
= __lookup_hash(&last
, path
->dentry
, 0);
2550 inode_unlock(path
->dentry
->d_inode
);
2556 struct dentry
*kern_path_locked(const char *name
, struct path
*path
)
2558 struct filename
*filename
= getname_kernel(name
);
2559 struct dentry
*res
= __kern_path_locked(filename
, path
);
2565 int kern_path(const char *name
, unsigned int flags
, struct path
*path
)
2567 struct filename
*filename
= getname_kernel(name
);
2568 int ret
= filename_lookup(AT_FDCWD
, filename
, flags
, path
, NULL
);
2574 EXPORT_SYMBOL(kern_path
);
2577 * vfs_path_lookup - lookup a file path relative to a dentry-vfsmount pair
2578 * @dentry: pointer to dentry of the base directory
2579 * @mnt: pointer to vfs mount of the base directory
2580 * @name: pointer to file name
2581 * @flags: lookup flags
2582 * @path: pointer to struct path to fill
2584 int vfs_path_lookup(struct dentry
*dentry
, struct vfsmount
*mnt
,
2585 const char *name
, unsigned int flags
,
2588 struct filename
*filename
;
2589 struct path root
= {.mnt
= mnt
, .dentry
= dentry
};
2592 filename
= getname_kernel(name
);
2593 /* the first argument of filename_lookup() is ignored with root */
2594 ret
= filename_lookup(AT_FDCWD
, filename
, flags
, path
, &root
);
2598 EXPORT_SYMBOL(vfs_path_lookup
);
2600 static int lookup_one_common(struct user_namespace
*mnt_userns
,
2601 const char *name
, struct dentry
*base
, int len
,
2606 this->hash
= full_name_hash(base
, name
, len
);
2610 if (unlikely(name
[0] == '.')) {
2611 if (len
< 2 || (len
== 2 && name
[1] == '.'))
2616 unsigned int c
= *(const unsigned char *)name
++;
2617 if (c
== '/' || c
== '\0')
2621 * See if the low-level filesystem might want
2622 * to use its own hash..
2624 if (base
->d_flags
& DCACHE_OP_HASH
) {
2625 int err
= base
->d_op
->d_hash(base
, this);
2630 return inode_permission(mnt_userns
, base
->d_inode
, MAY_EXEC
);
2634 * try_lookup_one_len - filesystem helper to lookup single pathname component
2635 * @name: pathname component to lookup
2636 * @base: base directory to lookup from
2637 * @len: maximum length @len should be interpreted to
2639 * Look up a dentry by name in the dcache, returning NULL if it does not
2640 * currently exist. The function does not try to create a dentry.
2642 * Note that this routine is purely a helper for filesystem usage and should
2643 * not be called by generic code.
2645 * The caller must hold base->i_mutex.
2647 struct dentry
*try_lookup_one_len(const char *name
, struct dentry
*base
, int len
)
2652 WARN_ON_ONCE(!inode_is_locked(base
->d_inode
));
2654 err
= lookup_one_common(&init_user_ns
, name
, base
, len
, &this);
2656 return ERR_PTR(err
);
2658 return lookup_dcache(&this, base
, 0);
2660 EXPORT_SYMBOL(try_lookup_one_len
);
2663 * lookup_one_len - filesystem helper to lookup single pathname component
2664 * @name: pathname component to lookup
2665 * @base: base directory to lookup from
2666 * @len: maximum length @len should be interpreted to
2668 * Note that this routine is purely a helper for filesystem usage and should
2669 * not be called by generic code.
2671 * The caller must hold base->i_mutex.
2673 struct dentry
*lookup_one_len(const char *name
, struct dentry
*base
, int len
)
2675 struct dentry
*dentry
;
2679 WARN_ON_ONCE(!inode_is_locked(base
->d_inode
));
2681 err
= lookup_one_common(&init_user_ns
, name
, base
, len
, &this);
2683 return ERR_PTR(err
);
2685 dentry
= lookup_dcache(&this, base
, 0);
2686 return dentry
? dentry
: __lookup_slow(&this, base
, 0);
2688 EXPORT_SYMBOL(lookup_one_len
);
2691 * lookup_one - filesystem helper to lookup single pathname component
2692 * @mnt_userns: user namespace of the mount the lookup is performed from
2693 * @name: pathname component to lookup
2694 * @base: base directory to lookup from
2695 * @len: maximum length @len should be interpreted to
2697 * Note that this routine is purely a helper for filesystem usage and should
2698 * not be called by generic code.
2700 * The caller must hold base->i_mutex.
2702 struct dentry
*lookup_one(struct user_namespace
*mnt_userns
, const char *name
,
2703 struct dentry
*base
, int len
)
2705 struct dentry
*dentry
;
2709 WARN_ON_ONCE(!inode_is_locked(base
->d_inode
));
2711 err
= lookup_one_common(mnt_userns
, name
, base
, len
, &this);
2713 return ERR_PTR(err
);
2715 dentry
= lookup_dcache(&this, base
, 0);
2716 return dentry
? dentry
: __lookup_slow(&this, base
, 0);
2718 EXPORT_SYMBOL(lookup_one
);
2721 * lookup_one_len_unlocked - filesystem helper to lookup single pathname component
2722 * @name: pathname component to lookup
2723 * @base: base directory to lookup from
2724 * @len: maximum length @len should be interpreted to
2726 * Note that this routine is purely a helper for filesystem usage and should
2727 * not be called by generic code.
2729 * Unlike lookup_one_len, it should be called without the parent
2730 * i_mutex held, and will take the i_mutex itself if necessary.
2732 struct dentry
*lookup_one_len_unlocked(const char *name
,
2733 struct dentry
*base
, int len
)
2739 err
= lookup_one_common(&init_user_ns
, name
, base
, len
, &this);
2741 return ERR_PTR(err
);
2743 ret
= lookup_dcache(&this, base
, 0);
2745 ret
= lookup_slow(&this, base
, 0);
2748 EXPORT_SYMBOL(lookup_one_len_unlocked
);
2751 * Like lookup_one_len_unlocked(), except that it yields ERR_PTR(-ENOENT)
2752 * on negatives. Returns known positive or ERR_PTR(); that's what
2753 * most of the users want. Note that pinned negative with unlocked parent
2754 * _can_ become positive at any time, so callers of lookup_one_len_unlocked()
2755 * need to be very careful; pinned positives have ->d_inode stable, so
2756 * this one avoids such problems.
2758 struct dentry
*lookup_positive_unlocked(const char *name
,
2759 struct dentry
*base
, int len
)
2761 struct dentry
*ret
= lookup_one_len_unlocked(name
, base
, len
);
2762 if (!IS_ERR(ret
) && d_flags_negative(smp_load_acquire(&ret
->d_flags
))) {
2764 ret
= ERR_PTR(-ENOENT
);
2768 EXPORT_SYMBOL(lookup_positive_unlocked
);
2770 #ifdef CONFIG_UNIX98_PTYS
2771 int path_pts(struct path
*path
)
2773 /* Find something mounted on "pts" in the same directory as
2776 struct dentry
*parent
= dget_parent(path
->dentry
);
2777 struct dentry
*child
;
2778 struct qstr
this = QSTR_INIT("pts", 3);
2780 if (unlikely(!path_connected(path
->mnt
, parent
))) {
2785 path
->dentry
= parent
;
2786 child
= d_hash_and_lookup(parent
, &this);
2790 path
->dentry
= child
;
2797 int user_path_at_empty(int dfd
, const char __user
*name
, unsigned flags
,
2798 struct path
*path
, int *empty
)
2800 struct filename
*filename
= getname_flags(name
, flags
, empty
);
2801 int ret
= filename_lookup(dfd
, filename
, flags
, path
, NULL
);
2806 EXPORT_SYMBOL(user_path_at_empty
);
2808 int __check_sticky(struct user_namespace
*mnt_userns
, struct inode
*dir
,
2809 struct inode
*inode
)
2811 kuid_t fsuid
= current_fsuid();
2813 if (uid_eq(i_uid_into_mnt(mnt_userns
, inode
), fsuid
))
2815 if (uid_eq(i_uid_into_mnt(mnt_userns
, dir
), fsuid
))
2817 return !capable_wrt_inode_uidgid(mnt_userns
, inode
, CAP_FOWNER
);
2819 EXPORT_SYMBOL(__check_sticky
);
2822 * Check whether we can remove a link victim from directory dir, check
2823 * whether the type of victim is right.
2824 * 1. We can't do it if dir is read-only (done in permission())
2825 * 2. We should have write and exec permissions on dir
2826 * 3. We can't remove anything from append-only dir
2827 * 4. We can't do anything with immutable dir (done in permission())
2828 * 5. If the sticky bit on dir is set we should either
2829 * a. be owner of dir, or
2830 * b. be owner of victim, or
2831 * c. have CAP_FOWNER capability
2832 * 6. If the victim is append-only or immutable we can't do antyhing with
2833 * links pointing to it.
2834 * 7. If the victim has an unknown uid or gid we can't change the inode.
2835 * 8. If we were asked to remove a directory and victim isn't one - ENOTDIR.
2836 * 9. If we were asked to remove a non-directory and victim isn't one - EISDIR.
2837 * 10. We can't remove a root or mountpoint.
2838 * 11. We don't allow removal of NFS sillyrenamed files; it's handled by
2839 * nfs_async_unlink().
2841 static int may_delete(struct user_namespace
*mnt_userns
, struct inode
*dir
,
2842 struct dentry
*victim
, bool isdir
)
2844 struct inode
*inode
= d_backing_inode(victim
);
2847 if (d_is_negative(victim
))
2851 BUG_ON(victim
->d_parent
->d_inode
!= dir
);
2853 /* Inode writeback is not safe when the uid or gid are invalid. */
2854 if (!uid_valid(i_uid_into_mnt(mnt_userns
, inode
)) ||
2855 !gid_valid(i_gid_into_mnt(mnt_userns
, inode
)))
2858 audit_inode_child(dir
, victim
, AUDIT_TYPE_CHILD_DELETE
);
2860 error
= inode_permission(mnt_userns
, dir
, MAY_WRITE
| MAY_EXEC
);
2866 if (check_sticky(mnt_userns
, dir
, inode
) || IS_APPEND(inode
) ||
2867 IS_IMMUTABLE(inode
) || IS_SWAPFILE(inode
) ||
2868 HAS_UNMAPPED_ID(mnt_userns
, inode
))
2871 if (!d_is_dir(victim
))
2873 if (IS_ROOT(victim
))
2875 } else if (d_is_dir(victim
))
2877 if (IS_DEADDIR(dir
))
2879 if (victim
->d_flags
& DCACHE_NFSFS_RENAMED
)
2884 /* Check whether we can create an object with dentry child in directory
2886 * 1. We can't do it if child already exists (open has special treatment for
2887 * this case, but since we are inlined it's OK)
2888 * 2. We can't do it if dir is read-only (done in permission())
2889 * 3. We can't do it if the fs can't represent the fsuid or fsgid.
2890 * 4. We should have write and exec permissions on dir
2891 * 5. We can't do it if dir is immutable (done in permission())
2893 static inline int may_create(struct user_namespace
*mnt_userns
,
2894 struct inode
*dir
, struct dentry
*child
)
2896 audit_inode_child(dir
, child
, AUDIT_TYPE_CHILD_CREATE
);
2899 if (IS_DEADDIR(dir
))
2901 if (!fsuidgid_has_mapping(dir
->i_sb
, mnt_userns
))
2904 return inode_permission(mnt_userns
, dir
, MAY_WRITE
| MAY_EXEC
);
2908 * p1 and p2 should be directories on the same fs.
2910 struct dentry
*lock_rename(struct dentry
*p1
, struct dentry
*p2
)
2915 inode_lock_nested(p1
->d_inode
, I_MUTEX_PARENT
);
2919 mutex_lock(&p1
->d_sb
->s_vfs_rename_mutex
);
2921 p
= d_ancestor(p2
, p1
);
2923 inode_lock_nested(p2
->d_inode
, I_MUTEX_PARENT
);
2924 inode_lock_nested(p1
->d_inode
, I_MUTEX_CHILD
);
2928 p
= d_ancestor(p1
, p2
);
2930 inode_lock_nested(p1
->d_inode
, I_MUTEX_PARENT
);
2931 inode_lock_nested(p2
->d_inode
, I_MUTEX_CHILD
);
2935 inode_lock_nested(p1
->d_inode
, I_MUTEX_PARENT
);
2936 inode_lock_nested(p2
->d_inode
, I_MUTEX_PARENT2
);
2939 EXPORT_SYMBOL(lock_rename
);
2941 void unlock_rename(struct dentry
*p1
, struct dentry
*p2
)
2943 inode_unlock(p1
->d_inode
);
2945 inode_unlock(p2
->d_inode
);
2946 mutex_unlock(&p1
->d_sb
->s_vfs_rename_mutex
);
2949 EXPORT_SYMBOL(unlock_rename
);
2952 * vfs_create - create new file
2953 * @mnt_userns: user namespace of the mount the inode was found from
2954 * @dir: inode of @dentry
2955 * @dentry: pointer to dentry of the base directory
2956 * @mode: mode of the new file
2957 * @want_excl: whether the file must not yet exist
2959 * Create a new file.
2961 * If the inode has been found through an idmapped mount the user namespace of
2962 * the vfsmount must be passed through @mnt_userns. This function will then take
2963 * care to map the inode according to @mnt_userns before checking permissions.
2964 * On non-idmapped mounts or if permission checking is to be performed on the
2965 * raw inode simply passs init_user_ns.
2967 int vfs_create(struct user_namespace
*mnt_userns
, struct inode
*dir
,
2968 struct dentry
*dentry
, umode_t mode
, bool want_excl
)
2970 int error
= may_create(mnt_userns
, dir
, dentry
);
2974 if (!dir
->i_op
->create
)
2975 return -EACCES
; /* shouldn't it be ENOSYS? */
2978 error
= security_inode_create(dir
, dentry
, mode
);
2981 error
= dir
->i_op
->create(mnt_userns
, dir
, dentry
, mode
, want_excl
);
2983 fsnotify_create(dir
, dentry
);
2986 EXPORT_SYMBOL(vfs_create
);
2988 int vfs_mkobj(struct dentry
*dentry
, umode_t mode
,
2989 int (*f
)(struct dentry
*, umode_t
, void *),
2992 struct inode
*dir
= dentry
->d_parent
->d_inode
;
2993 int error
= may_create(&init_user_ns
, dir
, dentry
);
2999 error
= security_inode_create(dir
, dentry
, mode
);
3002 error
= f(dentry
, mode
, arg
);
3004 fsnotify_create(dir
, dentry
);
3007 EXPORT_SYMBOL(vfs_mkobj
);
3009 bool may_open_dev(const struct path
*path
)
3011 return !(path
->mnt
->mnt_flags
& MNT_NODEV
) &&
3012 !(path
->mnt
->mnt_sb
->s_iflags
& SB_I_NODEV
);
3015 static int may_open(struct user_namespace
*mnt_userns
, const struct path
*path
,
3016 int acc_mode
, int flag
)
3018 struct dentry
*dentry
= path
->dentry
;
3019 struct inode
*inode
= dentry
->d_inode
;
3025 switch (inode
->i_mode
& S_IFMT
) {
3029 if (acc_mode
& MAY_WRITE
)
3031 if (acc_mode
& MAY_EXEC
)
3036 if (!may_open_dev(path
))
3041 if (acc_mode
& MAY_EXEC
)
3046 if ((acc_mode
& MAY_EXEC
) && path_noexec(path
))
3051 error
= inode_permission(mnt_userns
, inode
, MAY_OPEN
| acc_mode
);
3056 * An append-only file must be opened in append mode for writing.
3058 if (IS_APPEND(inode
)) {
3059 if ((flag
& O_ACCMODE
) != O_RDONLY
&& !(flag
& O_APPEND
))
3065 /* O_NOATIME can only be set by the owner or superuser */
3066 if (flag
& O_NOATIME
&& !inode_owner_or_capable(mnt_userns
, inode
))
3072 static int handle_truncate(struct user_namespace
*mnt_userns
, struct file
*filp
)
3074 const struct path
*path
= &filp
->f_path
;
3075 struct inode
*inode
= path
->dentry
->d_inode
;
3076 int error
= get_write_access(inode
);
3080 * Refuse to truncate files with mandatory locks held on them.
3082 error
= security_path_truncate(path
);
3084 error
= do_truncate(mnt_userns
, path
->dentry
, 0,
3085 ATTR_MTIME
|ATTR_CTIME
|ATTR_OPEN
,
3088 put_write_access(inode
);
3092 static inline int open_to_namei_flags(int flag
)
3094 if ((flag
& O_ACCMODE
) == 3)
3099 static int may_o_create(struct user_namespace
*mnt_userns
,
3100 const struct path
*dir
, struct dentry
*dentry
,
3103 int error
= security_path_mknod(dir
, dentry
, mode
, 0);
3107 if (!fsuidgid_has_mapping(dir
->dentry
->d_sb
, mnt_userns
))
3110 error
= inode_permission(mnt_userns
, dir
->dentry
->d_inode
,
3111 MAY_WRITE
| MAY_EXEC
);
3115 return security_inode_create(dir
->dentry
->d_inode
, dentry
, mode
);
3119 * Attempt to atomically look up, create and open a file from a negative
3122 * Returns 0 if successful. The file will have been created and attached to
3123 * @file by the filesystem calling finish_open().
3125 * If the file was looked up only or didn't need creating, FMODE_OPENED won't
3126 * be set. The caller will need to perform the open themselves. @path will
3127 * have been updated to point to the new dentry. This may be negative.
3129 * Returns an error code otherwise.
3131 static struct dentry
*atomic_open(struct nameidata
*nd
, struct dentry
*dentry
,
3133 int open_flag
, umode_t mode
)
3135 struct dentry
*const DENTRY_NOT_SET
= (void *) -1UL;
3136 struct inode
*dir
= nd
->path
.dentry
->d_inode
;
3139 if (nd
->flags
& LOOKUP_DIRECTORY
)
3140 open_flag
|= O_DIRECTORY
;
3142 file
->f_path
.dentry
= DENTRY_NOT_SET
;
3143 file
->f_path
.mnt
= nd
->path
.mnt
;
3144 error
= dir
->i_op
->atomic_open(dir
, dentry
, file
,
3145 open_to_namei_flags(open_flag
), mode
);
3146 d_lookup_done(dentry
);
3148 if (file
->f_mode
& FMODE_OPENED
) {
3149 if (unlikely(dentry
!= file
->f_path
.dentry
)) {
3151 dentry
= dget(file
->f_path
.dentry
);
3153 } else if (WARN_ON(file
->f_path
.dentry
== DENTRY_NOT_SET
)) {
3156 if (file
->f_path
.dentry
) {
3158 dentry
= file
->f_path
.dentry
;
3160 if (unlikely(d_is_negative(dentry
)))
3166 dentry
= ERR_PTR(error
);
3172 * Look up and maybe create and open the last component.
3174 * Must be called with parent locked (exclusive in O_CREAT case).
3176 * Returns 0 on success, that is, if
3177 * the file was successfully atomically created (if necessary) and opened, or
3178 * the file was not completely opened at this time, though lookups and
3179 * creations were performed.
3180 * These case are distinguished by presence of FMODE_OPENED on file->f_mode.
3181 * In the latter case dentry returned in @path might be negative if O_CREAT
3182 * hadn't been specified.
3184 * An error code is returned on failure.
3186 static struct dentry
*lookup_open(struct nameidata
*nd
, struct file
*file
,
3187 const struct open_flags
*op
,
3190 struct user_namespace
*mnt_userns
;
3191 struct dentry
*dir
= nd
->path
.dentry
;
3192 struct inode
*dir_inode
= dir
->d_inode
;
3193 int open_flag
= op
->open_flag
;
3194 struct dentry
*dentry
;
3195 int error
, create_error
= 0;
3196 umode_t mode
= op
->mode
;
3197 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq
);
3199 if (unlikely(IS_DEADDIR(dir_inode
)))
3200 return ERR_PTR(-ENOENT
);
3202 file
->f_mode
&= ~FMODE_CREATED
;
3203 dentry
= d_lookup(dir
, &nd
->last
);
3206 dentry
= d_alloc_parallel(dir
, &nd
->last
, &wq
);
3210 if (d_in_lookup(dentry
))
3213 error
= d_revalidate(dentry
, nd
->flags
);
3214 if (likely(error
> 0))
3218 d_invalidate(dentry
);
3222 if (dentry
->d_inode
) {
3223 /* Cached positive dentry: will open in f_op->open */
3228 * Checking write permission is tricky, bacuse we don't know if we are
3229 * going to actually need it: O_CREAT opens should work as long as the
3230 * file exists. But checking existence breaks atomicity. The trick is
3231 * to check access and if not granted clear O_CREAT from the flags.
3233 * Another problem is returing the "right" error value (e.g. for an
3234 * O_EXCL open we want to return EEXIST not EROFS).
3236 if (unlikely(!got_write
))
3237 open_flag
&= ~O_TRUNC
;
3238 mnt_userns
= mnt_user_ns(nd
->path
.mnt
);
3239 if (open_flag
& O_CREAT
) {
3240 if (open_flag
& O_EXCL
)
3241 open_flag
&= ~O_TRUNC
;
3242 if (!IS_POSIXACL(dir
->d_inode
))
3243 mode
&= ~current_umask();
3244 if (likely(got_write
))
3245 create_error
= may_o_create(mnt_userns
, &nd
->path
,
3248 create_error
= -EROFS
;
3251 open_flag
&= ~O_CREAT
;
3252 if (dir_inode
->i_op
->atomic_open
) {
3253 dentry
= atomic_open(nd
, dentry
, file
, open_flag
, mode
);
3254 if (unlikely(create_error
) && dentry
== ERR_PTR(-ENOENT
))
3255 dentry
= ERR_PTR(create_error
);
3259 if (d_in_lookup(dentry
)) {
3260 struct dentry
*res
= dir_inode
->i_op
->lookup(dir_inode
, dentry
,
3262 d_lookup_done(dentry
);
3263 if (unlikely(res
)) {
3265 error
= PTR_ERR(res
);
3273 /* Negative dentry, just create the file */
3274 if (!dentry
->d_inode
&& (open_flag
& O_CREAT
)) {
3275 file
->f_mode
|= FMODE_CREATED
;
3276 audit_inode_child(dir_inode
, dentry
, AUDIT_TYPE_CHILD_CREATE
);
3277 if (!dir_inode
->i_op
->create
) {
3282 error
= dir_inode
->i_op
->create(mnt_userns
, dir_inode
, dentry
,
3283 mode
, open_flag
& O_EXCL
);
3287 if (unlikely(create_error
) && !dentry
->d_inode
) {
3288 error
= create_error
;
3295 return ERR_PTR(error
);
3298 static const char *open_last_lookups(struct nameidata
*nd
,
3299 struct file
*file
, const struct open_flags
*op
)
3301 struct dentry
*dir
= nd
->path
.dentry
;
3302 int open_flag
= op
->open_flag
;
3303 bool got_write
= false;
3305 struct inode
*inode
;
3306 struct dentry
*dentry
;
3309 nd
->flags
|= op
->intent
;
3311 if (nd
->last_type
!= LAST_NORM
) {
3314 return handle_dots(nd
, nd
->last_type
);
3317 if (!(open_flag
& O_CREAT
)) {
3318 if (nd
->last
.name
[nd
->last
.len
])
3319 nd
->flags
|= LOOKUP_FOLLOW
| LOOKUP_DIRECTORY
;
3320 /* we _can_ be in RCU mode here */
3321 dentry
= lookup_fast(nd
, &inode
, &seq
);
3323 return ERR_CAST(dentry
);
3327 BUG_ON(nd
->flags
& LOOKUP_RCU
);
3329 /* create side of things */
3330 if (nd
->flags
& LOOKUP_RCU
) {
3331 if (!try_to_unlazy(nd
))
3332 return ERR_PTR(-ECHILD
);
3334 audit_inode(nd
->name
, dir
, AUDIT_INODE_PARENT
);
3335 /* trailing slashes? */
3336 if (unlikely(nd
->last
.name
[nd
->last
.len
]))
3337 return ERR_PTR(-EISDIR
);
3340 if (open_flag
& (O_CREAT
| O_TRUNC
| O_WRONLY
| O_RDWR
)) {
3341 got_write
= !mnt_want_write(nd
->path
.mnt
);
3343 * do _not_ fail yet - we might not need that or fail with
3344 * a different error; let lookup_open() decide; we'll be
3345 * dropping this one anyway.
3348 if (open_flag
& O_CREAT
)
3349 inode_lock(dir
->d_inode
);
3351 inode_lock_shared(dir
->d_inode
);
3352 dentry
= lookup_open(nd
, file
, op
, got_write
);
3353 if (!IS_ERR(dentry
) && (file
->f_mode
& FMODE_CREATED
))
3354 fsnotify_create(dir
->d_inode
, dentry
);
3355 if (open_flag
& O_CREAT
)
3356 inode_unlock(dir
->d_inode
);
3358 inode_unlock_shared(dir
->d_inode
);
3361 mnt_drop_write(nd
->path
.mnt
);
3364 return ERR_CAST(dentry
);
3366 if (file
->f_mode
& (FMODE_OPENED
| FMODE_CREATED
)) {
3367 dput(nd
->path
.dentry
);
3368 nd
->path
.dentry
= dentry
;
3375 res
= step_into(nd
, WALK_TRAILING
, dentry
, inode
, seq
);
3377 nd
->flags
&= ~(LOOKUP_OPEN
|LOOKUP_CREATE
|LOOKUP_EXCL
);
3382 * Handle the last step of open()
3384 static int do_open(struct nameidata
*nd
,
3385 struct file
*file
, const struct open_flags
*op
)
3387 struct user_namespace
*mnt_userns
;
3388 int open_flag
= op
->open_flag
;
3393 if (!(file
->f_mode
& (FMODE_OPENED
| FMODE_CREATED
))) {
3394 error
= complete_walk(nd
);
3398 if (!(file
->f_mode
& FMODE_CREATED
))
3399 audit_inode(nd
->name
, nd
->path
.dentry
, 0);
3400 mnt_userns
= mnt_user_ns(nd
->path
.mnt
);
3401 if (open_flag
& O_CREAT
) {
3402 if ((open_flag
& O_EXCL
) && !(file
->f_mode
& FMODE_CREATED
))
3404 if (d_is_dir(nd
->path
.dentry
))
3406 error
= may_create_in_sticky(mnt_userns
, nd
,
3407 d_backing_inode(nd
->path
.dentry
));
3408 if (unlikely(error
))
3411 if ((nd
->flags
& LOOKUP_DIRECTORY
) && !d_can_lookup(nd
->path
.dentry
))
3414 do_truncate
= false;
3415 acc_mode
= op
->acc_mode
;
3416 if (file
->f_mode
& FMODE_CREATED
) {
3417 /* Don't check for write permission, don't truncate */
3418 open_flag
&= ~O_TRUNC
;
3420 } else if (d_is_reg(nd
->path
.dentry
) && open_flag
& O_TRUNC
) {
3421 error
= mnt_want_write(nd
->path
.mnt
);
3426 error
= may_open(mnt_userns
, &nd
->path
, acc_mode
, open_flag
);
3427 if (!error
&& !(file
->f_mode
& FMODE_OPENED
))
3428 error
= vfs_open(&nd
->path
, file
);
3430 error
= ima_file_check(file
, op
->acc_mode
);
3431 if (!error
&& do_truncate
)
3432 error
= handle_truncate(mnt_userns
, file
);
3433 if (unlikely(error
> 0)) {
3438 mnt_drop_write(nd
->path
.mnt
);
3443 * vfs_tmpfile - create tmpfile
3444 * @mnt_userns: user namespace of the mount the inode was found from
3445 * @dentry: pointer to dentry of the base directory
3446 * @mode: mode of the new tmpfile
3449 * Create a temporary file.
3451 * If the inode has been found through an idmapped mount the user namespace of
3452 * the vfsmount must be passed through @mnt_userns. This function will then take
3453 * care to map the inode according to @mnt_userns before checking permissions.
3454 * On non-idmapped mounts or if permission checking is to be performed on the
3455 * raw inode simply passs init_user_ns.
3457 struct dentry
*vfs_tmpfile(struct user_namespace
*mnt_userns
,
3458 struct dentry
*dentry
, umode_t mode
, int open_flag
)
3460 struct dentry
*child
= NULL
;
3461 struct inode
*dir
= dentry
->d_inode
;
3462 struct inode
*inode
;
3465 /* we want directory to be writable */
3466 error
= inode_permission(mnt_userns
, dir
, MAY_WRITE
| MAY_EXEC
);
3469 error
= -EOPNOTSUPP
;
3470 if (!dir
->i_op
->tmpfile
)
3473 child
= d_alloc(dentry
, &slash_name
);
3474 if (unlikely(!child
))
3476 error
= dir
->i_op
->tmpfile(mnt_userns
, dir
, child
, mode
);
3480 inode
= child
->d_inode
;
3481 if (unlikely(!inode
))
3483 if (!(open_flag
& O_EXCL
)) {
3484 spin_lock(&inode
->i_lock
);
3485 inode
->i_state
|= I_LINKABLE
;
3486 spin_unlock(&inode
->i_lock
);
3488 ima_post_create_tmpfile(mnt_userns
, inode
);
3493 return ERR_PTR(error
);
3495 EXPORT_SYMBOL(vfs_tmpfile
);
3497 static int do_tmpfile(struct nameidata
*nd
, unsigned flags
,
3498 const struct open_flags
*op
,
3501 struct user_namespace
*mnt_userns
;
3502 struct dentry
*child
;
3504 int error
= path_lookupat(nd
, flags
| LOOKUP_DIRECTORY
, &path
);
3505 if (unlikely(error
))
3507 error
= mnt_want_write(path
.mnt
);
3508 if (unlikely(error
))
3510 mnt_userns
= mnt_user_ns(path
.mnt
);
3511 child
= vfs_tmpfile(mnt_userns
, path
.dentry
, op
->mode
, op
->open_flag
);
3512 error
= PTR_ERR(child
);
3516 path
.dentry
= child
;
3517 audit_inode(nd
->name
, child
, 0);
3518 /* Don't check for other permissions, the inode was just created */
3519 error
= may_open(mnt_userns
, &path
, 0, op
->open_flag
);
3521 error
= vfs_open(&path
, file
);
3523 mnt_drop_write(path
.mnt
);
3529 static int do_o_path(struct nameidata
*nd
, unsigned flags
, struct file
*file
)
3532 int error
= path_lookupat(nd
, flags
, &path
);
3534 audit_inode(nd
->name
, path
.dentry
, 0);
3535 error
= vfs_open(&path
, file
);
3541 static struct file
*path_openat(struct nameidata
*nd
,
3542 const struct open_flags
*op
, unsigned flags
)
3547 file
= alloc_empty_file(op
->open_flag
, current_cred());
3551 if (unlikely(file
->f_flags
& __O_TMPFILE
)) {
3552 error
= do_tmpfile(nd
, flags
, op
, file
);
3553 } else if (unlikely(file
->f_flags
& O_PATH
)) {
3554 error
= do_o_path(nd
, flags
, file
);
3556 const char *s
= path_init(nd
, flags
);
3557 while (!(error
= link_path_walk(s
, nd
)) &&
3558 (s
= open_last_lookups(nd
, file
, op
)) != NULL
)
3561 error
= do_open(nd
, file
, op
);
3564 if (likely(!error
)) {
3565 if (likely(file
->f_mode
& FMODE_OPENED
))
3571 if (error
== -EOPENSTALE
) {
3572 if (flags
& LOOKUP_RCU
)
3577 return ERR_PTR(error
);
3580 struct file
*do_filp_open(int dfd
, struct filename
*pathname
,
3581 const struct open_flags
*op
)
3583 struct nameidata nd
;
3584 int flags
= op
->lookup_flags
;
3587 set_nameidata(&nd
, dfd
, pathname
, NULL
);
3588 filp
= path_openat(&nd
, op
, flags
| LOOKUP_RCU
);
3589 if (unlikely(filp
== ERR_PTR(-ECHILD
)))
3590 filp
= path_openat(&nd
, op
, flags
);
3591 if (unlikely(filp
== ERR_PTR(-ESTALE
)))
3592 filp
= path_openat(&nd
, op
, flags
| LOOKUP_REVAL
);
3593 restore_nameidata();
3597 struct file
*do_file_open_root(const struct path
*root
,
3598 const char *name
, const struct open_flags
*op
)
3600 struct nameidata nd
;
3602 struct filename
*filename
;
3603 int flags
= op
->lookup_flags
;
3605 if (d_is_symlink(root
->dentry
) && op
->intent
& LOOKUP_OPEN
)
3606 return ERR_PTR(-ELOOP
);
3608 filename
= getname_kernel(name
);
3609 if (IS_ERR(filename
))
3610 return ERR_CAST(filename
);
3612 set_nameidata(&nd
, -1, filename
, root
);
3613 file
= path_openat(&nd
, op
, flags
| LOOKUP_RCU
);
3614 if (unlikely(file
== ERR_PTR(-ECHILD
)))
3615 file
= path_openat(&nd
, op
, flags
);
3616 if (unlikely(file
== ERR_PTR(-ESTALE
)))
3617 file
= path_openat(&nd
, op
, flags
| LOOKUP_REVAL
);
3618 restore_nameidata();
3623 static struct dentry
*filename_create(int dfd
, struct filename
*name
,
3624 struct path
*path
, unsigned int lookup_flags
)
3626 struct dentry
*dentry
= ERR_PTR(-EEXIST
);
3631 bool is_dir
= (lookup_flags
& LOOKUP_DIRECTORY
);
3634 * Note that only LOOKUP_REVAL and LOOKUP_DIRECTORY matter here. Any
3635 * other flags passed in are ignored!
3637 lookup_flags
&= LOOKUP_REVAL
;
3639 error
= filename_parentat(dfd
, name
, lookup_flags
, path
, &last
, &type
);
3641 return ERR_PTR(error
);
3644 * Yucky last component or no last component at all?
3645 * (foo/., foo/.., /////)
3647 if (unlikely(type
!= LAST_NORM
))
3650 /* don't fail immediately if it's r/o, at least try to report other errors */
3651 err2
= mnt_want_write(path
->mnt
);
3653 * Do the final lookup.
3655 lookup_flags
|= LOOKUP_CREATE
| LOOKUP_EXCL
;
3656 inode_lock_nested(path
->dentry
->d_inode
, I_MUTEX_PARENT
);
3657 dentry
= __lookup_hash(&last
, path
->dentry
, lookup_flags
);
3662 if (d_is_positive(dentry
))
3666 * Special case - lookup gave negative, but... we had foo/bar/
3667 * From the vfs_mknod() POV we just have a negative dentry -
3668 * all is fine. Let's be bastards - you had / on the end, you've
3669 * been asking for (non-existent) directory. -ENOENT for you.
3671 if (unlikely(!is_dir
&& last
.name
[last
.len
])) {
3675 if (unlikely(err2
)) {
3682 dentry
= ERR_PTR(error
);
3684 inode_unlock(path
->dentry
->d_inode
);
3686 mnt_drop_write(path
->mnt
);
3692 struct dentry
*kern_path_create(int dfd
, const char *pathname
,
3693 struct path
*path
, unsigned int lookup_flags
)
3695 struct filename
*filename
= getname_kernel(pathname
);
3696 struct dentry
*res
= filename_create(dfd
, filename
, path
, lookup_flags
);
3701 EXPORT_SYMBOL(kern_path_create
);
3703 void done_path_create(struct path
*path
, struct dentry
*dentry
)
3706 inode_unlock(path
->dentry
->d_inode
);
3707 mnt_drop_write(path
->mnt
);
3710 EXPORT_SYMBOL(done_path_create
);
3712 inline struct dentry
*user_path_create(int dfd
, const char __user
*pathname
,
3713 struct path
*path
, unsigned int lookup_flags
)
3715 struct filename
*filename
= getname(pathname
);
3716 struct dentry
*res
= filename_create(dfd
, filename
, path
, lookup_flags
);
3721 EXPORT_SYMBOL(user_path_create
);
3724 * vfs_mknod - create device node or file
3725 * @mnt_userns: user namespace of the mount the inode was found from
3726 * @dir: inode of @dentry
3727 * @dentry: pointer to dentry of the base directory
3728 * @mode: mode of the new device node or file
3729 * @dev: device number of device to create
3731 * Create a device node or file.
3733 * If the inode has been found through an idmapped mount the user namespace of
3734 * the vfsmount must be passed through @mnt_userns. This function will then take
3735 * care to map the inode according to @mnt_userns before checking permissions.
3736 * On non-idmapped mounts or if permission checking is to be performed on the
3737 * raw inode simply passs init_user_ns.
3739 int vfs_mknod(struct user_namespace
*mnt_userns
, struct inode
*dir
,
3740 struct dentry
*dentry
, umode_t mode
, dev_t dev
)
3742 bool is_whiteout
= S_ISCHR(mode
) && dev
== WHITEOUT_DEV
;
3743 int error
= may_create(mnt_userns
, dir
, dentry
);
3748 if ((S_ISCHR(mode
) || S_ISBLK(mode
)) && !is_whiteout
&&
3749 !capable(CAP_MKNOD
))
3752 if (!dir
->i_op
->mknod
)
3755 error
= devcgroup_inode_mknod(mode
, dev
);
3759 error
= security_inode_mknod(dir
, dentry
, mode
, dev
);
3763 error
= dir
->i_op
->mknod(mnt_userns
, dir
, dentry
, mode
, dev
);
3765 fsnotify_create(dir
, dentry
);
3768 EXPORT_SYMBOL(vfs_mknod
);
3770 static int may_mknod(umode_t mode
)
3772 switch (mode
& S_IFMT
) {
3778 case 0: /* zero mode translates to S_IFREG */
3787 static int do_mknodat(int dfd
, struct filename
*name
, umode_t mode
,
3790 struct user_namespace
*mnt_userns
;
3791 struct dentry
*dentry
;
3794 unsigned int lookup_flags
= 0;
3796 error
= may_mknod(mode
);
3800 dentry
= filename_create(dfd
, name
, &path
, lookup_flags
);
3801 error
= PTR_ERR(dentry
);
3805 if (!IS_POSIXACL(path
.dentry
->d_inode
))
3806 mode
&= ~current_umask();
3807 error
= security_path_mknod(&path
, dentry
, mode
, dev
);
3811 mnt_userns
= mnt_user_ns(path
.mnt
);
3812 switch (mode
& S_IFMT
) {
3813 case 0: case S_IFREG
:
3814 error
= vfs_create(mnt_userns
, path
.dentry
->d_inode
,
3815 dentry
, mode
, true);
3817 ima_post_path_mknod(mnt_userns
, dentry
);
3819 case S_IFCHR
: case S_IFBLK
:
3820 error
= vfs_mknod(mnt_userns
, path
.dentry
->d_inode
,
3821 dentry
, mode
, new_decode_dev(dev
));
3823 case S_IFIFO
: case S_IFSOCK
:
3824 error
= vfs_mknod(mnt_userns
, path
.dentry
->d_inode
,
3829 done_path_create(&path
, dentry
);
3830 if (retry_estale(error
, lookup_flags
)) {
3831 lookup_flags
|= LOOKUP_REVAL
;
3839 SYSCALL_DEFINE4(mknodat
, int, dfd
, const char __user
*, filename
, umode_t
, mode
,
3842 return do_mknodat(dfd
, getname(filename
), mode
, dev
);
3845 SYSCALL_DEFINE3(mknod
, const char __user
*, filename
, umode_t
, mode
, unsigned, dev
)
3847 return do_mknodat(AT_FDCWD
, getname(filename
), mode
, dev
);
3851 * vfs_mkdir - create directory
3852 * @mnt_userns: user namespace of the mount the inode was found from
3853 * @dir: inode of @dentry
3854 * @dentry: pointer to dentry of the base directory
3855 * @mode: mode of the new directory
3857 * Create a directory.
3859 * If the inode has been found through an idmapped mount the user namespace of
3860 * the vfsmount must be passed through @mnt_userns. This function will then take
3861 * care to map the inode according to @mnt_userns before checking permissions.
3862 * On non-idmapped mounts or if permission checking is to be performed on the
3863 * raw inode simply passs init_user_ns.
3865 int vfs_mkdir(struct user_namespace
*mnt_userns
, struct inode
*dir
,
3866 struct dentry
*dentry
, umode_t mode
)
3868 int error
= may_create(mnt_userns
, dir
, dentry
);
3869 unsigned max_links
= dir
->i_sb
->s_max_links
;
3874 if (!dir
->i_op
->mkdir
)
3877 mode
&= (S_IRWXUGO
|S_ISVTX
);
3878 error
= security_inode_mkdir(dir
, dentry
, mode
);
3882 if (max_links
&& dir
->i_nlink
>= max_links
)
3885 error
= dir
->i_op
->mkdir(mnt_userns
, dir
, dentry
, mode
);
3887 fsnotify_mkdir(dir
, dentry
);
3890 EXPORT_SYMBOL(vfs_mkdir
);
3892 int do_mkdirat(int dfd
, struct filename
*name
, umode_t mode
)
3894 struct dentry
*dentry
;
3897 unsigned int lookup_flags
= LOOKUP_DIRECTORY
;
3900 dentry
= filename_create(dfd
, name
, &path
, lookup_flags
);
3901 error
= PTR_ERR(dentry
);
3905 if (!IS_POSIXACL(path
.dentry
->d_inode
))
3906 mode
&= ~current_umask();
3907 error
= security_path_mkdir(&path
, dentry
, mode
);
3909 struct user_namespace
*mnt_userns
;
3910 mnt_userns
= mnt_user_ns(path
.mnt
);
3911 error
= vfs_mkdir(mnt_userns
, path
.dentry
->d_inode
, dentry
,
3914 done_path_create(&path
, dentry
);
3915 if (retry_estale(error
, lookup_flags
)) {
3916 lookup_flags
|= LOOKUP_REVAL
;
3924 SYSCALL_DEFINE3(mkdirat
, int, dfd
, const char __user
*, pathname
, umode_t
, mode
)
3926 return do_mkdirat(dfd
, getname(pathname
), mode
);
3929 SYSCALL_DEFINE2(mkdir
, const char __user
*, pathname
, umode_t
, mode
)
3931 return do_mkdirat(AT_FDCWD
, getname(pathname
), mode
);
3935 * vfs_rmdir - remove directory
3936 * @mnt_userns: user namespace of the mount the inode was found from
3937 * @dir: inode of @dentry
3938 * @dentry: pointer to dentry of the base directory
3940 * Remove a directory.
3942 * If the inode has been found through an idmapped mount the user namespace of
3943 * the vfsmount must be passed through @mnt_userns. This function will then take
3944 * care to map the inode according to @mnt_userns before checking permissions.
3945 * On non-idmapped mounts or if permission checking is to be performed on the
3946 * raw inode simply passs init_user_ns.
3948 int vfs_rmdir(struct user_namespace
*mnt_userns
, struct inode
*dir
,
3949 struct dentry
*dentry
)
3951 int error
= may_delete(mnt_userns
, dir
, dentry
, 1);
3956 if (!dir
->i_op
->rmdir
)
3960 inode_lock(dentry
->d_inode
);
3963 if (is_local_mountpoint(dentry
))
3966 error
= security_inode_rmdir(dir
, dentry
);
3970 error
= dir
->i_op
->rmdir(dir
, dentry
);
3974 shrink_dcache_parent(dentry
);
3975 dentry
->d_inode
->i_flags
|= S_DEAD
;
3977 detach_mounts(dentry
);
3978 fsnotify_rmdir(dir
, dentry
);
3981 inode_unlock(dentry
->d_inode
);
3987 EXPORT_SYMBOL(vfs_rmdir
);
3989 int do_rmdir(int dfd
, struct filename
*name
)
3991 struct user_namespace
*mnt_userns
;
3993 struct dentry
*dentry
;
3997 unsigned int lookup_flags
= 0;
3999 error
= filename_parentat(dfd
, name
, lookup_flags
, &path
, &last
, &type
);
4015 error
= mnt_want_write(path
.mnt
);
4019 inode_lock_nested(path
.dentry
->d_inode
, I_MUTEX_PARENT
);
4020 dentry
= __lookup_hash(&last
, path
.dentry
, lookup_flags
);
4021 error
= PTR_ERR(dentry
);
4024 if (!dentry
->d_inode
) {
4028 error
= security_path_rmdir(&path
, dentry
);
4031 mnt_userns
= mnt_user_ns(path
.mnt
);
4032 error
= vfs_rmdir(mnt_userns
, path
.dentry
->d_inode
, dentry
);
4036 inode_unlock(path
.dentry
->d_inode
);
4037 mnt_drop_write(path
.mnt
);
4040 if (retry_estale(error
, lookup_flags
)) {
4041 lookup_flags
|= LOOKUP_REVAL
;
4049 SYSCALL_DEFINE1(rmdir
, const char __user
*, pathname
)
4051 return do_rmdir(AT_FDCWD
, getname(pathname
));
4055 * vfs_unlink - unlink a filesystem object
4056 * @mnt_userns: user namespace of the mount the inode was found from
4057 * @dir: parent directory
4059 * @delegated_inode: returns victim inode, if the inode is delegated.
4061 * The caller must hold dir->i_mutex.
4063 * If vfs_unlink discovers a delegation, it will return -EWOULDBLOCK and
4064 * return a reference to the inode in delegated_inode. The caller
4065 * should then break the delegation on that inode and retry. Because
4066 * breaking a delegation may take a long time, the caller should drop
4067 * dir->i_mutex before doing so.
4069 * Alternatively, a caller may pass NULL for delegated_inode. This may
4070 * be appropriate for callers that expect the underlying filesystem not
4071 * to be NFS exported.
4073 * If the inode has been found through an idmapped mount the user namespace of
4074 * the vfsmount must be passed through @mnt_userns. This function will then take
4075 * care to map the inode according to @mnt_userns before checking permissions.
4076 * On non-idmapped mounts or if permission checking is to be performed on the
4077 * raw inode simply passs init_user_ns.
4079 int vfs_unlink(struct user_namespace
*mnt_userns
, struct inode
*dir
,
4080 struct dentry
*dentry
, struct inode
**delegated_inode
)
4082 struct inode
*target
= dentry
->d_inode
;
4083 int error
= may_delete(mnt_userns
, dir
, dentry
, 0);
4088 if (!dir
->i_op
->unlink
)
4092 if (IS_SWAPFILE(target
))
4094 else if (is_local_mountpoint(dentry
))
4097 error
= security_inode_unlink(dir
, dentry
);
4099 error
= try_break_deleg(target
, delegated_inode
);
4102 error
= dir
->i_op
->unlink(dir
, dentry
);
4105 detach_mounts(dentry
);
4106 fsnotify_unlink(dir
, dentry
);
4111 inode_unlock(target
);
4113 /* We don't d_delete() NFS sillyrenamed files--they still exist. */
4114 if (!error
&& !(dentry
->d_flags
& DCACHE_NFSFS_RENAMED
)) {
4115 fsnotify_link_count(target
);
4121 EXPORT_SYMBOL(vfs_unlink
);
4124 * Make sure that the actual truncation of the file will occur outside its
4125 * directory's i_mutex. Truncate can take a long time if there is a lot of
4126 * writeout happening, and we don't want to prevent access to the directory
4127 * while waiting on the I/O.
4129 int do_unlinkat(int dfd
, struct filename
*name
)
4132 struct dentry
*dentry
;
4136 struct inode
*inode
= NULL
;
4137 struct inode
*delegated_inode
= NULL
;
4138 unsigned int lookup_flags
= 0;
4140 error
= filename_parentat(dfd
, name
, lookup_flags
, &path
, &last
, &type
);
4145 if (type
!= LAST_NORM
)
4148 error
= mnt_want_write(path
.mnt
);
4152 inode_lock_nested(path
.dentry
->d_inode
, I_MUTEX_PARENT
);
4153 dentry
= __lookup_hash(&last
, path
.dentry
, lookup_flags
);
4154 error
= PTR_ERR(dentry
);
4155 if (!IS_ERR(dentry
)) {
4156 struct user_namespace
*mnt_userns
;
4158 /* Why not before? Because we want correct error value */
4159 if (last
.name
[last
.len
])
4161 inode
= dentry
->d_inode
;
4162 if (d_is_negative(dentry
))
4165 error
= security_path_unlink(&path
, dentry
);
4168 mnt_userns
= mnt_user_ns(path
.mnt
);
4169 error
= vfs_unlink(mnt_userns
, path
.dentry
->d_inode
, dentry
,
4174 inode_unlock(path
.dentry
->d_inode
);
4176 iput(inode
); /* truncate the inode here */
4178 if (delegated_inode
) {
4179 error
= break_deleg_wait(&delegated_inode
);
4183 mnt_drop_write(path
.mnt
);
4186 if (retry_estale(error
, lookup_flags
)) {
4187 lookup_flags
|= LOOKUP_REVAL
;
4196 if (d_is_negative(dentry
))
4198 else if (d_is_dir(dentry
))
4205 SYSCALL_DEFINE3(unlinkat
, int, dfd
, const char __user
*, pathname
, int, flag
)
4207 if ((flag
& ~AT_REMOVEDIR
) != 0)
4210 if (flag
& AT_REMOVEDIR
)
4211 return do_rmdir(dfd
, getname(pathname
));
4212 return do_unlinkat(dfd
, getname(pathname
));
4215 SYSCALL_DEFINE1(unlink
, const char __user
*, pathname
)
4217 return do_unlinkat(AT_FDCWD
, getname(pathname
));
4221 * vfs_symlink - create symlink
4222 * @mnt_userns: user namespace of the mount the inode was found from
4223 * @dir: inode of @dentry
4224 * @dentry: pointer to dentry of the base directory
4225 * @oldname: name of the file to link to
4229 * If the inode has been found through an idmapped mount the user namespace of
4230 * the vfsmount must be passed through @mnt_userns. This function will then take
4231 * care to map the inode according to @mnt_userns before checking permissions.
4232 * On non-idmapped mounts or if permission checking is to be performed on the
4233 * raw inode simply passs init_user_ns.
4235 int vfs_symlink(struct user_namespace
*mnt_userns
, struct inode
*dir
,
4236 struct dentry
*dentry
, const char *oldname
)
4238 int error
= may_create(mnt_userns
, dir
, dentry
);
4243 if (!dir
->i_op
->symlink
)
4246 error
= security_inode_symlink(dir
, dentry
, oldname
);
4250 error
= dir
->i_op
->symlink(mnt_userns
, dir
, dentry
, oldname
);
4252 fsnotify_create(dir
, dentry
);
4255 EXPORT_SYMBOL(vfs_symlink
);
4257 int do_symlinkat(struct filename
*from
, int newdfd
, struct filename
*to
)
4260 struct dentry
*dentry
;
4262 unsigned int lookup_flags
= 0;
4265 error
= PTR_ERR(from
);
4269 dentry
= filename_create(newdfd
, to
, &path
, lookup_flags
);
4270 error
= PTR_ERR(dentry
);
4274 error
= security_path_symlink(&path
, dentry
, from
->name
);
4276 struct user_namespace
*mnt_userns
;
4278 mnt_userns
= mnt_user_ns(path
.mnt
);
4279 error
= vfs_symlink(mnt_userns
, path
.dentry
->d_inode
, dentry
,
4282 done_path_create(&path
, dentry
);
4283 if (retry_estale(error
, lookup_flags
)) {
4284 lookup_flags
|= LOOKUP_REVAL
;
4293 SYSCALL_DEFINE3(symlinkat
, const char __user
*, oldname
,
4294 int, newdfd
, const char __user
*, newname
)
4296 return do_symlinkat(getname(oldname
), newdfd
, getname(newname
));
4299 SYSCALL_DEFINE2(symlink
, const char __user
*, oldname
, const char __user
*, newname
)
4301 return do_symlinkat(getname(oldname
), AT_FDCWD
, getname(newname
));
4305 * vfs_link - create a new link
4306 * @old_dentry: object to be linked
4307 * @mnt_userns: the user namespace of the mount
4309 * @new_dentry: where to create the new link
4310 * @delegated_inode: returns inode needing a delegation break
4312 * The caller must hold dir->i_mutex
4314 * If vfs_link discovers a delegation on the to-be-linked file in need
4315 * of breaking, it will return -EWOULDBLOCK and return a reference to the
4316 * inode in delegated_inode. The caller should then break the delegation
4317 * and retry. Because breaking a delegation may take a long time, the
4318 * caller should drop the i_mutex before doing so.
4320 * Alternatively, a caller may pass NULL for delegated_inode. This may
4321 * be appropriate for callers that expect the underlying filesystem not
4322 * to be NFS exported.
4324 * If the inode has been found through an idmapped mount the user namespace of
4325 * the vfsmount must be passed through @mnt_userns. This function will then take
4326 * care to map the inode according to @mnt_userns before checking permissions.
4327 * On non-idmapped mounts or if permission checking is to be performed on the
4328 * raw inode simply passs init_user_ns.
4330 int vfs_link(struct dentry
*old_dentry
, struct user_namespace
*mnt_userns
,
4331 struct inode
*dir
, struct dentry
*new_dentry
,
4332 struct inode
**delegated_inode
)
4334 struct inode
*inode
= old_dentry
->d_inode
;
4335 unsigned max_links
= dir
->i_sb
->s_max_links
;
4341 error
= may_create(mnt_userns
, dir
, new_dentry
);
4345 if (dir
->i_sb
!= inode
->i_sb
)
4349 * A link to an append-only or immutable file cannot be created.
4351 if (IS_APPEND(inode
) || IS_IMMUTABLE(inode
))
4354 * Updating the link count will likely cause i_uid and i_gid to
4355 * be writen back improperly if their true value is unknown to
4358 if (HAS_UNMAPPED_ID(mnt_userns
, inode
))
4360 if (!dir
->i_op
->link
)
4362 if (S_ISDIR(inode
->i_mode
))
4365 error
= security_inode_link(old_dentry
, dir
, new_dentry
);
4370 /* Make sure we don't allow creating hardlink to an unlinked file */
4371 if (inode
->i_nlink
== 0 && !(inode
->i_state
& I_LINKABLE
))
4373 else if (max_links
&& inode
->i_nlink
>= max_links
)
4376 error
= try_break_deleg(inode
, delegated_inode
);
4378 error
= dir
->i_op
->link(old_dentry
, dir
, new_dentry
);
4381 if (!error
&& (inode
->i_state
& I_LINKABLE
)) {
4382 spin_lock(&inode
->i_lock
);
4383 inode
->i_state
&= ~I_LINKABLE
;
4384 spin_unlock(&inode
->i_lock
);
4386 inode_unlock(inode
);
4388 fsnotify_link(dir
, inode
, new_dentry
);
4391 EXPORT_SYMBOL(vfs_link
);
4394 * Hardlinks are often used in delicate situations. We avoid
4395 * security-related surprises by not following symlinks on the
4398 * We don't follow them on the oldname either to be compatible
4399 * with linux 2.0, and to avoid hard-linking to directories
4400 * and other special files. --ADM
4402 int do_linkat(int olddfd
, struct filename
*old
, int newdfd
,
4403 struct filename
*new, int flags
)
4405 struct user_namespace
*mnt_userns
;
4406 struct dentry
*new_dentry
;
4407 struct path old_path
, new_path
;
4408 struct inode
*delegated_inode
= NULL
;
4412 if ((flags
& ~(AT_SYMLINK_FOLLOW
| AT_EMPTY_PATH
)) != 0) {
4417 * To use null names we require CAP_DAC_READ_SEARCH
4418 * This ensures that not everyone will be able to create
4419 * handlink using the passed filedescriptor.
4421 if (flags
& AT_EMPTY_PATH
&& !capable(CAP_DAC_READ_SEARCH
)) {
4426 if (flags
& AT_SYMLINK_FOLLOW
)
4427 how
|= LOOKUP_FOLLOW
;
4429 error
= filename_lookup(olddfd
, old
, how
, &old_path
, NULL
);
4433 new_dentry
= filename_create(newdfd
, new, &new_path
,
4434 (how
& LOOKUP_REVAL
));
4435 error
= PTR_ERR(new_dentry
);
4436 if (IS_ERR(new_dentry
))
4440 if (old_path
.mnt
!= new_path
.mnt
)
4442 mnt_userns
= mnt_user_ns(new_path
.mnt
);
4443 error
= may_linkat(mnt_userns
, &old_path
);
4444 if (unlikely(error
))
4446 error
= security_path_link(old_path
.dentry
, &new_path
, new_dentry
);
4449 error
= vfs_link(old_path
.dentry
, mnt_userns
, new_path
.dentry
->d_inode
,
4450 new_dentry
, &delegated_inode
);
4452 done_path_create(&new_path
, new_dentry
);
4453 if (delegated_inode
) {
4454 error
= break_deleg_wait(&delegated_inode
);
4456 path_put(&old_path
);
4460 if (retry_estale(error
, how
)) {
4461 path_put(&old_path
);
4462 how
|= LOOKUP_REVAL
;
4466 path_put(&old_path
);
4474 SYSCALL_DEFINE5(linkat
, int, olddfd
, const char __user
*, oldname
,
4475 int, newdfd
, const char __user
*, newname
, int, flags
)
4477 return do_linkat(olddfd
, getname_uflags(oldname
, flags
),
4478 newdfd
, getname(newname
), flags
);
4481 SYSCALL_DEFINE2(link
, const char __user
*, oldname
, const char __user
*, newname
)
4483 return do_linkat(AT_FDCWD
, getname(oldname
), AT_FDCWD
, getname(newname
), 0);
4487 * vfs_rename - rename a filesystem object
4488 * @rd: pointer to &struct renamedata info
4490 * The caller must hold multiple mutexes--see lock_rename()).
4492 * If vfs_rename discovers a delegation in need of breaking at either
4493 * the source or destination, it will return -EWOULDBLOCK and return a
4494 * reference to the inode in delegated_inode. The caller should then
4495 * break the delegation and retry. Because breaking a delegation may
4496 * take a long time, the caller should drop all locks before doing
4499 * Alternatively, a caller may pass NULL for delegated_inode. This may
4500 * be appropriate for callers that expect the underlying filesystem not
4501 * to be NFS exported.
4503 * The worst of all namespace operations - renaming directory. "Perverted"
4504 * doesn't even start to describe it. Somebody in UCB had a heck of a trip...
4507 * a) we can get into loop creation.
4508 * b) race potential - two innocent renames can create a loop together.
4509 * That's where 4.4 screws up. Current fix: serialization on
4510 * sb->s_vfs_rename_mutex. We might be more accurate, but that's another
4512 * c) we have to lock _four_ objects - parents and victim (if it exists),
4513 * and source (if it is not a directory).
4514 * And that - after we got ->i_mutex on parents (until then we don't know
4515 * whether the target exists). Solution: try to be smart with locking
4516 * order for inodes. We rely on the fact that tree topology may change
4517 * only under ->s_vfs_rename_mutex _and_ that parent of the object we
4518 * move will be locked. Thus we can rank directories by the tree
4519 * (ancestors first) and rank all non-directories after them.
4520 * That works since everybody except rename does "lock parent, lookup,
4521 * lock child" and rename is under ->s_vfs_rename_mutex.
4522 * HOWEVER, it relies on the assumption that any object with ->lookup()
4523 * has no more than 1 dentry. If "hybrid" objects will ever appear,
4524 * we'd better make sure that there's no link(2) for them.
4525 * d) conversion from fhandle to dentry may come in the wrong moment - when
4526 * we are removing the target. Solution: we will have to grab ->i_mutex
4527 * in the fhandle_to_dentry code. [FIXME - current nfsfh.c relies on
4528 * ->i_mutex on parents, which works but leads to some truly excessive
4531 int vfs_rename(struct renamedata
*rd
)
4534 struct inode
*old_dir
= rd
->old_dir
, *new_dir
= rd
->new_dir
;
4535 struct dentry
*old_dentry
= rd
->old_dentry
;
4536 struct dentry
*new_dentry
= rd
->new_dentry
;
4537 struct inode
**delegated_inode
= rd
->delegated_inode
;
4538 unsigned int flags
= rd
->flags
;
4539 bool is_dir
= d_is_dir(old_dentry
);
4540 struct inode
*source
= old_dentry
->d_inode
;
4541 struct inode
*target
= new_dentry
->d_inode
;
4542 bool new_is_dir
= false;
4543 unsigned max_links
= new_dir
->i_sb
->s_max_links
;
4544 struct name_snapshot old_name
;
4546 if (source
== target
)
4549 error
= may_delete(rd
->old_mnt_userns
, old_dir
, old_dentry
, is_dir
);
4554 error
= may_create(rd
->new_mnt_userns
, new_dir
, new_dentry
);
4556 new_is_dir
= d_is_dir(new_dentry
);
4558 if (!(flags
& RENAME_EXCHANGE
))
4559 error
= may_delete(rd
->new_mnt_userns
, new_dir
,
4560 new_dentry
, is_dir
);
4562 error
= may_delete(rd
->new_mnt_userns
, new_dir
,
4563 new_dentry
, new_is_dir
);
4568 if (!old_dir
->i_op
->rename
)
4572 * If we are going to change the parent - check write permissions,
4573 * we'll need to flip '..'.
4575 if (new_dir
!= old_dir
) {
4577 error
= inode_permission(rd
->old_mnt_userns
, source
,
4582 if ((flags
& RENAME_EXCHANGE
) && new_is_dir
) {
4583 error
= inode_permission(rd
->new_mnt_userns
, target
,
4590 error
= security_inode_rename(old_dir
, old_dentry
, new_dir
, new_dentry
,
4595 take_dentry_name_snapshot(&old_name
, old_dentry
);
4597 if (!is_dir
|| (flags
& RENAME_EXCHANGE
))
4598 lock_two_nondirectories(source
, target
);
4603 if (IS_SWAPFILE(source
) || (target
&& IS_SWAPFILE(target
)))
4607 if (is_local_mountpoint(old_dentry
) || is_local_mountpoint(new_dentry
))
4610 if (max_links
&& new_dir
!= old_dir
) {
4612 if (is_dir
&& !new_is_dir
&& new_dir
->i_nlink
>= max_links
)
4614 if ((flags
& RENAME_EXCHANGE
) && !is_dir
&& new_is_dir
&&
4615 old_dir
->i_nlink
>= max_links
)
4619 error
= try_break_deleg(source
, delegated_inode
);
4623 if (target
&& !new_is_dir
) {
4624 error
= try_break_deleg(target
, delegated_inode
);
4628 error
= old_dir
->i_op
->rename(rd
->new_mnt_userns
, old_dir
, old_dentry
,
4629 new_dir
, new_dentry
, flags
);
4633 if (!(flags
& RENAME_EXCHANGE
) && target
) {
4635 shrink_dcache_parent(new_dentry
);
4636 target
->i_flags
|= S_DEAD
;
4638 dont_mount(new_dentry
);
4639 detach_mounts(new_dentry
);
4641 if (!(old_dir
->i_sb
->s_type
->fs_flags
& FS_RENAME_DOES_D_MOVE
)) {
4642 if (!(flags
& RENAME_EXCHANGE
))
4643 d_move(old_dentry
, new_dentry
);
4645 d_exchange(old_dentry
, new_dentry
);
4648 if (!is_dir
|| (flags
& RENAME_EXCHANGE
))
4649 unlock_two_nondirectories(source
, target
);
4651 inode_unlock(target
);
4654 fsnotify_move(old_dir
, new_dir
, &old_name
.name
, is_dir
,
4655 !(flags
& RENAME_EXCHANGE
) ? target
: NULL
, old_dentry
);
4656 if (flags
& RENAME_EXCHANGE
) {
4657 fsnotify_move(new_dir
, old_dir
, &old_dentry
->d_name
,
4658 new_is_dir
, NULL
, new_dentry
);
4661 release_dentry_name_snapshot(&old_name
);
4665 EXPORT_SYMBOL(vfs_rename
);
4667 int do_renameat2(int olddfd
, struct filename
*from
, int newdfd
,
4668 struct filename
*to
, unsigned int flags
)
4670 struct renamedata rd
;
4671 struct dentry
*old_dentry
, *new_dentry
;
4672 struct dentry
*trap
;
4673 struct path old_path
, new_path
;
4674 struct qstr old_last
, new_last
;
4675 int old_type
, new_type
;
4676 struct inode
*delegated_inode
= NULL
;
4677 unsigned int lookup_flags
= 0, target_flags
= LOOKUP_RENAME_TARGET
;
4678 bool should_retry
= false;
4679 int error
= -EINVAL
;
4681 if (flags
& ~(RENAME_NOREPLACE
| RENAME_EXCHANGE
| RENAME_WHITEOUT
))
4684 if ((flags
& (RENAME_NOREPLACE
| RENAME_WHITEOUT
)) &&
4685 (flags
& RENAME_EXCHANGE
))
4688 if (flags
& RENAME_EXCHANGE
)
4692 error
= filename_parentat(olddfd
, from
, lookup_flags
, &old_path
,
4693 &old_last
, &old_type
);
4697 error
= filename_parentat(newdfd
, to
, lookup_flags
, &new_path
, &new_last
,
4703 if (old_path
.mnt
!= new_path
.mnt
)
4707 if (old_type
!= LAST_NORM
)
4710 if (flags
& RENAME_NOREPLACE
)
4712 if (new_type
!= LAST_NORM
)
4715 error
= mnt_want_write(old_path
.mnt
);
4720 trap
= lock_rename(new_path
.dentry
, old_path
.dentry
);
4722 old_dentry
= __lookup_hash(&old_last
, old_path
.dentry
, lookup_flags
);
4723 error
= PTR_ERR(old_dentry
);
4724 if (IS_ERR(old_dentry
))
4726 /* source must exist */
4728 if (d_is_negative(old_dentry
))
4730 new_dentry
= __lookup_hash(&new_last
, new_path
.dentry
, lookup_flags
| target_flags
);
4731 error
= PTR_ERR(new_dentry
);
4732 if (IS_ERR(new_dentry
))
4735 if ((flags
& RENAME_NOREPLACE
) && d_is_positive(new_dentry
))
4737 if (flags
& RENAME_EXCHANGE
) {
4739 if (d_is_negative(new_dentry
))
4742 if (!d_is_dir(new_dentry
)) {
4744 if (new_last
.name
[new_last
.len
])
4748 /* unless the source is a directory trailing slashes give -ENOTDIR */
4749 if (!d_is_dir(old_dentry
)) {
4751 if (old_last
.name
[old_last
.len
])
4753 if (!(flags
& RENAME_EXCHANGE
) && new_last
.name
[new_last
.len
])
4756 /* source should not be ancestor of target */
4758 if (old_dentry
== trap
)
4760 /* target should not be an ancestor of source */
4761 if (!(flags
& RENAME_EXCHANGE
))
4763 if (new_dentry
== trap
)
4766 error
= security_path_rename(&old_path
, old_dentry
,
4767 &new_path
, new_dentry
, flags
);
4771 rd
.old_dir
= old_path
.dentry
->d_inode
;
4772 rd
.old_dentry
= old_dentry
;
4773 rd
.old_mnt_userns
= mnt_user_ns(old_path
.mnt
);
4774 rd
.new_dir
= new_path
.dentry
->d_inode
;
4775 rd
.new_dentry
= new_dentry
;
4776 rd
.new_mnt_userns
= mnt_user_ns(new_path
.mnt
);
4777 rd
.delegated_inode
= &delegated_inode
;
4779 error
= vfs_rename(&rd
);
4785 unlock_rename(new_path
.dentry
, old_path
.dentry
);
4786 if (delegated_inode
) {
4787 error
= break_deleg_wait(&delegated_inode
);
4791 mnt_drop_write(old_path
.mnt
);
4793 if (retry_estale(error
, lookup_flags
))
4794 should_retry
= true;
4795 path_put(&new_path
);
4797 path_put(&old_path
);
4799 should_retry
= false;
4800 lookup_flags
|= LOOKUP_REVAL
;
4809 SYSCALL_DEFINE5(renameat2
, int, olddfd
, const char __user
*, oldname
,
4810 int, newdfd
, const char __user
*, newname
, unsigned int, flags
)
4812 return do_renameat2(olddfd
, getname(oldname
), newdfd
, getname(newname
),
4816 SYSCALL_DEFINE4(renameat
, int, olddfd
, const char __user
*, oldname
,
4817 int, newdfd
, const char __user
*, newname
)
4819 return do_renameat2(olddfd
, getname(oldname
), newdfd
, getname(newname
),
4823 SYSCALL_DEFINE2(rename
, const char __user
*, oldname
, const char __user
*, newname
)
4825 return do_renameat2(AT_FDCWD
, getname(oldname
), AT_FDCWD
,
4826 getname(newname
), 0);
4829 int readlink_copy(char __user
*buffer
, int buflen
, const char *link
)
4831 int len
= PTR_ERR(link
);
4836 if (len
> (unsigned) buflen
)
4838 if (copy_to_user(buffer
, link
, len
))
4845 * vfs_readlink - copy symlink body into userspace buffer
4846 * @dentry: dentry on which to get symbolic link
4847 * @buffer: user memory pointer
4848 * @buflen: size of buffer
4850 * Does not touch atime. That's up to the caller if necessary
4852 * Does not call security hook.
4854 int vfs_readlink(struct dentry
*dentry
, char __user
*buffer
, int buflen
)
4856 struct inode
*inode
= d_inode(dentry
);
4857 DEFINE_DELAYED_CALL(done
);
4861 if (unlikely(!(inode
->i_opflags
& IOP_DEFAULT_READLINK
))) {
4862 if (unlikely(inode
->i_op
->readlink
))
4863 return inode
->i_op
->readlink(dentry
, buffer
, buflen
);
4865 if (!d_is_symlink(dentry
))
4868 spin_lock(&inode
->i_lock
);
4869 inode
->i_opflags
|= IOP_DEFAULT_READLINK
;
4870 spin_unlock(&inode
->i_lock
);
4873 link
= READ_ONCE(inode
->i_link
);
4875 link
= inode
->i_op
->get_link(dentry
, inode
, &done
);
4877 return PTR_ERR(link
);
4879 res
= readlink_copy(buffer
, buflen
, link
);
4880 do_delayed_call(&done
);
4883 EXPORT_SYMBOL(vfs_readlink
);
4886 * vfs_get_link - get symlink body
4887 * @dentry: dentry on which to get symbolic link
4888 * @done: caller needs to free returned data with this
4890 * Calls security hook and i_op->get_link() on the supplied inode.
4892 * It does not touch atime. That's up to the caller if necessary.
4894 * Does not work on "special" symlinks like /proc/$$/fd/N
4896 const char *vfs_get_link(struct dentry
*dentry
, struct delayed_call
*done
)
4898 const char *res
= ERR_PTR(-EINVAL
);
4899 struct inode
*inode
= d_inode(dentry
);
4901 if (d_is_symlink(dentry
)) {
4902 res
= ERR_PTR(security_inode_readlink(dentry
));
4904 res
= inode
->i_op
->get_link(dentry
, inode
, done
);
4908 EXPORT_SYMBOL(vfs_get_link
);
4910 /* get the link contents into pagecache */
4911 const char *page_get_link(struct dentry
*dentry
, struct inode
*inode
,
4912 struct delayed_call
*callback
)
4916 struct address_space
*mapping
= inode
->i_mapping
;
4919 page
= find_get_page(mapping
, 0);
4921 return ERR_PTR(-ECHILD
);
4922 if (!PageUptodate(page
)) {
4924 return ERR_PTR(-ECHILD
);
4927 page
= read_mapping_page(mapping
, 0, NULL
);
4931 set_delayed_call(callback
, page_put_link
, page
);
4932 BUG_ON(mapping_gfp_mask(mapping
) & __GFP_HIGHMEM
);
4933 kaddr
= page_address(page
);
4934 nd_terminate_link(kaddr
, inode
->i_size
, PAGE_SIZE
- 1);
4938 EXPORT_SYMBOL(page_get_link
);
4940 void page_put_link(void *arg
)
4944 EXPORT_SYMBOL(page_put_link
);
4946 int page_readlink(struct dentry
*dentry
, char __user
*buffer
, int buflen
)
4948 DEFINE_DELAYED_CALL(done
);
4949 int res
= readlink_copy(buffer
, buflen
,
4950 page_get_link(dentry
, d_inode(dentry
),
4952 do_delayed_call(&done
);
4955 EXPORT_SYMBOL(page_readlink
);
4958 * The nofs argument instructs pagecache_write_begin to pass AOP_FLAG_NOFS
4960 int __page_symlink(struct inode
*inode
, const char *symname
, int len
, int nofs
)
4962 struct address_space
*mapping
= inode
->i_mapping
;
4966 unsigned int flags
= 0;
4968 flags
|= AOP_FLAG_NOFS
;
4971 err
= pagecache_write_begin(NULL
, mapping
, 0, len
-1,
4972 flags
, &page
, &fsdata
);
4976 memcpy(page_address(page
), symname
, len
-1);
4978 err
= pagecache_write_end(NULL
, mapping
, 0, len
-1, len
-1,
4985 mark_inode_dirty(inode
);
4990 EXPORT_SYMBOL(__page_symlink
);
4992 int page_symlink(struct inode
*inode
, const char *symname
, int len
)
4994 return __page_symlink(inode
, symname
, len
,
4995 !mapping_gfp_constraint(inode
->i_mapping
, __GFP_FS
));
4997 EXPORT_SYMBOL(page_symlink
);
4999 const struct inode_operations page_symlink_inode_operations
= {
5000 .get_link
= page_get_link
,
5002 EXPORT_SYMBOL(page_symlink_inode_operations
);