unsigned int hash)
{
hash += (unsigned long) parent / L1_CACHE_BYTES;
- hash = hash + (hash >> d_hash_shift);
- return dentry_hashtable + (hash & d_hash_mask);
+ return dentry_hashtable + hash_32(hash, d_hash_shift);
}
/* Statistics gathering. */
return dentry_string_cmp(cs, ct, tcount);
}
+struct external_name {
+ union {
+ atomic_t count;
+ struct rcu_head head;
+ } u;
+ unsigned char name[];
+};
+
+static inline struct external_name *external_name(struct dentry *dentry)
+{
+ return container_of(dentry->d_name.name, struct external_name, name[0]);
+}
+
static void __d_free(struct rcu_head *head)
{
struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
WARN_ON(!hlist_unhashed(&dentry->d_alias));
- if (dname_external(dentry))
- kfree(dentry->d_name.name);
+ kmem_cache_free(dentry_cache, dentry);
+}
+
+static void __d_free_external(struct rcu_head *head)
+{
+ struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
+ WARN_ON(!hlist_unhashed(&dentry->d_alias));
+ kfree(external_name(dentry));
kmem_cache_free(dentry_cache, dentry);
}
static void dentry_free(struct dentry *dentry)
{
+ if (unlikely(dname_external(dentry))) {
+ struct external_name *p = external_name(dentry);
+ if (likely(atomic_dec_and_test(&p->u.count))) {
+ call_rcu(&dentry->d_u.d_rcu, __d_free_external);
+ return;
+ }
+ }
/* if dentry was never visible to RCU, immediate free is OK */
if (!(dentry->d_flags & DCACHE_RCUACCESS))
__d_free(&dentry->d_u.d_rcu);
}
EXPORT_SYMBOL(dput);
-/**
- * d_invalidate - invalidate a dentry
- * @dentry: dentry to invalidate
- *
- * Try to invalidate the dentry if it turns out to be
- * possible. If there are other dentries that can be
- * reached through this one we can't delete it and we
- * return -EBUSY. On success we return 0.
- *
- * no dcache lock.
- */
-
-int d_invalidate(struct dentry * dentry)
-{
- /*
- * If it's already been dropped, return OK.
- */
- spin_lock(&dentry->d_lock);
- if (d_unhashed(dentry)) {
- spin_unlock(&dentry->d_lock);
- return 0;
- }
- /*
- * Check whether to do a partial shrink_dcache
- * to get rid of unused child entries.
- */
- if (!list_empty(&dentry->d_subdirs)) {
- spin_unlock(&dentry->d_lock);
- shrink_dcache_parent(dentry);
- spin_lock(&dentry->d_lock);
- }
-
- /*
- * Somebody else still using it?
- *
- * If it's a directory, we can't drop it
- * for fear of somebody re-populating it
- * with children (even though dropping it
- * would make it unreachable from the root,
- * we might still populate it if it was a
- * working directory or similar).
- * We also need to leave mountpoints alone,
- * directory or not.
- */
- if (dentry->d_lockref.count > 1 && dentry->d_inode) {
- if (S_ISDIR(dentry->d_inode->i_mode) || d_mountpoint(dentry)) {
- spin_unlock(&dentry->d_lock);
- return -EBUSY;
- }
- }
-
- __d_drop(dentry);
- spin_unlock(&dentry->d_lock);
- return 0;
-}
-EXPORT_SYMBOL(d_invalidate);
/* This must be called with d_lock held */
static inline void __dget_dlock(struct dentry *dentry)
* acquire the reference to alias and return it. Otherwise return NULL.
* Notice that if inode is a directory there can be only one alias and
* it can be unhashed only if it has no children, or if it is the root
- * of a filesystem.
+ * of a filesystem, or if the directory was renamed and d_revalidate
+ * was the first vfs operation to notice.
*
* If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
* any other hashed alias over that one.
* reachable (e.g. NFS can unhash a directory dentry and then the complete
* subtree can become unreachable).
*
- * Only one of check_submounts_and_drop() and d_set_mounted() must succeed. For
+ * Only one of d_invalidate() and d_set_mounted() must succeed. For
* this reason take rename_lock and d_lock on dentry and ancestors.
*/
int d_set_mounted(struct dentry *dentry)
int ret = -ENOENT;
write_seqlock(&rename_lock);
for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
- /* Need exclusion wrt. check_submounts_and_drop() */
+ /* Need exclusion wrt. d_invalidate() */
spin_lock(&p->d_lock);
if (unlikely(d_unhashed(p))) {
spin_unlock(&p->d_lock);
}
}
-static enum d_walk_ret check_and_collect(void *_data, struct dentry *dentry)
+struct detach_data {
+ struct select_data select;
+ struct dentry *mountpoint;
+};
+static enum d_walk_ret detach_and_collect(void *_data, struct dentry *dentry)
{
- struct select_data *data = _data;
+ struct detach_data *data = _data;
if (d_mountpoint(dentry)) {
- data->found = -EBUSY;
+ __dget_dlock(dentry);
+ data->mountpoint = dentry;
return D_WALK_QUIT;
}
- return select_collect(_data, dentry);
+ return select_collect(&data->select, dentry);
}
static void check_and_drop(void *_data)
{
- struct select_data *data = _data;
+ struct detach_data *data = _data;
- if (d_mountpoint(data->start))
- data->found = -EBUSY;
- if (!data->found)
- __d_drop(data->start);
+ if (!data->mountpoint && !data->select.found)
+ __d_drop(data->select.start);
}
/**
- * check_submounts_and_drop - prune dcache, check for submounts and drop
+ * d_invalidate - detach submounts, prune dcache, and drop
+ * @dentry: dentry to invalidate (aka detach, prune and drop)
+ *
+ * Try to invalidate the dentry if it turns out to be
+ * possible. If there are reasons not to delete it
+ * return -EBUSY. On success return 0.
*
- * All done as a single atomic operation relative to has_unlinked_ancestor().
- * Returns 0 if successfully unhashed @parent. If there were submounts then
- * return -EBUSY.
+ * no dcache lock.
*
- * @dentry: dentry to prune and drop
+ * The final d_drop is done as an atomic operation relative to
+ * rename_lock ensuring there are no races with d_set_mounted. This
+ * ensures there are no unhashed dentries on the path to a mountpoint.
*/
-int check_submounts_and_drop(struct dentry *dentry)
+int d_invalidate(struct dentry *dentry)
{
int ret = 0;
+ /*
+ * If it's already been dropped, return OK.
+ */
+ spin_lock(&dentry->d_lock);
+ if (d_unhashed(dentry)) {
+ spin_unlock(&dentry->d_lock);
+ return 0;
+ }
+ spin_unlock(&dentry->d_lock);
+
/* Negative dentries can be dropped without further checks */
if (!dentry->d_inode) {
d_drop(dentry);
}
for (;;) {
- struct select_data data;
+ struct detach_data data;
- INIT_LIST_HEAD(&data.dispose);
- data.start = dentry;
- data.found = 0;
+ data.mountpoint = NULL;
+ INIT_LIST_HEAD(&data.select.dispose);
+ data.select.start = dentry;
+ data.select.found = 0;
+
+ d_walk(dentry, &data, detach_and_collect, check_and_drop);
- d_walk(dentry, &data, check_and_collect, check_and_drop);
- ret = data.found;
+ if (data.select.found)
+ shrink_dentry_list(&data.select.dispose);
- if (!list_empty(&data.dispose))
- shrink_dentry_list(&data.dispose);
+ if (data.mountpoint) {
+ detach_mounts(data.mountpoint);
+ dput(data.mountpoint);
+ }
- if (ret <= 0)
+ if (!data.mountpoint && !data.select.found)
break;
cond_resched();
out:
return ret;
}
-EXPORT_SYMBOL(check_submounts_and_drop);
+EXPORT_SYMBOL(d_invalidate);
/**
* __d_alloc - allocate a dcache entry
*/
dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
if (name->len > DNAME_INLINE_LEN-1) {
- dname = kmalloc(name->len + 1, GFP_KERNEL);
- if (!dname) {
+ size_t size = offsetof(struct external_name, name[1]);
+ struct external_name *p = kmalloc(size + name->len, GFP_KERNEL);
+ if (!p) {
kmem_cache_free(dentry_cache, dentry);
return NULL;
}
+ atomic_set(&p->u.count, 1);
+ dname = p->name;
} else {
dname = dentry->d_iname;
}
}
EXPORT_SYMBOL(dentry_update_name_case);
-static void switch_names(struct dentry *dentry, struct dentry *target)
+static void swap_names(struct dentry *dentry, struct dentry *target)
{
- if (dname_external(target)) {
- if (dname_external(dentry)) {
+ if (unlikely(dname_external(target))) {
+ if (unlikely(dname_external(dentry))) {
/*
* Both external: swap the pointers
*/
target->d_name.name = target->d_iname;
}
} else {
- if (dname_external(dentry)) {
+ if (unlikely(dname_external(dentry))) {
/*
* dentry:external, target:internal. Give dentry's
* storage to target and make dentry internal
}
}
}
- swap(dentry->d_name.len, target->d_name.len);
+ swap(dentry->d_name.hash_len, target->d_name.hash_len);
+}
+
+static void copy_name(struct dentry *dentry, struct dentry *target)
+{
+ struct external_name *old_name = NULL;
+ if (unlikely(dname_external(dentry)))
+ old_name = external_name(dentry);
+ if (unlikely(dname_external(target))) {
+ atomic_inc(&external_name(target)->u.count);
+ dentry->d_name = target->d_name;
+ } else {
+ memcpy(dentry->d_iname, target->d_name.name,
+ target->d_name.len + 1);
+ dentry->d_name.name = dentry->d_iname;
+ dentry->d_name.hash_len = target->d_name.hash_len;
+ }
+ if (old_name && likely(atomic_dec_and_test(&old_name->u.count)))
+ kfree_rcu(old_name, u.head);
}
static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target)
}
}
-static void dentry_unlock_parents_for_move(struct dentry *dentry,
- struct dentry *target)
+static void dentry_unlock_for_move(struct dentry *dentry, struct dentry *target)
{
if (target->d_parent != dentry->d_parent)
spin_unlock(&dentry->d_parent->d_lock);
if (target->d_parent != target)
spin_unlock(&target->d_parent->d_lock);
+ spin_unlock(&target->d_lock);
+ spin_unlock(&dentry->d_lock);
}
/*
* When switching names, the actual string doesn't strictly have to
* be preserved in the target - because we're dropping the target
* anyway. As such, we can just do a simple memcpy() to copy over
- * the new name before we switch.
- *
- * Note that we have to be a lot more careful about getting the hash
- * switched - we have to switch the hash value properly even if it
- * then no longer matches the actual (corrupted) string of the target.
- * The hash value has to match the hash queue that the dentry is on..
+ * the new name before we switch, unless we are going to rehash
+ * it. Note that if we *do* unhash the target, we are not allowed
+ * to rehash it without giving it a new name/hash key - whether
+ * we swap or overwrite the names here, resulting name won't match
+ * the reality in filesystem; it's only there for d_path() purposes.
+ * Note that all of this is happening under rename_lock, so the
+ * any hash lookup seeing it in the middle of manipulations will
+ * be discarded anyway. So we do not care what happens to the hash
+ * key in that case.
*/
/*
* __d_move - move a dentry
d_hash(dentry->d_parent, dentry->d_name.hash));
}
- list_del(&dentry->d_u.d_child);
- list_del(&target->d_u.d_child);
-
/* Switch the names.. */
- switch_names(dentry, target);
- swap(dentry->d_name.hash, target->d_name.hash);
+ if (exchange)
+ swap_names(dentry, target);
+ else
+ copy_name(dentry, target);
- /* ... and switch the parents */
+ /* ... and switch them in the tree */
if (IS_ROOT(dentry)) {
+ /* splicing a tree */
dentry->d_parent = target->d_parent;
target->d_parent = target;
- INIT_LIST_HEAD(&target->d_u.d_child);
+ list_del_init(&target->d_u.d_child);
+ list_move(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
} else {
+ /* swapping two dentries */
swap(dentry->d_parent, target->d_parent);
-
- /* And add them back to the (new) parent lists */
- list_add(&target->d_u.d_child, &target->d_parent->d_subdirs);
+ list_move(&target->d_u.d_child, &target->d_parent->d_subdirs);
+ list_move(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
+ if (exchange)
+ fsnotify_d_move(target);
+ fsnotify_d_move(dentry);
}
- list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
-
write_seqcount_end(&target->d_seq);
write_seqcount_end(&dentry->d_seq);
- dentry_unlock_parents_for_move(dentry, target);
- if (exchange)
- fsnotify_d_move(target);
- spin_unlock(&target->d_lock);
- fsnotify_d_move(dentry);
- spin_unlock(&dentry->d_lock);
+ dentry_unlock_for_move(dentry, target);
}
/*
goto out_err;
m2 = &alias->d_parent->d_inode->i_mutex;
out_unalias:
- if (likely(!d_mountpoint(alias))) {
- __d_move(alias, dentry, false);
- ret = alias;
- }
+ __d_move(alias, dentry, false);
+ ret = alias;
out_err:
spin_unlock(&inode->i_lock);
if (m2)
return ret;
}
-/*
- * Prepare an anonymous dentry for life in the superblock's dentry tree as a
- * named dentry in place of the dentry to be replaced.
- * returns with anon->d_lock held!
- */
-static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
-{
- struct dentry *dparent;
-
- dentry_lock_for_move(anon, dentry);
-
- write_seqcount_begin(&dentry->d_seq);
- write_seqcount_begin_nested(&anon->d_seq, DENTRY_D_LOCK_NESTED);
-
- dparent = dentry->d_parent;
-
- switch_names(dentry, anon);
- swap(dentry->d_name.hash, anon->d_name.hash);
-
- dentry->d_parent = dentry;
- list_del_init(&dentry->d_u.d_child);
- anon->d_parent = dparent;
- list_move(&anon->d_u.d_child, &dparent->d_subdirs);
-
- write_seqcount_end(&dentry->d_seq);
- write_seqcount_end(&anon->d_seq);
-
- dentry_unlock_parents_for_move(anon, dentry);
- spin_unlock(&dentry->d_lock);
-
- /* anon->d_lock still locked, returns locked */
-}
-
/**
* d_splice_alias - splice a disconnected dentry into the tree if one exists
* @inode: the inode which may have a disconnected dentry
return ERR_PTR(-EIO);
}
write_seqlock(&rename_lock);
- __d_materialise_dentry(dentry, new);
+ __d_move(new, dentry, false);
write_sequnlock(&rename_lock);
- __d_drop(new);
- _d_rehash(new);
- spin_unlock(&new->d_lock);
spin_unlock(&inode->i_lock);
security_d_instantiate(new, inode);
iput(inode);
} else if (IS_ROOT(alias)) {
/* Is this an anonymous mountpoint that we
* could splice into our tree? */
- __d_materialise_dentry(dentry, alias);
+ __d_move(alias, dentry, false);
write_sequnlock(&rename_lock);
- __d_drop(alias);
goto found;
} else {
/* Nope, but we must(!) avoid directory
actual = __d_instantiate_unique(dentry, inode);
if (!actual)
actual = dentry;
- else
- BUG_ON(!d_unhashed(actual));
- spin_lock(&actual->d_lock);
+ d_rehash(actual);
found:
- _d_rehash(actual);
- spin_unlock(&actual->d_lock);
spin_unlock(&inode->i_lock);
out_nolock:
if (actual == dentry) {
* the beginning of the name. The sequence number check at the caller will
* retry it again when a d_move() does happen. So any garbage in the buffer
* due to mismatched pointer and length will be discarded.
+ *
+ * Data dependency barrier is needed to make sure that we see that terminating
+ * NUL. Alpha strikes again, film at 11...
*/
static int prepend_name(char **buffer, int *buflen, struct qstr *name)
{
u32 dlen = ACCESS_ONCE(name->len);
char *p;
+ smp_read_barrier_depends();
+
*buflen -= dlen + 1;
if (*buflen < 0)
return -ENAMETOOLONG;
projects / mirror_ubuntu-artful-kernel.git / blobdiff