2 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
5 * This copyrighted material is made available to anyone wishing to use,
6 * modify, copy, or redistribute it subject to the terms and conditions
7 * of the GNU General Public License version 2.
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 #include <linux/sched.h>
13 #include <linux/slab.h>
14 #include <linux/spinlock.h>
15 #include <linux/buffer_head.h>
16 #include <linux/delay.h>
17 #include <linux/sort.h>
18 #include <linux/jhash.h>
19 #include <linux/kallsyms.h>
20 #include <linux/gfs2_ondisk.h>
21 #include <linux/list.h>
22 #include <linux/wait.h>
23 #include <linux/module.h>
24 #include <linux/uaccess.h>
25 #include <linux/seq_file.h>
26 #include <linux/debugfs.h>
27 #include <linux/kthread.h>
28 #include <linux/freezer.h>
29 #include <linux/workqueue.h>
30 #include <linux/jiffies.h>
31 #include <linux/rcupdate.h>
32 #include <linux/rculist_bl.h>
33 #include <linux/bit_spinlock.h>
34 #include <linux/percpu.h>
35 #include <linux/list_sort.h>
36 #include <linux/lockref.h>
37 #include <linux/rhashtable.h>
50 #define CREATE_TRACE_POINTS
51 #include "trace_gfs2.h"
53 struct gfs2_glock_iter
{
54 struct gfs2_sbd
*sdp
; /* incore superblock */
55 struct rhashtable_iter hti
; /* rhashtable iterator */
56 struct gfs2_glock
*gl
; /* current glock struct */
57 loff_t last_pos
; /* last position */
60 typedef void (*glock_examiner
) (struct gfs2_glock
* gl
);
62 static void do_xmote(struct gfs2_glock
*gl
, struct gfs2_holder
*gh
, unsigned int target
);
64 static struct dentry
*gfs2_root
;
65 static struct workqueue_struct
*glock_workqueue
;
66 struct workqueue_struct
*gfs2_delete_workqueue
;
67 static LIST_HEAD(lru_list
);
68 static atomic_t lru_count
= ATOMIC_INIT(0);
69 static DEFINE_SPINLOCK(lru_lock
);
71 #define GFS2_GL_HASH_SHIFT 15
72 #define GFS2_GL_HASH_SIZE BIT(GFS2_GL_HASH_SHIFT)
74 static struct rhashtable_params ht_parms
= {
75 .nelem_hint
= GFS2_GL_HASH_SIZE
* 3 / 4,
76 .key_len
= offsetofend(struct lm_lockname
, ln_type
),
77 .key_offset
= offsetof(struct gfs2_glock
, gl_name
),
78 .head_offset
= offsetof(struct gfs2_glock
, gl_node
),
81 static struct rhashtable gl_hash_table
;
83 static void gfs2_glock_dealloc(struct rcu_head
*rcu
)
85 struct gfs2_glock
*gl
= container_of(rcu
, struct gfs2_glock
, gl_rcu
);
87 if (gl
->gl_ops
->go_flags
& GLOF_ASPACE
) {
88 kmem_cache_free(gfs2_glock_aspace_cachep
, gl
);
90 kfree(gl
->gl_lksb
.sb_lvbptr
);
91 kmem_cache_free(gfs2_glock_cachep
, gl
);
95 void gfs2_glock_free(struct gfs2_glock
*gl
)
97 struct gfs2_sbd
*sdp
= gl
->gl_name
.ln_sbd
;
99 call_rcu(&gl
->gl_rcu
, gfs2_glock_dealloc
);
100 if (atomic_dec_and_test(&sdp
->sd_glock_disposal
))
101 wake_up(&sdp
->sd_glock_wait
);
105 * gfs2_glock_hold() - increment reference count on glock
106 * @gl: The glock to hold
110 static void gfs2_glock_hold(struct gfs2_glock
*gl
)
112 GLOCK_BUG_ON(gl
, __lockref_is_dead(&gl
->gl_lockref
));
113 lockref_get(&gl
->gl_lockref
);
117 * demote_ok - Check to see if it's ok to unlock a glock
120 * Returns: 1 if it's ok
123 static int demote_ok(const struct gfs2_glock
*gl
)
125 const struct gfs2_glock_operations
*glops
= gl
->gl_ops
;
127 if (gl
->gl_state
== LM_ST_UNLOCKED
)
129 if (!list_empty(&gl
->gl_holders
))
131 if (glops
->go_demote_ok
)
132 return glops
->go_demote_ok(gl
);
137 void gfs2_glock_add_to_lru(struct gfs2_glock
*gl
)
139 spin_lock(&lru_lock
);
141 if (!list_empty(&gl
->gl_lru
))
142 list_del_init(&gl
->gl_lru
);
144 atomic_inc(&lru_count
);
146 list_add_tail(&gl
->gl_lru
, &lru_list
);
147 set_bit(GLF_LRU
, &gl
->gl_flags
);
148 spin_unlock(&lru_lock
);
151 static void gfs2_glock_remove_from_lru(struct gfs2_glock
*gl
)
153 spin_lock(&lru_lock
);
154 if (!list_empty(&gl
->gl_lru
)) {
155 list_del_init(&gl
->gl_lru
);
156 atomic_dec(&lru_count
);
157 clear_bit(GLF_LRU
, &gl
->gl_flags
);
159 spin_unlock(&lru_lock
);
163 * Enqueue the glock on the work queue. Passes one glock reference on to the
166 static void __gfs2_glock_queue_work(struct gfs2_glock
*gl
, unsigned long delay
) {
167 if (!queue_delayed_work(glock_workqueue
, &gl
->gl_work
, delay
)) {
169 * We are holding the lockref spinlock, and the work was still
170 * queued above. The queued work (glock_work_func) takes that
171 * spinlock before dropping its glock reference(s), so it
172 * cannot have dropped them in the meantime.
174 GLOCK_BUG_ON(gl
, gl
->gl_lockref
.count
< 2);
175 gl
->gl_lockref
.count
--;
179 static void gfs2_glock_queue_work(struct gfs2_glock
*gl
, unsigned long delay
) {
180 spin_lock(&gl
->gl_lockref
.lock
);
181 __gfs2_glock_queue_work(gl
, delay
);
182 spin_unlock(&gl
->gl_lockref
.lock
);
185 static void __gfs2_glock_put(struct gfs2_glock
*gl
)
187 struct gfs2_sbd
*sdp
= gl
->gl_name
.ln_sbd
;
188 struct address_space
*mapping
= gfs2_glock2aspace(gl
);
190 lockref_mark_dead(&gl
->gl_lockref
);
192 gfs2_glock_remove_from_lru(gl
);
193 spin_unlock(&gl
->gl_lockref
.lock
);
194 rhashtable_remove_fast(&gl_hash_table
, &gl
->gl_node
, ht_parms
);
195 GLOCK_BUG_ON(gl
, !list_empty(&gl
->gl_holders
));
196 GLOCK_BUG_ON(gl
, mapping
&& mapping
->nrpages
);
197 trace_gfs2_glock_put(gl
);
198 sdp
->sd_lockstruct
.ls_ops
->lm_put_lock(gl
);
202 * gfs2_glock_put() - Decrement reference count on glock
203 * @gl: The glock to put
207 void gfs2_glock_put(struct gfs2_glock
*gl
)
209 if (lockref_put_or_lock(&gl
->gl_lockref
))
212 __gfs2_glock_put(gl
);
216 * may_grant - check if its ok to grant a new lock
218 * @gh: The lock request which we wish to grant
220 * Returns: true if its ok to grant the lock
223 static inline int may_grant(const struct gfs2_glock
*gl
, const struct gfs2_holder
*gh
)
225 const struct gfs2_holder
*gh_head
= list_entry(gl
->gl_holders
.next
, const struct gfs2_holder
, gh_list
);
226 if ((gh
->gh_state
== LM_ST_EXCLUSIVE
||
227 gh_head
->gh_state
== LM_ST_EXCLUSIVE
) && gh
!= gh_head
)
229 if (gl
->gl_state
== gh
->gh_state
)
231 if (gh
->gh_flags
& GL_EXACT
)
233 if (gl
->gl_state
== LM_ST_EXCLUSIVE
) {
234 if (gh
->gh_state
== LM_ST_SHARED
&& gh_head
->gh_state
== LM_ST_SHARED
)
236 if (gh
->gh_state
== LM_ST_DEFERRED
&& gh_head
->gh_state
== LM_ST_DEFERRED
)
239 if (gl
->gl_state
!= LM_ST_UNLOCKED
&& (gh
->gh_flags
& LM_FLAG_ANY
))
244 static void gfs2_holder_wake(struct gfs2_holder
*gh
)
246 clear_bit(HIF_WAIT
, &gh
->gh_iflags
);
247 smp_mb__after_atomic();
248 wake_up_bit(&gh
->gh_iflags
, HIF_WAIT
);
252 * do_error - Something unexpected has happened during a lock request
256 static void do_error(struct gfs2_glock
*gl
, const int ret
)
258 struct gfs2_holder
*gh
, *tmp
;
260 list_for_each_entry_safe(gh
, tmp
, &gl
->gl_holders
, gh_list
) {
261 if (test_bit(HIF_HOLDER
, &gh
->gh_iflags
))
263 if (ret
& LM_OUT_ERROR
)
265 else if (gh
->gh_flags
& (LM_FLAG_TRY
| LM_FLAG_TRY_1CB
))
266 gh
->gh_error
= GLR_TRYFAILED
;
269 list_del_init(&gh
->gh_list
);
270 trace_gfs2_glock_queue(gh
, 0);
271 gfs2_holder_wake(gh
);
276 * do_promote - promote as many requests as possible on the current queue
279 * Returns: 1 if there is a blocked holder at the head of the list, or 2
280 * if a type specific operation is underway.
283 static int do_promote(struct gfs2_glock
*gl
)
284 __releases(&gl
->gl_lockref
.lock
)
285 __acquires(&gl
->gl_lockref
.lock
)
287 const struct gfs2_glock_operations
*glops
= gl
->gl_ops
;
288 struct gfs2_holder
*gh
, *tmp
;
292 list_for_each_entry_safe(gh
, tmp
, &gl
->gl_holders
, gh_list
) {
293 if (test_bit(HIF_HOLDER
, &gh
->gh_iflags
))
295 if (may_grant(gl
, gh
)) {
296 if (gh
->gh_list
.prev
== &gl
->gl_holders
&&
298 spin_unlock(&gl
->gl_lockref
.lock
);
299 /* FIXME: eliminate this eventually */
300 ret
= glops
->go_lock(gh
);
301 spin_lock(&gl
->gl_lockref
.lock
);
306 list_del_init(&gh
->gh_list
);
307 trace_gfs2_glock_queue(gh
, 0);
308 gfs2_holder_wake(gh
);
311 set_bit(HIF_HOLDER
, &gh
->gh_iflags
);
312 trace_gfs2_promote(gh
, 1);
313 gfs2_holder_wake(gh
);
316 set_bit(HIF_HOLDER
, &gh
->gh_iflags
);
317 trace_gfs2_promote(gh
, 0);
318 gfs2_holder_wake(gh
);
321 if (gh
->gh_list
.prev
== &gl
->gl_holders
)
330 * find_first_waiter - find the first gh that's waiting for the glock
334 static inline struct gfs2_holder
*find_first_waiter(const struct gfs2_glock
*gl
)
336 struct gfs2_holder
*gh
;
338 list_for_each_entry(gh
, &gl
->gl_holders
, gh_list
) {
339 if (!test_bit(HIF_HOLDER
, &gh
->gh_iflags
))
346 * state_change - record that the glock is now in a different state
348 * @new_state the new state
352 static void state_change(struct gfs2_glock
*gl
, unsigned int new_state
)
356 held1
= (gl
->gl_state
!= LM_ST_UNLOCKED
);
357 held2
= (new_state
!= LM_ST_UNLOCKED
);
359 if (held1
!= held2
) {
360 GLOCK_BUG_ON(gl
, __lockref_is_dead(&gl
->gl_lockref
));
362 gl
->gl_lockref
.count
++;
364 gl
->gl_lockref
.count
--;
366 if (held1
&& held2
&& list_empty(&gl
->gl_holders
))
367 clear_bit(GLF_QUEUED
, &gl
->gl_flags
);
369 if (new_state
!= gl
->gl_target
)
370 /* shorten our minimum hold time */
371 gl
->gl_hold_time
= max(gl
->gl_hold_time
- GL_GLOCK_HOLD_DECR
,
373 gl
->gl_state
= new_state
;
374 gl
->gl_tchange
= jiffies
;
377 static void gfs2_demote_wake(struct gfs2_glock
*gl
)
379 gl
->gl_demote_state
= LM_ST_EXCLUSIVE
;
380 clear_bit(GLF_DEMOTE
, &gl
->gl_flags
);
381 smp_mb__after_atomic();
382 wake_up_bit(&gl
->gl_flags
, GLF_DEMOTE
);
386 * finish_xmote - The DLM has replied to one of our lock requests
388 * @ret: The status from the DLM
392 static void finish_xmote(struct gfs2_glock
*gl
, unsigned int ret
)
394 const struct gfs2_glock_operations
*glops
= gl
->gl_ops
;
395 struct gfs2_holder
*gh
;
396 unsigned state
= ret
& LM_OUT_ST_MASK
;
399 spin_lock(&gl
->gl_lockref
.lock
);
400 trace_gfs2_glock_state_change(gl
, state
);
401 state_change(gl
, state
);
402 gh
= find_first_waiter(gl
);
404 /* Demote to UN request arrived during demote to SH or DF */
405 if (test_bit(GLF_DEMOTE_IN_PROGRESS
, &gl
->gl_flags
) &&
406 state
!= LM_ST_UNLOCKED
&& gl
->gl_demote_state
== LM_ST_UNLOCKED
)
407 gl
->gl_target
= LM_ST_UNLOCKED
;
409 /* Check for state != intended state */
410 if (unlikely(state
!= gl
->gl_target
)) {
411 if (gh
&& !test_bit(GLF_DEMOTE_IN_PROGRESS
, &gl
->gl_flags
)) {
412 /* move to back of queue and try next entry */
413 if (ret
& LM_OUT_CANCELED
) {
414 if ((gh
->gh_flags
& LM_FLAG_PRIORITY
) == 0)
415 list_move_tail(&gh
->gh_list
, &gl
->gl_holders
);
416 gh
= find_first_waiter(gl
);
417 gl
->gl_target
= gh
->gh_state
;
420 /* Some error or failed "try lock" - report it */
421 if ((ret
& LM_OUT_ERROR
) ||
422 (gh
->gh_flags
& (LM_FLAG_TRY
| LM_FLAG_TRY_1CB
))) {
423 gl
->gl_target
= gl
->gl_state
;
429 /* Unlocked due to conversion deadlock, try again */
432 do_xmote(gl
, gh
, gl
->gl_target
);
434 /* Conversion fails, unlock and try again */
437 do_xmote(gl
, gh
, LM_ST_UNLOCKED
);
439 default: /* Everything else */
440 pr_err("wanted %u got %u\n", gl
->gl_target
, state
);
443 spin_unlock(&gl
->gl_lockref
.lock
);
447 /* Fast path - we got what we asked for */
448 if (test_and_clear_bit(GLF_DEMOTE_IN_PROGRESS
, &gl
->gl_flags
))
449 gfs2_demote_wake(gl
);
450 if (state
!= LM_ST_UNLOCKED
) {
451 if (glops
->go_xmote_bh
) {
452 spin_unlock(&gl
->gl_lockref
.lock
);
453 rv
= glops
->go_xmote_bh(gl
, gh
);
454 spin_lock(&gl
->gl_lockref
.lock
);
465 clear_bit(GLF_LOCK
, &gl
->gl_flags
);
467 spin_unlock(&gl
->gl_lockref
.lock
);
471 * do_xmote - Calls the DLM to change the state of a lock
472 * @gl: The lock state
473 * @gh: The holder (only for promotes)
474 * @target: The target lock state
478 static void do_xmote(struct gfs2_glock
*gl
, struct gfs2_holder
*gh
, unsigned int target
)
479 __releases(&gl
->gl_lockref
.lock
)
480 __acquires(&gl
->gl_lockref
.lock
)
482 const struct gfs2_glock_operations
*glops
= gl
->gl_ops
;
483 struct gfs2_sbd
*sdp
= gl
->gl_name
.ln_sbd
;
484 unsigned int lck_flags
= (unsigned int)(gh
? gh
->gh_flags
: 0);
487 if (unlikely(test_bit(SDF_SHUTDOWN
, &sdp
->sd_flags
)) &&
488 target
!= LM_ST_UNLOCKED
)
490 lck_flags
&= (LM_FLAG_TRY
| LM_FLAG_TRY_1CB
| LM_FLAG_NOEXP
|
492 GLOCK_BUG_ON(gl
, gl
->gl_state
== target
);
493 GLOCK_BUG_ON(gl
, gl
->gl_state
== gl
->gl_target
);
494 if ((target
== LM_ST_UNLOCKED
|| target
== LM_ST_DEFERRED
) &&
496 set_bit(GLF_INVALIDATE_IN_PROGRESS
, &gl
->gl_flags
);
497 do_error(gl
, 0); /* Fail queued try locks */
500 set_bit(GLF_BLOCKING
, &gl
->gl_flags
);
501 if ((gl
->gl_req
== LM_ST_UNLOCKED
) ||
502 (gl
->gl_state
== LM_ST_EXCLUSIVE
) ||
503 (lck_flags
& (LM_FLAG_TRY
|LM_FLAG_TRY_1CB
)))
504 clear_bit(GLF_BLOCKING
, &gl
->gl_flags
);
505 spin_unlock(&gl
->gl_lockref
.lock
);
508 if (test_bit(GLF_INVALIDATE_IN_PROGRESS
, &gl
->gl_flags
))
509 glops
->go_inval(gl
, target
== LM_ST_DEFERRED
? 0 : DIO_METADATA
);
510 clear_bit(GLF_INVALIDATE_IN_PROGRESS
, &gl
->gl_flags
);
513 if (sdp
->sd_lockstruct
.ls_ops
->lm_lock
) {
515 ret
= sdp
->sd_lockstruct
.ls_ops
->lm_lock(gl
, target
, lck_flags
);
516 if (ret
== -EINVAL
&& gl
->gl_target
== LM_ST_UNLOCKED
&&
517 target
== LM_ST_UNLOCKED
&&
518 test_bit(SDF_SKIP_DLM_UNLOCK
, &sdp
->sd_flags
)) {
519 finish_xmote(gl
, target
);
520 gfs2_glock_queue_work(gl
, 0);
523 pr_err("lm_lock ret %d\n", ret
);
524 GLOCK_BUG_ON(gl
, !test_bit(SDF_SHUTDOWN
,
527 } else { /* lock_nolock */
528 finish_xmote(gl
, target
);
529 gfs2_glock_queue_work(gl
, 0);
532 spin_lock(&gl
->gl_lockref
.lock
);
536 * find_first_holder - find the first "holder" gh
540 static inline struct gfs2_holder
*find_first_holder(const struct gfs2_glock
*gl
)
542 struct gfs2_holder
*gh
;
544 if (!list_empty(&gl
->gl_holders
)) {
545 gh
= list_entry(gl
->gl_holders
.next
, struct gfs2_holder
, gh_list
);
546 if (test_bit(HIF_HOLDER
, &gh
->gh_iflags
))
553 * run_queue - do all outstanding tasks related to a glock
554 * @gl: The glock in question
555 * @nonblock: True if we must not block in run_queue
559 static void run_queue(struct gfs2_glock
*gl
, const int nonblock
)
560 __releases(&gl
->gl_lockref
.lock
)
561 __acquires(&gl
->gl_lockref
.lock
)
563 struct gfs2_holder
*gh
= NULL
;
566 if (test_and_set_bit(GLF_LOCK
, &gl
->gl_flags
))
569 GLOCK_BUG_ON(gl
, test_bit(GLF_DEMOTE_IN_PROGRESS
, &gl
->gl_flags
));
571 if (test_bit(GLF_DEMOTE
, &gl
->gl_flags
) &&
572 gl
->gl_demote_state
!= gl
->gl_state
) {
573 if (find_first_holder(gl
))
577 set_bit(GLF_DEMOTE_IN_PROGRESS
, &gl
->gl_flags
);
578 GLOCK_BUG_ON(gl
, gl
->gl_demote_state
== LM_ST_EXCLUSIVE
);
579 gl
->gl_target
= gl
->gl_demote_state
;
581 if (test_bit(GLF_DEMOTE
, &gl
->gl_flags
))
582 gfs2_demote_wake(gl
);
583 ret
= do_promote(gl
);
588 gh
= find_first_waiter(gl
);
589 gl
->gl_target
= gh
->gh_state
;
590 if (!(gh
->gh_flags
& (LM_FLAG_TRY
| LM_FLAG_TRY_1CB
)))
591 do_error(gl
, 0); /* Fail queued try locks */
593 do_xmote(gl
, gh
, gl
->gl_target
);
598 clear_bit(GLF_LOCK
, &gl
->gl_flags
);
599 smp_mb__after_atomic();
600 gl
->gl_lockref
.count
++;
601 __gfs2_glock_queue_work(gl
, 0);
605 clear_bit(GLF_LOCK
, &gl
->gl_flags
);
606 smp_mb__after_atomic();
610 static void delete_work_func(struct work_struct
*work
)
612 struct gfs2_glock
*gl
= container_of(work
, struct gfs2_glock
, gl_delete
);
613 struct gfs2_sbd
*sdp
= gl
->gl_name
.ln_sbd
;
615 u64 no_addr
= gl
->gl_name
.ln_number
;
617 /* If someone's using this glock to create a new dinode, the block must
618 have been freed by another node, then re-used, in which case our
619 iopen callback is too late after the fact. Ignore it. */
620 if (test_bit(GLF_INODE_CREATING
, &gl
->gl_flags
))
623 inode
= gfs2_lookup_by_inum(sdp
, no_addr
, NULL
, GFS2_BLKST_UNLINKED
);
624 if (inode
&& !IS_ERR(inode
)) {
625 d_prune_aliases(inode
);
632 static void glock_work_func(struct work_struct
*work
)
634 unsigned long delay
= 0;
635 struct gfs2_glock
*gl
= container_of(work
, struct gfs2_glock
, gl_work
.work
);
636 unsigned int drop_refs
= 1;
638 if (test_and_clear_bit(GLF_REPLY_PENDING
, &gl
->gl_flags
)) {
639 finish_xmote(gl
, gl
->gl_reply
);
642 spin_lock(&gl
->gl_lockref
.lock
);
643 if (test_bit(GLF_PENDING_DEMOTE
, &gl
->gl_flags
) &&
644 gl
->gl_state
!= LM_ST_UNLOCKED
&&
645 gl
->gl_demote_state
!= LM_ST_EXCLUSIVE
) {
646 unsigned long holdtime
, now
= jiffies
;
648 holdtime
= gl
->gl_tchange
+ gl
->gl_hold_time
;
649 if (time_before(now
, holdtime
))
650 delay
= holdtime
- now
;
653 clear_bit(GLF_PENDING_DEMOTE
, &gl
->gl_flags
);
654 set_bit(GLF_DEMOTE
, &gl
->gl_flags
);
659 /* Keep one glock reference for the work we requeue. */
661 if (gl
->gl_name
.ln_type
!= LM_TYPE_INODE
)
663 __gfs2_glock_queue_work(gl
, delay
);
667 * Drop the remaining glock references manually here. (Mind that
668 * __gfs2_glock_queue_work depends on the lockref spinlock begin held
671 gl
->gl_lockref
.count
-= drop_refs
;
672 if (!gl
->gl_lockref
.count
) {
673 __gfs2_glock_put(gl
);
676 spin_unlock(&gl
->gl_lockref
.lock
);
680 * gfs2_glock_get() - Get a glock, or create one if one doesn't exist
681 * @sdp: The GFS2 superblock
682 * @number: the lock number
683 * @glops: The glock_operations to use
684 * @create: If 0, don't create the glock if it doesn't exist
685 * @glp: the glock is returned here
687 * This does not lock a glock, just finds/creates structures for one.
692 int gfs2_glock_get(struct gfs2_sbd
*sdp
, u64 number
,
693 const struct gfs2_glock_operations
*glops
, int create
,
694 struct gfs2_glock
**glp
)
696 struct super_block
*s
= sdp
->sd_vfs
;
697 struct lm_lockname name
= { .ln_number
= number
,
698 .ln_type
= glops
->go_type
,
700 struct gfs2_glock
*gl
, *tmp
;
701 struct address_space
*mapping
;
702 struct kmem_cache
*cachep
;
706 gl
= rhashtable_lookup_fast(&gl_hash_table
, &name
, ht_parms
);
707 if (gl
&& !lockref_get_not_dead(&gl
->gl_lockref
))
717 if (glops
->go_flags
& GLOF_ASPACE
)
718 cachep
= gfs2_glock_aspace_cachep
;
720 cachep
= gfs2_glock_cachep
;
721 gl
= kmem_cache_alloc(cachep
, GFP_NOFS
);
725 memset(&gl
->gl_lksb
, 0, sizeof(struct dlm_lksb
));
727 if (glops
->go_flags
& GLOF_LVB
) {
728 gl
->gl_lksb
.sb_lvbptr
= kzalloc(GFS2_MIN_LVB_SIZE
, GFP_NOFS
);
729 if (!gl
->gl_lksb
.sb_lvbptr
) {
730 kmem_cache_free(cachep
, gl
);
735 atomic_inc(&sdp
->sd_glock_disposal
);
736 gl
->gl_node
.next
= NULL
;
739 gl
->gl_lockref
.count
= 1;
740 gl
->gl_state
= LM_ST_UNLOCKED
;
741 gl
->gl_target
= LM_ST_UNLOCKED
;
742 gl
->gl_demote_state
= LM_ST_EXCLUSIVE
;
746 /* We use the global stats to estimate the initial per-glock stats */
747 gl
->gl_stats
= this_cpu_ptr(sdp
->sd_lkstats
)->lkstats
[glops
->go_type
];
749 gl
->gl_stats
.stats
[GFS2_LKS_DCOUNT
] = 0;
750 gl
->gl_stats
.stats
[GFS2_LKS_QCOUNT
] = 0;
751 gl
->gl_tchange
= jiffies
;
752 gl
->gl_object
= NULL
;
753 gl
->gl_hold_time
= GL_GLOCK_DFT_HOLD
;
754 INIT_DELAYED_WORK(&gl
->gl_work
, glock_work_func
);
755 INIT_WORK(&gl
->gl_delete
, delete_work_func
);
757 mapping
= gfs2_glock2aspace(gl
);
759 mapping
->a_ops
= &gfs2_meta_aops
;
760 mapping
->host
= s
->s_bdev
->bd_inode
;
762 mapping_set_gfp_mask(mapping
, GFP_NOFS
);
763 mapping
->private_data
= NULL
;
764 mapping
->writeback_index
= 0;
769 tmp
= rhashtable_lookup_get_insert_fast(&gl_hash_table
, &gl
->gl_node
,
779 if (lockref_get_not_dead(&tmp
->gl_lockref
)) {
788 kfree(gl
->gl_lksb
.sb_lvbptr
);
789 kmem_cache_free(cachep
, gl
);
790 atomic_dec(&sdp
->sd_glock_disposal
);
798 * gfs2_holder_init - initialize a struct gfs2_holder in the default way
800 * @state: the state we're requesting
801 * @flags: the modifier flags
802 * @gh: the holder structure
806 void gfs2_holder_init(struct gfs2_glock
*gl
, unsigned int state
, u16 flags
,
807 struct gfs2_holder
*gh
)
809 INIT_LIST_HEAD(&gh
->gh_list
);
811 gh
->gh_ip
= _RET_IP_
;
812 gh
->gh_owner_pid
= get_pid(task_pid(current
));
813 gh
->gh_state
= state
;
814 gh
->gh_flags
= flags
;
821 * gfs2_holder_reinit - reinitialize a struct gfs2_holder so we can requeue it
822 * @state: the state we're requesting
823 * @flags: the modifier flags
824 * @gh: the holder structure
826 * Don't mess with the glock.
830 void gfs2_holder_reinit(unsigned int state
, u16 flags
, struct gfs2_holder
*gh
)
832 gh
->gh_state
= state
;
833 gh
->gh_flags
= flags
;
835 gh
->gh_ip
= _RET_IP_
;
836 put_pid(gh
->gh_owner_pid
);
837 gh
->gh_owner_pid
= get_pid(task_pid(current
));
841 * gfs2_holder_uninit - uninitialize a holder structure (drop glock reference)
842 * @gh: the holder structure
846 void gfs2_holder_uninit(struct gfs2_holder
*gh
)
848 put_pid(gh
->gh_owner_pid
);
849 gfs2_glock_put(gh
->gh_gl
);
850 gfs2_holder_mark_uninitialized(gh
);
855 * gfs2_glock_wait - wait on a glock acquisition
856 * @gh: the glock holder
858 * Returns: 0 on success
861 int gfs2_glock_wait(struct gfs2_holder
*gh
)
863 unsigned long time1
= jiffies
;
866 wait_on_bit(&gh
->gh_iflags
, HIF_WAIT
, TASK_UNINTERRUPTIBLE
);
867 if (time_after(jiffies
, time1
+ HZ
)) /* have we waited > a second? */
868 /* Lengthen the minimum hold time. */
869 gh
->gh_gl
->gl_hold_time
= min(gh
->gh_gl
->gl_hold_time
+
876 * handle_callback - process a demote request
878 * @state: the state the caller wants us to change to
880 * There are only two requests that we are going to see in actual
881 * practise: LM_ST_SHARED and LM_ST_UNLOCKED
884 static void handle_callback(struct gfs2_glock
*gl
, unsigned int state
,
885 unsigned long delay
, bool remote
)
887 int bit
= delay
? GLF_PENDING_DEMOTE
: GLF_DEMOTE
;
889 set_bit(bit
, &gl
->gl_flags
);
890 if (gl
->gl_demote_state
== LM_ST_EXCLUSIVE
) {
891 gl
->gl_demote_state
= state
;
892 gl
->gl_demote_time
= jiffies
;
893 } else if (gl
->gl_demote_state
!= LM_ST_UNLOCKED
&&
894 gl
->gl_demote_state
!= state
) {
895 gl
->gl_demote_state
= LM_ST_UNLOCKED
;
897 if (gl
->gl_ops
->go_callback
)
898 gl
->gl_ops
->go_callback(gl
, remote
);
899 trace_gfs2_demote_rq(gl
, remote
);
902 void gfs2_print_dbg(struct seq_file
*seq
, const char *fmt
, ...)
904 struct va_format vaf
;
910 seq_vprintf(seq
, fmt
, args
);
922 * add_to_queue - Add a holder to the wait queue (but look for recursion)
923 * @gh: the holder structure to add
925 * Eventually we should move the recursive locking trap to a
926 * debugging option or something like that. This is the fast
927 * path and needs to have the minimum number of distractions.
931 static inline void add_to_queue(struct gfs2_holder
*gh
)
932 __releases(&gl
->gl_lockref
.lock
)
933 __acquires(&gl
->gl_lockref
.lock
)
935 struct gfs2_glock
*gl
= gh
->gh_gl
;
936 struct gfs2_sbd
*sdp
= gl
->gl_name
.ln_sbd
;
937 struct list_head
*insert_pt
= NULL
;
938 struct gfs2_holder
*gh2
;
941 BUG_ON(gh
->gh_owner_pid
== NULL
);
942 if (test_and_set_bit(HIF_WAIT
, &gh
->gh_iflags
))
945 if (gh
->gh_flags
& (LM_FLAG_TRY
| LM_FLAG_TRY_1CB
)) {
946 if (test_bit(GLF_LOCK
, &gl
->gl_flags
))
947 try_futile
= !may_grant(gl
, gh
);
948 if (test_bit(GLF_INVALIDATE_IN_PROGRESS
, &gl
->gl_flags
))
952 list_for_each_entry(gh2
, &gl
->gl_holders
, gh_list
) {
953 if (unlikely(gh2
->gh_owner_pid
== gh
->gh_owner_pid
&&
954 (gh
->gh_gl
->gl_ops
->go_type
!= LM_TYPE_FLOCK
)))
957 !(gh2
->gh_flags
& (LM_FLAG_TRY
| LM_FLAG_TRY_1CB
))) {
959 gh
->gh_error
= GLR_TRYFAILED
;
960 gfs2_holder_wake(gh
);
963 if (test_bit(HIF_HOLDER
, &gh2
->gh_iflags
))
965 if (unlikely((gh
->gh_flags
& LM_FLAG_PRIORITY
) && !insert_pt
))
966 insert_pt
= &gh2
->gh_list
;
968 set_bit(GLF_QUEUED
, &gl
->gl_flags
);
969 trace_gfs2_glock_queue(gh
, 1);
970 gfs2_glstats_inc(gl
, GFS2_LKS_QCOUNT
);
971 gfs2_sbstats_inc(gl
, GFS2_LKS_QCOUNT
);
972 if (likely(insert_pt
== NULL
)) {
973 list_add_tail(&gh
->gh_list
, &gl
->gl_holders
);
974 if (unlikely(gh
->gh_flags
& LM_FLAG_PRIORITY
))
978 list_add_tail(&gh
->gh_list
, insert_pt
);
980 gh
= list_entry(gl
->gl_holders
.next
, struct gfs2_holder
, gh_list
);
981 if (!(gh
->gh_flags
& LM_FLAG_PRIORITY
)) {
982 spin_unlock(&gl
->gl_lockref
.lock
);
983 if (sdp
->sd_lockstruct
.ls_ops
->lm_cancel
)
984 sdp
->sd_lockstruct
.ls_ops
->lm_cancel(gl
);
985 spin_lock(&gl
->gl_lockref
.lock
);
990 pr_err("original: %pSR\n", (void *)gh2
->gh_ip
);
991 pr_err("pid: %d\n", pid_nr(gh2
->gh_owner_pid
));
992 pr_err("lock type: %d req lock state : %d\n",
993 gh2
->gh_gl
->gl_name
.ln_type
, gh2
->gh_state
);
994 pr_err("new: %pSR\n", (void *)gh
->gh_ip
);
995 pr_err("pid: %d\n", pid_nr(gh
->gh_owner_pid
));
996 pr_err("lock type: %d req lock state : %d\n",
997 gh
->gh_gl
->gl_name
.ln_type
, gh
->gh_state
);
998 gfs2_dump_glock(NULL
, gl
);
1003 * gfs2_glock_nq - enqueue a struct gfs2_holder onto a glock (acquire a glock)
1004 * @gh: the holder structure
1006 * if (gh->gh_flags & GL_ASYNC), this never returns an error
1008 * Returns: 0, GLR_TRYFAILED, or errno on failure
1011 int gfs2_glock_nq(struct gfs2_holder
*gh
)
1013 struct gfs2_glock
*gl
= gh
->gh_gl
;
1014 struct gfs2_sbd
*sdp
= gl
->gl_name
.ln_sbd
;
1017 if (unlikely(test_bit(SDF_SHUTDOWN
, &sdp
->sd_flags
)))
1020 if (test_bit(GLF_LRU
, &gl
->gl_flags
))
1021 gfs2_glock_remove_from_lru(gl
);
1023 spin_lock(&gl
->gl_lockref
.lock
);
1025 if (unlikely((LM_FLAG_NOEXP
& gh
->gh_flags
) &&
1026 test_and_clear_bit(GLF_FROZEN
, &gl
->gl_flags
))) {
1027 set_bit(GLF_REPLY_PENDING
, &gl
->gl_flags
);
1028 gl
->gl_lockref
.count
++;
1029 __gfs2_glock_queue_work(gl
, 0);
1032 spin_unlock(&gl
->gl_lockref
.lock
);
1034 if (!(gh
->gh_flags
& GL_ASYNC
))
1035 error
= gfs2_glock_wait(gh
);
1041 * gfs2_glock_poll - poll to see if an async request has been completed
1044 * Returns: 1 if the request is ready to be gfs2_glock_wait()ed on
1047 int gfs2_glock_poll(struct gfs2_holder
*gh
)
1049 return test_bit(HIF_WAIT
, &gh
->gh_iflags
) ? 0 : 1;
1053 * gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock)
1054 * @gh: the glock holder
1058 void gfs2_glock_dq(struct gfs2_holder
*gh
)
1060 struct gfs2_glock
*gl
= gh
->gh_gl
;
1061 const struct gfs2_glock_operations
*glops
= gl
->gl_ops
;
1065 spin_lock(&gl
->gl_lockref
.lock
);
1066 if (gh
->gh_flags
& GL_NOCACHE
)
1067 handle_callback(gl
, LM_ST_UNLOCKED
, 0, false);
1069 list_del_init(&gh
->gh_list
);
1070 clear_bit(HIF_HOLDER
, &gh
->gh_iflags
);
1071 if (find_first_holder(gl
) == NULL
) {
1072 if (glops
->go_unlock
) {
1073 GLOCK_BUG_ON(gl
, test_and_set_bit(GLF_LOCK
, &gl
->gl_flags
));
1074 spin_unlock(&gl
->gl_lockref
.lock
);
1075 glops
->go_unlock(gh
);
1076 spin_lock(&gl
->gl_lockref
.lock
);
1077 clear_bit(GLF_LOCK
, &gl
->gl_flags
);
1079 if (list_empty(&gl
->gl_holders
) &&
1080 !test_bit(GLF_PENDING_DEMOTE
, &gl
->gl_flags
) &&
1081 !test_bit(GLF_DEMOTE
, &gl
->gl_flags
))
1084 if (!test_bit(GLF_LFLUSH
, &gl
->gl_flags
) && demote_ok(gl
) &&
1085 (glops
->go_flags
& GLOF_LRU
))
1086 gfs2_glock_add_to_lru(gl
);
1088 trace_gfs2_glock_queue(gh
, 0);
1089 if (unlikely(!fast_path
)) {
1090 gl
->gl_lockref
.count
++;
1091 if (test_bit(GLF_PENDING_DEMOTE
, &gl
->gl_flags
) &&
1092 !test_bit(GLF_DEMOTE
, &gl
->gl_flags
) &&
1093 gl
->gl_name
.ln_type
== LM_TYPE_INODE
)
1094 delay
= gl
->gl_hold_time
;
1095 __gfs2_glock_queue_work(gl
, delay
);
1097 spin_unlock(&gl
->gl_lockref
.lock
);
1100 void gfs2_glock_dq_wait(struct gfs2_holder
*gh
)
1102 struct gfs2_glock
*gl
= gh
->gh_gl
;
1105 wait_on_bit(&gl
->gl_flags
, GLF_DEMOTE
, TASK_UNINTERRUPTIBLE
);
1109 * gfs2_glock_dq_uninit - dequeue a holder from a glock and initialize it
1110 * @gh: the holder structure
1114 void gfs2_glock_dq_uninit(struct gfs2_holder
*gh
)
1117 gfs2_holder_uninit(gh
);
1121 * gfs2_glock_nq_num - acquire a glock based on lock number
1122 * @sdp: the filesystem
1123 * @number: the lock number
1124 * @glops: the glock operations for the type of glock
1125 * @state: the state to acquire the glock in
1126 * @flags: modifier flags for the acquisition
1127 * @gh: the struct gfs2_holder
1132 int gfs2_glock_nq_num(struct gfs2_sbd
*sdp
, u64 number
,
1133 const struct gfs2_glock_operations
*glops
,
1134 unsigned int state
, u16 flags
, struct gfs2_holder
*gh
)
1136 struct gfs2_glock
*gl
;
1139 error
= gfs2_glock_get(sdp
, number
, glops
, CREATE
, &gl
);
1141 error
= gfs2_glock_nq_init(gl
, state
, flags
, gh
);
1149 * glock_compare - Compare two struct gfs2_glock structures for sorting
1150 * @arg_a: the first structure
1151 * @arg_b: the second structure
1155 static int glock_compare(const void *arg_a
, const void *arg_b
)
1157 const struct gfs2_holder
*gh_a
= *(const struct gfs2_holder
**)arg_a
;
1158 const struct gfs2_holder
*gh_b
= *(const struct gfs2_holder
**)arg_b
;
1159 const struct lm_lockname
*a
= &gh_a
->gh_gl
->gl_name
;
1160 const struct lm_lockname
*b
= &gh_b
->gh_gl
->gl_name
;
1162 if (a
->ln_number
> b
->ln_number
)
1164 if (a
->ln_number
< b
->ln_number
)
1166 BUG_ON(gh_a
->gh_gl
->gl_ops
->go_type
== gh_b
->gh_gl
->gl_ops
->go_type
);
1171 * nq_m_sync - synchonously acquire more than one glock in deadlock free order
1172 * @num_gh: the number of structures
1173 * @ghs: an array of struct gfs2_holder structures
1175 * Returns: 0 on success (all glocks acquired),
1176 * errno on failure (no glocks acquired)
1179 static int nq_m_sync(unsigned int num_gh
, struct gfs2_holder
*ghs
,
1180 struct gfs2_holder
**p
)
1185 for (x
= 0; x
< num_gh
; x
++)
1188 sort(p
, num_gh
, sizeof(struct gfs2_holder
*), glock_compare
, NULL
);
1190 for (x
= 0; x
< num_gh
; x
++) {
1191 p
[x
]->gh_flags
&= ~(LM_FLAG_TRY
| GL_ASYNC
);
1193 error
= gfs2_glock_nq(p
[x
]);
1196 gfs2_glock_dq(p
[x
]);
1205 * gfs2_glock_nq_m - acquire multiple glocks
1206 * @num_gh: the number of structures
1207 * @ghs: an array of struct gfs2_holder structures
1210 * Returns: 0 on success (all glocks acquired),
1211 * errno on failure (no glocks acquired)
1214 int gfs2_glock_nq_m(unsigned int num_gh
, struct gfs2_holder
*ghs
)
1216 struct gfs2_holder
*tmp
[4];
1217 struct gfs2_holder
**pph
= tmp
;
1224 ghs
->gh_flags
&= ~(LM_FLAG_TRY
| GL_ASYNC
);
1225 return gfs2_glock_nq(ghs
);
1229 pph
= kmalloc(num_gh
* sizeof(struct gfs2_holder
*), GFP_NOFS
);
1234 error
= nq_m_sync(num_gh
, ghs
, pph
);
1243 * gfs2_glock_dq_m - release multiple glocks
1244 * @num_gh: the number of structures
1245 * @ghs: an array of struct gfs2_holder structures
1249 void gfs2_glock_dq_m(unsigned int num_gh
, struct gfs2_holder
*ghs
)
1252 gfs2_glock_dq(&ghs
[num_gh
]);
1255 void gfs2_glock_cb(struct gfs2_glock
*gl
, unsigned int state
)
1257 unsigned long delay
= 0;
1258 unsigned long holdtime
;
1259 unsigned long now
= jiffies
;
1261 gfs2_glock_hold(gl
);
1262 holdtime
= gl
->gl_tchange
+ gl
->gl_hold_time
;
1263 if (test_bit(GLF_QUEUED
, &gl
->gl_flags
) &&
1264 gl
->gl_name
.ln_type
== LM_TYPE_INODE
) {
1265 if (time_before(now
, holdtime
))
1266 delay
= holdtime
- now
;
1267 if (test_bit(GLF_REPLY_PENDING
, &gl
->gl_flags
))
1268 delay
= gl
->gl_hold_time
;
1271 spin_lock(&gl
->gl_lockref
.lock
);
1272 handle_callback(gl
, state
, delay
, true);
1273 __gfs2_glock_queue_work(gl
, delay
);
1274 spin_unlock(&gl
->gl_lockref
.lock
);
1278 * gfs2_should_freeze - Figure out if glock should be frozen
1279 * @gl: The glock in question
1281 * Glocks are not frozen if (a) the result of the dlm operation is
1282 * an error, (b) the locking operation was an unlock operation or
1283 * (c) if there is a "noexp" flagged request anywhere in the queue
1285 * Returns: 1 if freezing should occur, 0 otherwise
1288 static int gfs2_should_freeze(const struct gfs2_glock
*gl
)
1290 const struct gfs2_holder
*gh
;
1292 if (gl
->gl_reply
& ~LM_OUT_ST_MASK
)
1294 if (gl
->gl_target
== LM_ST_UNLOCKED
)
1297 list_for_each_entry(gh
, &gl
->gl_holders
, gh_list
) {
1298 if (test_bit(HIF_HOLDER
, &gh
->gh_iflags
))
1300 if (LM_FLAG_NOEXP
& gh
->gh_flags
)
1308 * gfs2_glock_complete - Callback used by locking
1309 * @gl: Pointer to the glock
1310 * @ret: The return value from the dlm
1312 * The gl_reply field is under the gl_lockref.lock lock so that it is ok
1313 * to use a bitfield shared with other glock state fields.
1316 void gfs2_glock_complete(struct gfs2_glock
*gl
, int ret
)
1318 struct lm_lockstruct
*ls
= &gl
->gl_name
.ln_sbd
->sd_lockstruct
;
1320 spin_lock(&gl
->gl_lockref
.lock
);
1323 if (unlikely(test_bit(DFL_BLOCK_LOCKS
, &ls
->ls_recover_flags
))) {
1324 if (gfs2_should_freeze(gl
)) {
1325 set_bit(GLF_FROZEN
, &gl
->gl_flags
);
1326 spin_unlock(&gl
->gl_lockref
.lock
);
1331 gl
->gl_lockref
.count
++;
1332 set_bit(GLF_REPLY_PENDING
, &gl
->gl_flags
);
1333 __gfs2_glock_queue_work(gl
, 0);
1334 spin_unlock(&gl
->gl_lockref
.lock
);
1337 static int glock_cmp(void *priv
, struct list_head
*a
, struct list_head
*b
)
1339 struct gfs2_glock
*gla
, *glb
;
1341 gla
= list_entry(a
, struct gfs2_glock
, gl_lru
);
1342 glb
= list_entry(b
, struct gfs2_glock
, gl_lru
);
1344 if (gla
->gl_name
.ln_number
> glb
->gl_name
.ln_number
)
1346 if (gla
->gl_name
.ln_number
< glb
->gl_name
.ln_number
)
1353 * gfs2_dispose_glock_lru - Demote a list of glocks
1354 * @list: The list to dispose of
1356 * Disposing of glocks may involve disk accesses, so that here we sort
1357 * the glocks by number (i.e. disk location of the inodes) so that if
1358 * there are any such accesses, they'll be sent in order (mostly).
1360 * Must be called under the lru_lock, but may drop and retake this
1361 * lock. While the lru_lock is dropped, entries may vanish from the
1362 * list, but no new entries will appear on the list (since it is
1366 static void gfs2_dispose_glock_lru(struct list_head
*list
)
1367 __releases(&lru_lock
)
1368 __acquires(&lru_lock
)
1370 struct gfs2_glock
*gl
;
1372 list_sort(NULL
, list
, glock_cmp
);
1374 while(!list_empty(list
)) {
1375 gl
= list_entry(list
->next
, struct gfs2_glock
, gl_lru
);
1376 list_del_init(&gl
->gl_lru
);
1377 if (!spin_trylock(&gl
->gl_lockref
.lock
)) {
1379 list_add(&gl
->gl_lru
, &lru_list
);
1380 atomic_inc(&lru_count
);
1383 if (test_and_set_bit(GLF_LOCK
, &gl
->gl_flags
)) {
1384 spin_unlock(&gl
->gl_lockref
.lock
);
1385 goto add_back_to_lru
;
1387 clear_bit(GLF_LRU
, &gl
->gl_flags
);
1388 gl
->gl_lockref
.count
++;
1390 handle_callback(gl
, LM_ST_UNLOCKED
, 0, false);
1391 WARN_ON(!test_and_clear_bit(GLF_LOCK
, &gl
->gl_flags
));
1392 __gfs2_glock_queue_work(gl
, 0);
1393 spin_unlock(&gl
->gl_lockref
.lock
);
1394 cond_resched_lock(&lru_lock
);
1399 * gfs2_scan_glock_lru - Scan the LRU looking for locks to demote
1400 * @nr: The number of entries to scan
1402 * This function selects the entries on the LRU which are able to
1403 * be demoted, and then kicks off the process by calling
1404 * gfs2_dispose_glock_lru() above.
1407 static long gfs2_scan_glock_lru(int nr
)
1409 struct gfs2_glock
*gl
;
1414 spin_lock(&lru_lock
);
1415 while ((nr
-- >= 0) && !list_empty(&lru_list
)) {
1416 gl
= list_entry(lru_list
.next
, struct gfs2_glock
, gl_lru
);
1418 /* Test for being demotable */
1419 if (!test_bit(GLF_LOCK
, &gl
->gl_flags
)) {
1420 list_move(&gl
->gl_lru
, &dispose
);
1421 atomic_dec(&lru_count
);
1426 list_move(&gl
->gl_lru
, &skipped
);
1428 list_splice(&skipped
, &lru_list
);
1429 if (!list_empty(&dispose
))
1430 gfs2_dispose_glock_lru(&dispose
);
1431 spin_unlock(&lru_lock
);
1436 static unsigned long gfs2_glock_shrink_scan(struct shrinker
*shrink
,
1437 struct shrink_control
*sc
)
1439 if (!(sc
->gfp_mask
& __GFP_FS
))
1441 return gfs2_scan_glock_lru(sc
->nr_to_scan
);
1444 static unsigned long gfs2_glock_shrink_count(struct shrinker
*shrink
,
1445 struct shrink_control
*sc
)
1447 return vfs_pressure_ratio(atomic_read(&lru_count
));
1450 static struct shrinker glock_shrinker
= {
1451 .seeks
= DEFAULT_SEEKS
,
1452 .count_objects
= gfs2_glock_shrink_count
,
1453 .scan_objects
= gfs2_glock_shrink_scan
,
1457 * examine_bucket - Call a function for glock in a hash bucket
1458 * @examiner: the function
1459 * @sdp: the filesystem
1460 * @bucket: the bucket
1462 * Note that the function can be called multiple times on the same
1463 * object. So the user must ensure that the function can cope with
1467 static void glock_hash_walk(glock_examiner examiner
, const struct gfs2_sbd
*sdp
)
1469 struct gfs2_glock
*gl
;
1470 struct rhashtable_iter iter
;
1472 rhashtable_walk_enter(&gl_hash_table
, &iter
);
1475 gl
= ERR_PTR(rhashtable_walk_start(&iter
));
1479 while ((gl
= rhashtable_walk_next(&iter
)) && !IS_ERR(gl
))
1480 if ((gl
->gl_name
.ln_sbd
== sdp
) &&
1481 lockref_get_not_dead(&gl
->gl_lockref
))
1484 rhashtable_walk_stop(&iter
);
1485 } while (cond_resched(), gl
== ERR_PTR(-EAGAIN
));
1487 rhashtable_walk_exit(&iter
);
1491 * thaw_glock - thaw out a glock which has an unprocessed reply waiting
1492 * @gl: The glock to thaw
1496 static void thaw_glock(struct gfs2_glock
*gl
)
1498 if (!test_and_clear_bit(GLF_FROZEN
, &gl
->gl_flags
)) {
1502 set_bit(GLF_REPLY_PENDING
, &gl
->gl_flags
);
1503 gfs2_glock_queue_work(gl
, 0);
1507 * clear_glock - look at a glock and see if we can free it from glock cache
1508 * @gl: the glock to look at
1512 static void clear_glock(struct gfs2_glock
*gl
)
1514 gfs2_glock_remove_from_lru(gl
);
1516 spin_lock(&gl
->gl_lockref
.lock
);
1517 if (gl
->gl_state
!= LM_ST_UNLOCKED
)
1518 handle_callback(gl
, LM_ST_UNLOCKED
, 0, false);
1519 __gfs2_glock_queue_work(gl
, 0);
1520 spin_unlock(&gl
->gl_lockref
.lock
);
1524 * gfs2_glock_thaw - Thaw any frozen glocks
1525 * @sdp: The super block
1529 void gfs2_glock_thaw(struct gfs2_sbd
*sdp
)
1531 glock_hash_walk(thaw_glock
, sdp
);
1534 static void dump_glock(struct seq_file
*seq
, struct gfs2_glock
*gl
)
1536 spin_lock(&gl
->gl_lockref
.lock
);
1537 gfs2_dump_glock(seq
, gl
);
1538 spin_unlock(&gl
->gl_lockref
.lock
);
1541 static void dump_glock_func(struct gfs2_glock
*gl
)
1543 dump_glock(NULL
, gl
);
1547 * gfs2_gl_hash_clear - Empty out the glock hash table
1548 * @sdp: the filesystem
1549 * @wait: wait until it's all gone
1551 * Called when unmounting the filesystem.
1554 void gfs2_gl_hash_clear(struct gfs2_sbd
*sdp
)
1556 set_bit(SDF_SKIP_DLM_UNLOCK
, &sdp
->sd_flags
);
1557 flush_workqueue(glock_workqueue
);
1558 glock_hash_walk(clear_glock
, sdp
);
1559 flush_workqueue(glock_workqueue
);
1560 wait_event_timeout(sdp
->sd_glock_wait
,
1561 atomic_read(&sdp
->sd_glock_disposal
) == 0,
1563 glock_hash_walk(dump_glock_func
, sdp
);
1566 void gfs2_glock_finish_truncate(struct gfs2_inode
*ip
)
1568 struct gfs2_glock
*gl
= ip
->i_gl
;
1571 ret
= gfs2_truncatei_resume(ip
);
1572 gfs2_assert_withdraw(gl
->gl_name
.ln_sbd
, ret
== 0);
1574 spin_lock(&gl
->gl_lockref
.lock
);
1575 clear_bit(GLF_LOCK
, &gl
->gl_flags
);
1577 spin_unlock(&gl
->gl_lockref
.lock
);
1580 static const char *state2str(unsigned state
)
1583 case LM_ST_UNLOCKED
:
1587 case LM_ST_DEFERRED
:
1589 case LM_ST_EXCLUSIVE
:
1595 static const char *hflags2str(char *buf
, u16 flags
, unsigned long iflags
)
1598 if (flags
& LM_FLAG_TRY
)
1600 if (flags
& LM_FLAG_TRY_1CB
)
1602 if (flags
& LM_FLAG_NOEXP
)
1604 if (flags
& LM_FLAG_ANY
)
1606 if (flags
& LM_FLAG_PRIORITY
)
1608 if (flags
& GL_ASYNC
)
1610 if (flags
& GL_EXACT
)
1612 if (flags
& GL_NOCACHE
)
1614 if (test_bit(HIF_HOLDER
, &iflags
))
1616 if (test_bit(HIF_WAIT
, &iflags
))
1618 if (test_bit(HIF_FIRST
, &iflags
))
1625 * dump_holder - print information about a glock holder
1626 * @seq: the seq_file struct
1627 * @gh: the glock holder
1631 static void dump_holder(struct seq_file
*seq
, const struct gfs2_holder
*gh
)
1633 struct task_struct
*gh_owner
= NULL
;
1637 if (gh
->gh_owner_pid
)
1638 gh_owner
= pid_task(gh
->gh_owner_pid
, PIDTYPE_PID
);
1639 gfs2_print_dbg(seq
, " H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
1640 state2str(gh
->gh_state
),
1641 hflags2str(flags_buf
, gh
->gh_flags
, gh
->gh_iflags
),
1643 gh
->gh_owner_pid
? (long)pid_nr(gh
->gh_owner_pid
) : -1,
1644 gh_owner
? gh_owner
->comm
: "(ended)",
1649 static const char *gflags2str(char *buf
, const struct gfs2_glock
*gl
)
1651 const unsigned long *gflags
= &gl
->gl_flags
;
1654 if (test_bit(GLF_LOCK
, gflags
))
1656 if (test_bit(GLF_DEMOTE
, gflags
))
1658 if (test_bit(GLF_PENDING_DEMOTE
, gflags
))
1660 if (test_bit(GLF_DEMOTE_IN_PROGRESS
, gflags
))
1662 if (test_bit(GLF_DIRTY
, gflags
))
1664 if (test_bit(GLF_LFLUSH
, gflags
))
1666 if (test_bit(GLF_INVALIDATE_IN_PROGRESS
, gflags
))
1668 if (test_bit(GLF_REPLY_PENDING
, gflags
))
1670 if (test_bit(GLF_INITIAL
, gflags
))
1672 if (test_bit(GLF_FROZEN
, gflags
))
1674 if (test_bit(GLF_QUEUED
, gflags
))
1676 if (test_bit(GLF_LRU
, gflags
))
1680 if (test_bit(GLF_BLOCKING
, gflags
))
1687 * gfs2_dump_glock - print information about a glock
1688 * @seq: The seq_file struct
1691 * The file format is as follows:
1692 * One line per object, capital letters are used to indicate objects
1693 * G = glock, I = Inode, R = rgrp, H = holder. Glocks are not indented,
1694 * other objects are indented by a single space and follow the glock to
1695 * which they are related. Fields are indicated by lower case letters
1696 * followed by a colon and the field value, except for strings which are in
1697 * [] so that its possible to see if they are composed of spaces for
1698 * example. The field's are n = number (id of the object), f = flags,
1699 * t = type, s = state, r = refcount, e = error, p = pid.
1703 void gfs2_dump_glock(struct seq_file
*seq
, const struct gfs2_glock
*gl
)
1705 const struct gfs2_glock_operations
*glops
= gl
->gl_ops
;
1706 unsigned long long dtime
;
1707 const struct gfs2_holder
*gh
;
1708 char gflags_buf
[32];
1710 dtime
= jiffies
- gl
->gl_demote_time
;
1711 dtime
*= 1000000/HZ
; /* demote time in uSec */
1712 if (!test_bit(GLF_DEMOTE
, &gl
->gl_flags
))
1714 gfs2_print_dbg(seq
, "G: s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d v:%d r:%d m:%ld\n",
1715 state2str(gl
->gl_state
),
1716 gl
->gl_name
.ln_type
,
1717 (unsigned long long)gl
->gl_name
.ln_number
,
1718 gflags2str(gflags_buf
, gl
),
1719 state2str(gl
->gl_target
),
1720 state2str(gl
->gl_demote_state
), dtime
,
1721 atomic_read(&gl
->gl_ail_count
),
1722 atomic_read(&gl
->gl_revokes
),
1723 (int)gl
->gl_lockref
.count
, gl
->gl_hold_time
);
1725 list_for_each_entry(gh
, &gl
->gl_holders
, gh_list
)
1726 dump_holder(seq
, gh
);
1728 if (gl
->gl_state
!= LM_ST_UNLOCKED
&& glops
->go_dump
)
1729 glops
->go_dump(seq
, gl
);
1732 static int gfs2_glstats_seq_show(struct seq_file
*seq
, void *iter_ptr
)
1734 struct gfs2_glock
*gl
= iter_ptr
;
1736 seq_printf(seq
, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
1737 gl
->gl_name
.ln_type
,
1738 (unsigned long long)gl
->gl_name
.ln_number
,
1739 (unsigned long long)gl
->gl_stats
.stats
[GFS2_LKS_SRTT
],
1740 (unsigned long long)gl
->gl_stats
.stats
[GFS2_LKS_SRTTVAR
],
1741 (unsigned long long)gl
->gl_stats
.stats
[GFS2_LKS_SRTTB
],
1742 (unsigned long long)gl
->gl_stats
.stats
[GFS2_LKS_SRTTVARB
],
1743 (unsigned long long)gl
->gl_stats
.stats
[GFS2_LKS_SIRT
],
1744 (unsigned long long)gl
->gl_stats
.stats
[GFS2_LKS_SIRTVAR
],
1745 (unsigned long long)gl
->gl_stats
.stats
[GFS2_LKS_DCOUNT
],
1746 (unsigned long long)gl
->gl_stats
.stats
[GFS2_LKS_QCOUNT
]);
1750 static const char *gfs2_gltype
[] = {
1764 static const char *gfs2_stype
[] = {
1765 [GFS2_LKS_SRTT
] = "srtt",
1766 [GFS2_LKS_SRTTVAR
] = "srttvar",
1767 [GFS2_LKS_SRTTB
] = "srttb",
1768 [GFS2_LKS_SRTTVARB
] = "srttvarb",
1769 [GFS2_LKS_SIRT
] = "sirt",
1770 [GFS2_LKS_SIRTVAR
] = "sirtvar",
1771 [GFS2_LKS_DCOUNT
] = "dlm",
1772 [GFS2_LKS_QCOUNT
] = "queue",
1775 #define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype))
1777 static int gfs2_sbstats_seq_show(struct seq_file
*seq
, void *iter_ptr
)
1779 struct gfs2_sbd
*sdp
= seq
->private;
1780 loff_t pos
= *(loff_t
*)iter_ptr
;
1781 unsigned index
= pos
>> 3;
1782 unsigned subindex
= pos
& 0x07;
1785 if (index
== 0 && subindex
!= 0)
1788 seq_printf(seq
, "%-10s %8s:", gfs2_gltype
[index
],
1789 (index
== 0) ? "cpu": gfs2_stype
[subindex
]);
1791 for_each_possible_cpu(i
) {
1792 const struct gfs2_pcpu_lkstats
*lkstats
= per_cpu_ptr(sdp
->sd_lkstats
, i
);
1795 seq_printf(seq
, " %15u", i
);
1797 seq_printf(seq
, " %15llu", (unsigned long long)lkstats
->
1798 lkstats
[index
- 1].stats
[subindex
]);
1800 seq_putc(seq
, '\n');
1804 int __init
gfs2_glock_init(void)
1808 ret
= rhashtable_init(&gl_hash_table
, &ht_parms
);
1812 glock_workqueue
= alloc_workqueue("glock_workqueue", WQ_MEM_RECLAIM
|
1813 WQ_HIGHPRI
| WQ_FREEZABLE
, 0);
1814 if (!glock_workqueue
) {
1815 rhashtable_destroy(&gl_hash_table
);
1818 gfs2_delete_workqueue
= alloc_workqueue("delete_workqueue",
1819 WQ_MEM_RECLAIM
| WQ_FREEZABLE
,
1821 if (!gfs2_delete_workqueue
) {
1822 destroy_workqueue(glock_workqueue
);
1823 rhashtable_destroy(&gl_hash_table
);
1827 ret
= register_shrinker(&glock_shrinker
);
1829 destroy_workqueue(gfs2_delete_workqueue
);
1830 destroy_workqueue(glock_workqueue
);
1831 rhashtable_destroy(&gl_hash_table
);
1838 void gfs2_glock_exit(void)
1840 unregister_shrinker(&glock_shrinker
);
1841 rhashtable_destroy(&gl_hash_table
);
1842 destroy_workqueue(glock_workqueue
);
1843 destroy_workqueue(gfs2_delete_workqueue
);
1846 static void gfs2_glock_iter_next(struct gfs2_glock_iter
*gi
)
1848 while ((gi
->gl
= rhashtable_walk_next(&gi
->hti
))) {
1849 if (IS_ERR(gi
->gl
)) {
1850 if (PTR_ERR(gi
->gl
) == -EAGAIN
)
1855 /* Skip entries for other sb and dead entries */
1856 if (gi
->sdp
== gi
->gl
->gl_name
.ln_sbd
&&
1857 !__lockref_is_dead(&gi
->gl
->gl_lockref
))
1862 static void *gfs2_glock_seq_start(struct seq_file
*seq
, loff_t
*pos
)
1864 struct gfs2_glock_iter
*gi
= seq
->private;
1867 rhashtable_walk_enter(&gl_hash_table
, &gi
->hti
);
1868 if (rhashtable_walk_start(&gi
->hti
) != 0)
1872 gfs2_glock_iter_next(gi
);
1873 } while (gi
->gl
&& n
--);
1875 gi
->last_pos
= *pos
;
1880 static void *gfs2_glock_seq_next(struct seq_file
*seq
, void *iter_ptr
,
1883 struct gfs2_glock_iter
*gi
= seq
->private;
1886 gi
->last_pos
= *pos
;
1887 gfs2_glock_iter_next(gi
);
1892 static void gfs2_glock_seq_stop(struct seq_file
*seq
, void *iter_ptr
)
1894 struct gfs2_glock_iter
*gi
= seq
->private;
1897 rhashtable_walk_stop(&gi
->hti
);
1898 rhashtable_walk_exit(&gi
->hti
);
1901 static int gfs2_glock_seq_show(struct seq_file
*seq
, void *iter_ptr
)
1903 dump_glock(seq
, iter_ptr
);
1907 static void *gfs2_sbstats_seq_start(struct seq_file
*seq
, loff_t
*pos
)
1910 if (*pos
>= GFS2_NR_SBSTATS
)
1915 static void *gfs2_sbstats_seq_next(struct seq_file
*seq
, void *iter_ptr
,
1919 if (*pos
>= GFS2_NR_SBSTATS
)
1924 static void gfs2_sbstats_seq_stop(struct seq_file
*seq
, void *iter_ptr
)
1929 static const struct seq_operations gfs2_glock_seq_ops
= {
1930 .start
= gfs2_glock_seq_start
,
1931 .next
= gfs2_glock_seq_next
,
1932 .stop
= gfs2_glock_seq_stop
,
1933 .show
= gfs2_glock_seq_show
,
1936 static const struct seq_operations gfs2_glstats_seq_ops
= {
1937 .start
= gfs2_glock_seq_start
,
1938 .next
= gfs2_glock_seq_next
,
1939 .stop
= gfs2_glock_seq_stop
,
1940 .show
= gfs2_glstats_seq_show
,
1943 static const struct seq_operations gfs2_sbstats_seq_ops
= {
1944 .start
= gfs2_sbstats_seq_start
,
1945 .next
= gfs2_sbstats_seq_next
,
1946 .stop
= gfs2_sbstats_seq_stop
,
1947 .show
= gfs2_sbstats_seq_show
,
1950 #define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL)
1952 static int __gfs2_glocks_open(struct inode
*inode
, struct file
*file
,
1953 const struct seq_operations
*ops
)
1955 int ret
= seq_open_private(file
, ops
, sizeof(struct gfs2_glock_iter
));
1957 struct seq_file
*seq
= file
->private_data
;
1958 struct gfs2_glock_iter
*gi
= seq
->private;
1960 gi
->sdp
= inode
->i_private
;
1961 seq
->buf
= kmalloc(GFS2_SEQ_GOODSIZE
, GFP_KERNEL
| __GFP_NOWARN
);
1963 seq
->size
= GFS2_SEQ_GOODSIZE
;
1969 static int gfs2_glocks_open(struct inode
*inode
, struct file
*file
)
1971 return __gfs2_glocks_open(inode
, file
, &gfs2_glock_seq_ops
);
1974 static int gfs2_glocks_release(struct inode
*inode
, struct file
*file
)
1976 struct seq_file
*seq
= file
->private_data
;
1977 struct gfs2_glock_iter
*gi
= seq
->private;
1980 return seq_release_private(inode
, file
);
1983 static int gfs2_glstats_open(struct inode
*inode
, struct file
*file
)
1985 return __gfs2_glocks_open(inode
, file
, &gfs2_glstats_seq_ops
);
1988 static int gfs2_sbstats_open(struct inode
*inode
, struct file
*file
)
1990 int ret
= seq_open(file
, &gfs2_sbstats_seq_ops
);
1992 struct seq_file
*seq
= file
->private_data
;
1993 seq
->private = inode
->i_private
; /* sdp */
1998 static const struct file_operations gfs2_glocks_fops
= {
1999 .owner
= THIS_MODULE
,
2000 .open
= gfs2_glocks_open
,
2002 .llseek
= seq_lseek
,
2003 .release
= gfs2_glocks_release
,
2006 static const struct file_operations gfs2_glstats_fops
= {
2007 .owner
= THIS_MODULE
,
2008 .open
= gfs2_glstats_open
,
2010 .llseek
= seq_lseek
,
2011 .release
= gfs2_glocks_release
,
2014 static const struct file_operations gfs2_sbstats_fops
= {
2015 .owner
= THIS_MODULE
,
2016 .open
= gfs2_sbstats_open
,
2018 .llseek
= seq_lseek
,
2019 .release
= seq_release
,
2022 int gfs2_create_debugfs_file(struct gfs2_sbd
*sdp
)
2024 struct dentry
*dent
;
2026 dent
= debugfs_create_dir(sdp
->sd_table_name
, gfs2_root
);
2027 if (IS_ERR_OR_NULL(dent
))
2029 sdp
->debugfs_dir
= dent
;
2031 dent
= debugfs_create_file("glocks",
2033 sdp
->debugfs_dir
, sdp
,
2035 if (IS_ERR_OR_NULL(dent
))
2037 sdp
->debugfs_dentry_glocks
= dent
;
2039 dent
= debugfs_create_file("glstats",
2041 sdp
->debugfs_dir
, sdp
,
2042 &gfs2_glstats_fops
);
2043 if (IS_ERR_OR_NULL(dent
))
2045 sdp
->debugfs_dentry_glstats
= dent
;
2047 dent
= debugfs_create_file("sbstats",
2049 sdp
->debugfs_dir
, sdp
,
2050 &gfs2_sbstats_fops
);
2051 if (IS_ERR_OR_NULL(dent
))
2053 sdp
->debugfs_dentry_sbstats
= dent
;
2057 gfs2_delete_debugfs_file(sdp
);
2058 return dent
? PTR_ERR(dent
) : -ENOMEM
;
2061 void gfs2_delete_debugfs_file(struct gfs2_sbd
*sdp
)
2063 if (sdp
->debugfs_dir
) {
2064 if (sdp
->debugfs_dentry_glocks
) {
2065 debugfs_remove(sdp
->debugfs_dentry_glocks
);
2066 sdp
->debugfs_dentry_glocks
= NULL
;
2068 if (sdp
->debugfs_dentry_glstats
) {
2069 debugfs_remove(sdp
->debugfs_dentry_glstats
);
2070 sdp
->debugfs_dentry_glstats
= NULL
;
2072 if (sdp
->debugfs_dentry_sbstats
) {
2073 debugfs_remove(sdp
->debugfs_dentry_sbstats
);
2074 sdp
->debugfs_dentry_sbstats
= NULL
;
2076 debugfs_remove(sdp
->debugfs_dir
);
2077 sdp
->debugfs_dir
= NULL
;
2081 int gfs2_register_debugfs(void)
2083 gfs2_root
= debugfs_create_dir("gfs2", NULL
);
2084 if (IS_ERR(gfs2_root
))
2085 return PTR_ERR(gfs2_root
);
2086 return gfs2_root
? 0 : -ENOMEM
;
2089 void gfs2_unregister_debugfs(void)
2091 debugfs_remove(gfs2_root
);