1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
17 #ifndef CEPH_MDCACHE_H
18 #define CEPH_MDCACHE_H
20 #include "include/types.h"
21 #include "include/filepath.h"
22 #include "include/elist.h"
24 #include "osdc/Filer.h"
28 #include "include/Context.h"
29 #include "events/EMetaBlob.h"
30 #include "RecoveryQueue.h"
31 #include "StrayManager.h"
32 #include "MDSContext.h"
36 #include "messages/MClientRequest.h"
37 #include "messages/MMDSSlaveRequest.h"
50 class MMDSCacheRejoin
;
59 struct MMDSFindInoReply
;
61 struct MMDSOpenInoReply
;
65 class MMDSSlaveRequest
;
68 class MMDSFragmentNotify
;
74 // How many inodes currently in stray dentries
76 // How many stray dentries are currently delayed for purge due to refs
77 l_mdc_num_strays_delayed
,
78 // How many stray dentries are currently being enqueued for purge
79 l_mdc_num_strays_enqueuing
,
81 // How many dentries have ever been added to stray dir
83 // How many dentries have been passed on to PurgeQueue
84 l_mdc_strays_enqueued
,
85 // How many strays have been reintegrated?
86 l_mdc_strays_reintegrated
,
87 // How many strays have been migrated?
88 l_mdc_strays_migrated
,
90 // How many inode sizes currently being recovered
91 l_mdc_num_recovering_processing
,
92 // How many inodes currently waiting to have size recovered
93 l_mdc_num_recovering_enqueued
,
94 // How many inodes waiting with elevated priority for recovery
95 l_mdc_num_recovering_prioritized
,
96 // How many inodes ever started size recovery
97 l_mdc_recovery_started
,
98 // How many inodes ever completed size recovery
99 l_mdc_recovery_completed
,
105 // flags for predirty_journal_parents()
106 static const int PREDIRTY_PRIMARY
= 1; // primary dn, adjust nested accounting
107 static const int PREDIRTY_DIR
= 2; // update parent dir mtime/size
108 static const int PREDIRTY_SHALLOW
= 4; // only go to immediate parent (for easier rollback)
116 LRU lru
; // dentry lru for expiring items from cache
117 LRU bottom_lru
; // dentries that should be trimmed ASAP
119 ceph::unordered_map
<vinodeno_t
,CInode
*> inode_map
; // map of inodes by ino
120 CInode
*root
; // root inode
121 CInode
*myin
; // .ceph/mds%d dir
124 void set_readonly() { readonly
= true; }
126 CInode
*strays
[NUM_STRAY
]; // my stray dir
129 CInode
*get_stray() {
130 return strays
[stray_index
];
133 set
<CInode
*> base_inodes
;
135 std::unique_ptr
<PerfCounters
> logger
;
139 bool exceeded_size_limit
;
142 void advance_stray() {
143 stray_index
= (stray_index
+1)%NUM_STRAY
;
146 void activate_stray_manager();
149 * Call this when you know that a CDentry is ready to be passed
150 * on to StrayManager (i.e. this is a stray you've just created)
152 void notify_stray(CDentry
*dn
) {
153 assert(dn
->get_dir()->get_inode()->is_stray());
154 stray_manager
.eval_stray(dn
);
157 void maybe_eval_stray(CInode
*in
, bool delay
=false);
158 void clear_dirty_bits_for_stray(CInode
* diri
);
160 bool is_readonly() { return readonly
; }
161 void force_readonly();
165 int num_inodes_with_caps
;
167 unsigned max_dir_commit_size
;
169 static file_layout_t
gen_default_file_layout(const MDSMap
&mdsmap
);
170 static file_layout_t
gen_default_log_layout(const MDSMap
&mdsmap
);
172 file_layout_t default_file_layout
;
173 file_layout_t default_log_layout
;
175 void register_perfcounters();
177 // -- client leases --
179 static const int client_lease_pools
= 3;
180 float client_lease_durations
[client_lease_pools
];
182 xlist
<ClientLease
*> client_leases
[client_lease_pools
];
184 void touch_client_lease(ClientLease
*r
, int pool
, utime_t ttl
) {
185 client_leases
[pool
].push_back(&r
->item_lease
);
189 void notify_stray_removed()
191 stray_manager
.notify_stray_removed();
194 void notify_stray_created()
196 stray_manager
.notify_stray_created();
199 void eval_remote(CDentry
*dn
)
201 stray_manager
.eval_remote(dn
);
205 uint64_t last_cap_id
;
210 struct discover_info_t
{
222 tid(0), mds(-1), snap(CEPH_NOSNAP
), basei(NULL
),
223 want_base_dir(false), want_xlocked(false) {}
226 basei
->put(MDSCacheObject::PIN_DISCOVERBASE
);
228 void pin_base(CInode
*b
) {
230 basei
->get(MDSCacheObject::PIN_DISCOVERBASE
);
234 map
<ceph_tid_t
, discover_info_t
> discovers
;
235 ceph_tid_t discover_last_tid
;
237 void _send_discover(discover_info_t
& dis
);
238 discover_info_t
& _create_discover(mds_rank_t mds
) {
239 ceph_tid_t t
= ++discover_last_tid
;
240 discover_info_t
& d
= discovers
[t
];
247 map
<int, map
<inodeno_t
, list
<MDSInternalContextBase
*> > > waiting_for_base_ino
;
249 void discover_base_ino(inodeno_t want_ino
, MDSInternalContextBase
*onfinish
, mds_rank_t from
=MDS_RANK_NONE
);
250 void discover_dir_frag(CInode
*base
, frag_t approx_fg
, MDSInternalContextBase
*onfinish
,
251 mds_rank_t from
=MDS_RANK_NONE
);
252 void discover_path(CInode
*base
, snapid_t snap
, filepath want_path
, MDSInternalContextBase
*onfinish
,
253 bool want_xlocked
=false, mds_rank_t from
=MDS_RANK_NONE
);
254 void discover_path(CDir
*base
, snapid_t snap
, filepath want_path
, MDSInternalContextBase
*onfinish
,
255 bool want_xlocked
=false);
256 void kick_discovers(mds_rank_t who
); // after a failure.
261 /* subtree keys and each tree's non-recursive nested subtrees (the "bounds") */
262 map
<CDir
*,set
<CDir
*> > subtrees
;
263 map
<CInode
*,list
<pair
<CDir
*,CDir
*> > > projected_subtree_renames
; // renamed ino -> target dir
265 // adjust subtree auth specification
267 // imports/exports/nested_exports
268 // join/split subtrees as appropriate
270 bool is_subtrees() { return !subtrees
.empty(); }
271 void list_subtrees(list
<CDir
*>& ls
);
272 void adjust_subtree_auth(CDir
*root
, mds_authority_t auth
, bool do_eval
=true);
273 void adjust_subtree_auth(CDir
*root
, mds_rank_t a
, mds_rank_t b
=CDIR_AUTH_UNKNOWN
, bool do_eval
=true) {
274 adjust_subtree_auth(root
, mds_authority_t(a
,b
), do_eval
);
276 void adjust_bounded_subtree_auth(CDir
*dir
, set
<CDir
*>& bounds
, mds_authority_t auth
);
277 void adjust_bounded_subtree_auth(CDir
*dir
, set
<CDir
*>& bounds
, mds_rank_t a
) {
278 adjust_bounded_subtree_auth(dir
, bounds
, mds_authority_t(a
, CDIR_AUTH_UNKNOWN
));
280 void adjust_bounded_subtree_auth(CDir
*dir
, vector
<dirfrag_t
>& bounds
, mds_authority_t auth
);
281 void adjust_bounded_subtree_auth(CDir
*dir
, vector
<dirfrag_t
>& bounds
, mds_rank_t a
) {
282 adjust_bounded_subtree_auth(dir
, bounds
, mds_authority_t(a
, CDIR_AUTH_UNKNOWN
));
284 void map_dirfrag_set(list
<dirfrag_t
>& dfs
, set
<CDir
*>& result
);
285 void try_subtree_merge(CDir
*root
);
286 void try_subtree_merge_at(CDir
*root
, bool do_eval
=true);
287 void subtree_merge_writebehind_finish(CInode
*in
, MutationRef
& mut
);
288 void eval_subtree_root(CInode
*diri
);
289 CDir
*get_subtree_root(CDir
*dir
);
290 CDir
*get_projected_subtree_root(CDir
*dir
);
291 bool is_leaf_subtree(CDir
*dir
) {
292 assert(subtrees
.count(dir
));
293 return subtrees
[dir
].empty();
295 void remove_subtree(CDir
*dir
);
296 bool is_subtree(CDir
*root
) {
297 return subtrees
.count(root
);
299 void get_subtree_bounds(CDir
*root
, set
<CDir
*>& bounds
);
300 void get_wouldbe_subtree_bounds(CDir
*root
, set
<CDir
*>& bounds
);
301 void verify_subtree_bounds(CDir
*root
, const set
<CDir
*>& bounds
);
302 void verify_subtree_bounds(CDir
*root
, const list
<dirfrag_t
>& bounds
);
304 void project_subtree_rename(CInode
*diri
, CDir
*olddir
, CDir
*newdir
);
305 void adjust_subtree_after_rename(CInode
*diri
, CDir
*olddir
,
306 bool pop
, bool imported
= false);
308 void get_auth_subtrees(set
<CDir
*>& s
);
309 void get_fullauth_subtrees(set
<CDir
*>& s
);
312 int num_subtrees_fullauth();
313 int num_subtrees_fullnonauth();
317 // delayed cache expire
318 map
<CDir
*, map
<mds_rank_t
, MCacheExpire
*> > delayed_expire
; // subtree root -> expire msg
322 ceph::unordered_map
<metareqid_t
, MDRequestRef
> active_requests
;
325 int get_num_client_requests();
327 MDRequestRef
request_start(MClientRequest
*req
);
328 MDRequestRef
request_start_slave(metareqid_t rid
, __u32 attempt
, Message
*m
);
329 MDRequestRef
request_start_internal(int op
);
330 bool have_request(metareqid_t rid
) {
331 return active_requests
.count(rid
);
333 MDRequestRef
request_get(metareqid_t rid
);
334 void request_pin_ref(MDRequestRef
& r
, CInode
*ref
, vector
<CDentry
*>& trace
);
335 void request_finish(MDRequestRef
& mdr
);
336 void request_forward(MDRequestRef
& mdr
, mds_rank_t mds
, int port
=0);
337 void dispatch_request(MDRequestRef
& mdr
);
338 void request_drop_foreign_locks(MDRequestRef
& mdr
);
339 void request_drop_non_rdlocks(MDRequestRef
& r
);
340 void request_drop_locks(MDRequestRef
& r
);
341 void request_cleanup(MDRequestRef
& r
);
343 void request_kill(MDRequestRef
& r
); // called when session closes
345 // journal/snap helpers
346 CInode
*pick_inode_snap(CInode
*in
, snapid_t follows
);
347 CInode
*cow_inode(CInode
*in
, snapid_t last
);
348 void journal_cow_dentry(MutationImpl
*mut
, EMetaBlob
*metablob
, CDentry
*dn
,
349 snapid_t follows
=CEPH_NOSNAP
,
350 CInode
**pcow_inode
=0, CDentry::linkage_t
*dnl
=0);
351 void journal_cow_inode(MutationRef
& mut
, EMetaBlob
*metablob
, CInode
*in
, snapid_t follows
=CEPH_NOSNAP
,
352 CInode
**pcow_inode
=0);
353 void journal_dirty_inode(MutationImpl
*mut
, EMetaBlob
*metablob
, CInode
*in
, snapid_t follows
=CEPH_NOSNAP
);
355 void project_rstat_inode_to_frag(CInode
*cur
, CDir
*parent
, snapid_t first
,
356 int linkunlink
, SnapRealm
*prealm
);
357 void _project_rstat_inode_to_frag(inode_t
& inode
, snapid_t ofirst
, snapid_t last
,
358 CDir
*parent
, int linkunlink
, bool update_inode
);
359 void project_rstat_frag_to_inode(nest_info_t
& rstat
, nest_info_t
& accounted_rstat
,
360 snapid_t ofirst
, snapid_t last
,
361 CInode
*pin
, bool cow_head
);
362 void broadcast_quota_to_client(CInode
*in
);
363 void predirty_journal_parents(MutationRef mut
, EMetaBlob
*blob
,
364 CInode
*in
, CDir
*parent
,
365 int flags
, int linkunlink
=0,
366 snapid_t follows
=CEPH_NOSNAP
);
369 void add_uncommitted_master(metareqid_t reqid
, LogSegment
*ls
, set
<mds_rank_t
> &slaves
, bool safe
=false) {
370 uncommitted_masters
[reqid
].ls
= ls
;
371 uncommitted_masters
[reqid
].slaves
= slaves
;
372 uncommitted_masters
[reqid
].safe
= safe
;
374 void wait_for_uncommitted_master(metareqid_t reqid
, MDSInternalContextBase
*c
) {
375 uncommitted_masters
[reqid
].waiters
.push_back(c
);
377 bool have_uncommitted_master(metareqid_t reqid
, mds_rank_t from
) {
378 auto p
= uncommitted_masters
.find(reqid
);
379 return p
!= uncommitted_masters
.end() && p
->second
.slaves
.count(from
) > 0;
381 void log_master_commit(metareqid_t reqid
);
382 void logged_master_update(metareqid_t reqid
);
383 void _logged_master_commit(metareqid_t reqid
);
384 void committed_master_slave(metareqid_t r
, mds_rank_t from
);
385 void finish_committed_masters();
387 void _logged_slave_commit(mds_rank_t from
, metareqid_t reqid
);
391 set
<mds_rank_t
> recovery_set
;
394 void set_recovery_set(set
<mds_rank_t
>& s
);
395 void handle_mds_failure(mds_rank_t who
);
396 void handle_mds_recovery(mds_rank_t who
);
400 // from EImportStart w/o EImportFinish during journal replay
401 map
<dirfrag_t
, vector
<dirfrag_t
> > my_ambiguous_imports
;
403 map
<mds_rank_t
, map
<dirfrag_t
, vector
<dirfrag_t
> > > other_ambiguous_imports
;
405 map
<mds_rank_t
, map
<metareqid_t
, MDSlaveUpdate
*> > uncommitted_slave_updates
; // slave: for replay.
406 map
<CInode
*, int> uncommitted_slave_rename_olddir
; // slave: preserve the non-auth dir until seeing commit.
407 map
<CInode
*, int> uncommitted_slave_unlink
; // slave: preserve the unlinked inode until seeing commit.
409 // track master requests whose slaves haven't acknowledged commit
411 set
<mds_rank_t
> slaves
;
413 list
<MDSInternalContextBase
*> waiters
;
417 umaster() : ls(NULL
), safe(false), committing(false), recovering(false) {}
419 map
<metareqid_t
, umaster
> uncommitted_masters
; // master: req -> slave set
421 set
<metareqid_t
> pending_masters
;
422 map
<int, set
<metareqid_t
> > ambiguous_slave_updates
;
424 friend class ESlaveUpdate
;
425 friend class ECommitted
;
427 bool resolves_pending
;
428 set
<mds_rank_t
> resolve_gather
; // nodes i need resolves from
429 set
<mds_rank_t
> resolve_ack_gather
; // nodes i need a resolve_ack from
430 map
<metareqid_t
, mds_rank_t
> need_resolve_rollback
; // rollbacks i'm writing to the journal
431 map
<mds_rank_t
, MMDSResolve
*> delayed_resolve
;
433 void handle_resolve(MMDSResolve
*m
);
434 void handle_resolve_ack(MMDSResolveAck
*m
);
435 void process_delayed_resolve();
436 void discard_delayed_resolve(mds_rank_t who
);
437 void maybe_resolve_finish();
438 void disambiguate_my_imports();
439 void disambiguate_other_imports();
440 void trim_unlinked_inodes();
441 void add_uncommitted_slave_update(metareqid_t reqid
, mds_rank_t master
, MDSlaveUpdate
*);
442 void finish_uncommitted_slave_update(metareqid_t reqid
, mds_rank_t master
);
443 MDSlaveUpdate
* get_uncommitted_slave_update(metareqid_t reqid
, mds_rank_t master
);
445 void recalc_auth_bits(bool replay
);
446 void remove_inode_recursive(CInode
*in
);
448 bool is_ambiguous_slave_update(metareqid_t reqid
, mds_rank_t master
) {
449 auto p
= ambiguous_slave_updates
.find(master
);
450 return p
!= ambiguous_slave_updates
.end() && p
->second
.count(reqid
);
452 void add_ambiguous_slave_update(metareqid_t reqid
, mds_rank_t master
) {
453 ambiguous_slave_updates
[master
].insert(reqid
);
455 void remove_ambiguous_slave_update(metareqid_t reqid
, mds_rank_t master
) {
456 auto p
= ambiguous_slave_updates
.find(master
);
457 auto q
= p
->second
.find(reqid
);
458 assert(q
!= p
->second
.end());
460 if (p
->second
.empty())
461 ambiguous_slave_updates
.erase(p
);
464 void add_rollback(metareqid_t reqid
, mds_rank_t master
) {
465 need_resolve_rollback
[reqid
] = master
;
467 void finish_rollback(metareqid_t reqid
);
470 void add_ambiguous_import(dirfrag_t base
, const vector
<dirfrag_t
>& bounds
);
471 void add_ambiguous_import(CDir
*base
, const set
<CDir
*>& bounds
);
472 bool have_ambiguous_import(dirfrag_t base
) {
473 return my_ambiguous_imports
.count(base
);
475 void get_ambiguous_import_bounds(dirfrag_t base
, vector
<dirfrag_t
>& bounds
) {
476 assert(my_ambiguous_imports
.count(base
));
477 bounds
= my_ambiguous_imports
[base
];
479 void cancel_ambiguous_import(CDir
*);
480 void finish_ambiguous_import(dirfrag_t dirino
);
481 void resolve_start(MDSInternalContext
*resolve_done_
);
482 void send_resolves();
483 void send_slave_resolves();
484 void send_subtree_resolves();
485 void maybe_send_pending_resolves() {
486 if (resolves_pending
)
487 send_subtree_resolves();
490 void _move_subtree_map_bound(dirfrag_t df
, dirfrag_t oldparent
, dirfrag_t newparent
,
491 map
<dirfrag_t
,vector
<dirfrag_t
> >& subtrees
);
492 ESubtreeMap
*create_subtree_map();
495 void clean_open_file_lists();
499 bool rejoins_pending
;
500 set
<mds_rank_t
> rejoin_gather
; // nodes from whom i need a rejoin
501 set
<mds_rank_t
> rejoin_sent
; // nodes i sent a rejoin to
502 set
<mds_rank_t
> rejoin_ack_sent
; // nodes i sent a rejoin to
503 set
<mds_rank_t
> rejoin_ack_gather
; // nodes from whom i need a rejoin ack
504 map
<mds_rank_t
,map
<inodeno_t
,map
<client_t
,Capability::Import
> > > rejoin_imported_caps
;
505 map
<inodeno_t
,pair
<mds_rank_t
,map
<client_t
,Capability::Export
> > > rejoin_slave_exports
;
506 map
<client_t
,entity_inst_t
> rejoin_client_map
;
508 map
<inodeno_t
,map
<client_t
,cap_reconnect_t
> > cap_exports
; // ino -> client -> capex
509 map
<inodeno_t
,mds_rank_t
> cap_export_targets
; // ino -> auth mds
511 map
<inodeno_t
,map
<client_t
,map
<mds_rank_t
,cap_reconnect_t
> > > cap_imports
; // ino -> client -> frommds -> capex
512 set
<inodeno_t
> cap_imports_missing
;
513 map
<inodeno_t
, list
<MDSInternalContextBase
*> > cap_reconnect_waiters
;
514 int cap_imports_num_opening
;
516 set
<CInode
*> rejoin_undef_inodes
;
517 set
<CInode
*> rejoin_potential_updated_scatterlocks
;
518 set
<CDir
*> rejoin_undef_dirfrags
;
519 map
<mds_rank_t
, set
<CInode
*> > rejoin_unlinked_inodes
;
521 vector
<CInode
*> rejoin_recover_q
, rejoin_check_q
;
522 list
<SimpleLock
*> rejoin_eval_locks
;
523 list
<MDSInternalContextBase
*> rejoin_waiters
;
525 void rejoin_walk(CDir
*dir
, MMDSCacheRejoin
*rejoin
);
526 void handle_cache_rejoin(MMDSCacheRejoin
*m
);
527 void handle_cache_rejoin_weak(MMDSCacheRejoin
*m
);
528 CInode
* rejoin_invent_inode(inodeno_t ino
, snapid_t last
);
529 CDir
* rejoin_invent_dirfrag(dirfrag_t df
);
530 void handle_cache_rejoin_strong(MMDSCacheRejoin
*m
);
531 void rejoin_scour_survivor_replicas(mds_rank_t from
, MMDSCacheRejoin
*ack
,
532 set
<vinodeno_t
>& acked_inodes
,
533 set
<SimpleLock
*>& gather_locks
);
534 void handle_cache_rejoin_ack(MMDSCacheRejoin
*m
);
535 void rejoin_send_acks();
536 void rejoin_trim_undef_inodes();
537 void maybe_send_pending_rejoins() {
539 rejoin_send_rejoins();
541 std::unique_ptr
<MDSInternalContext
> rejoin_done
;
542 std::unique_ptr
<MDSInternalContext
> resolve_done
;
544 void rejoin_start(MDSInternalContext
*rejoin_done_
);
545 void rejoin_gather_finish();
546 void rejoin_send_rejoins();
547 void rejoin_export_caps(inodeno_t ino
, client_t client
, const cap_reconnect_t
& icr
,
549 cap_exports
[ino
][client
] = icr
;
550 cap_export_targets
[ino
] = target
;
552 void rejoin_recovered_caps(inodeno_t ino
, client_t client
, const cap_reconnect_t
& icr
,
553 mds_rank_t frommds
=MDS_RANK_NONE
) {
554 cap_imports
[ino
][client
][frommds
] = icr
;
556 const cap_reconnect_t
*get_replay_cap_reconnect(inodeno_t ino
, client_t client
) {
557 if (cap_imports
.count(ino
) &&
558 cap_imports
[ino
].count(client
) &&
559 cap_imports
[ino
][client
].count(MDS_RANK_NONE
)) {
560 return &cap_imports
[ino
][client
][MDS_RANK_NONE
];
564 void remove_replay_cap_reconnect(inodeno_t ino
, client_t client
) {
565 assert(cap_imports
[ino
].size() == 1);
566 assert(cap_imports
[ino
][client
].size() == 1);
567 cap_imports
.erase(ino
);
569 void wait_replay_cap_reconnect(inodeno_t ino
, MDSInternalContextBase
*c
) {
570 cap_reconnect_waiters
[ino
].push_back(c
);
573 // [reconnect/rejoin caps]
574 struct reconnected_cap_info_t
{
576 snapid_t snap_follows
;
578 reconnected_cap_info_t() :
579 realm_ino(0), snap_follows(0), dirty_caps(0) {}
581 map
<inodeno_t
,map
<client_t
, reconnected_cap_info_t
> > reconnected_caps
; // inode -> client -> snap_follows,realmino
582 map
<inodeno_t
,map
<client_t
, snapid_t
> > reconnected_snaprealms
; // realmino -> client -> realmseq
584 void add_reconnected_cap(client_t client
, inodeno_t ino
, const cap_reconnect_t
& icr
) {
585 reconnected_cap_info_t
&info
= reconnected_caps
[ino
][client
];
586 info
.realm_ino
= inodeno_t(icr
.capinfo
.snaprealm
);
587 info
.snap_follows
= icr
.snap_follows
;
589 void set_reconnected_dirty_caps(client_t client
, inodeno_t ino
, int dirty
) {
590 reconnected_cap_info_t
&info
= reconnected_caps
[ino
][client
];
591 info
.dirty_caps
|= dirty
;
593 void add_reconnected_snaprealm(client_t client
, inodeno_t ino
, snapid_t seq
) {
594 reconnected_snaprealms
[ino
][client
] = seq
;
597 friend class C_MDC_RejoinOpenInoFinish
;
598 friend class C_MDC_RejoinSessionsOpened
;
599 void rejoin_open_ino_finish(inodeno_t ino
, int ret
);
600 void rejoin_open_sessions_finish(map
<client_t
,entity_inst_t
> client_map
,
601 map
<client_t
,uint64_t>& sseqmap
);
602 bool process_imported_caps();
603 void choose_lock_states_and_reconnect_caps();
604 void prepare_realm_split(SnapRealm
*realm
, client_t client
, inodeno_t ino
,
605 map
<client_t
,MClientSnap
*>& splits
);
606 void do_realm_invalidate_and_update_notify(CInode
*in
, int snapop
, bool nosend
=false);
607 void send_snaps(map
<client_t
,MClientSnap
*>& splits
);
608 Capability
* rejoin_import_cap(CInode
*in
, client_t client
, const cap_reconnect_t
& icr
, mds_rank_t frommds
);
609 void finish_snaprealm_reconnect(client_t client
, SnapRealm
*realm
, snapid_t seq
);
610 void try_reconnect_cap(CInode
*in
, Session
*session
);
611 void export_remaining_imported_caps();
613 // cap imports. delayed snap parent opens.
614 // realm inode -> client -> cap inodes needing to split to this realm
615 map
<CInode
*,set
<CInode
*> > missing_snap_parents
;
616 map
<client_t
,set
<CInode
*> > delayed_imported_caps
;
618 void do_cap_import(Session
*session
, CInode
*in
, Capability
*cap
,
619 uint64_t p_cap_id
, ceph_seq_t p_seq
, ceph_seq_t p_mseq
,
620 int peer
, int p_flags
);
621 void do_delayed_cap_imports();
622 void rebuild_need_snapflush(CInode
*head_in
, SnapRealm
*realm
, client_t client
,
623 snapid_t snap_follows
);
624 void check_realm_past_parents(SnapRealm
*realm
, bool reconnect
);
625 void open_snap_parents();
627 bool open_undef_inodes_dirfrags();
628 void opened_undef_inode(CInode
*in
);
629 void opened_undef_dirfrag(CDir
*dir
) {
630 rejoin_undef_dirfrags
.erase(dir
);
633 void reissue_all_caps();
637 friend class Migrator
;
638 friend class MDBalancer
;
640 // StrayManager needs to be able to remove_inode() from us
641 // when it is done purging
642 friend class StrayManager
;
644 // File size recovery
646 RecoveryQueue recovery_queue
;
647 void identify_files_to_recover();
649 void start_files_to_recover();
650 void do_file_recover();
651 void queue_file_recover(CInode
*in
);
652 void _queued_file_recover_cow(CInode
*in
, MutationRef
& mut
);
655 std::unique_ptr
<Migrator
> migrator
;
658 explicit MDCache(MDSRank
*m
, PurgeQueue
&purge_queue_
);
665 CInode
*get_root() { return root
; }
666 CInode
*get_myin() { return myin
; }
669 void set_cache_size(size_t max
) { lru
.lru_set_max(max
); }
670 size_t get_cache_size() { return lru
.lru_get_size(); }
673 bool trim(int max
=-1, int count
=-1); // trim cache
674 bool trim_dentry(CDentry
*dn
, map
<mds_rank_t
, MCacheExpire
*>& expiremap
);
675 void trim_dirfrag(CDir
*dir
, CDir
*con
,
676 map
<mds_rank_t
, MCacheExpire
*>& expiremap
);
677 bool trim_inode(CDentry
*dn
, CInode
*in
, CDir
*con
,
678 map
<mds_rank_t
,class MCacheExpire
*>& expiremap
);
679 void send_expire_messages(map
<mds_rank_t
, MCacheExpire
*>& expiremap
);
680 void trim_non_auth(); // trim out trimmable non-auth items
681 bool trim_non_auth_subtree(CDir
*directory
);
682 void standby_trim_segment(LogSegment
*ls
);
683 void try_trim_non_auth_subtree(CDir
*dir
);
684 bool can_trim_non_auth_dirfrag(CDir
*dir
) {
685 return my_ambiguous_imports
.count((dir
)->dirfrag()) == 0 &&
686 uncommitted_slave_rename_olddir
.count(dir
->inode
) == 0;
690 * For all unreferenced inodes, dirs, dentries below an inode, compose
691 * expiry messages. This is used when giving up all replicas of entities
692 * for an MDS peer in the 'stopping' state, such that the peer can
693 * empty its cache and finish shutting down.
695 * We have to make sure we're only expiring un-referenced items to
696 * avoid interfering with ongoing stray-movement (we can't distinguish
697 * between the "moving my strays" and "waiting for my cache to empty"
698 * phases within 'stopping')
700 * @return false if we completed cleanly, true if caller should stop
701 * expiring because we hit something with refs.
703 bool expire_recursive(
705 std::map
<mds_rank_t
, MCacheExpire
*>& expiremap
);
707 void trim_client_leases();
708 void check_memory_usage();
710 utime_t last_recall_state
;
714 set
<inodeno_t
> shutdown_exported_strays
;
716 void shutdown_start();
717 void shutdown_check();
718 bool shutdown_pass();
719 bool shutdown_export_strays();
720 bool shutdown(); // clear cache (ie at shutodwn)
722 bool did_shutdown_log_cap
;
725 bool have_inode(vinodeno_t vino
) {
726 return inode_map
.count(vino
) ? true:false;
728 bool have_inode(inodeno_t ino
, snapid_t snap
=CEPH_NOSNAP
) {
729 return have_inode(vinodeno_t(ino
, snap
));
731 CInode
* get_inode(vinodeno_t vino
) {
732 if (have_inode(vino
))
733 return inode_map
[vino
];
736 CInode
* get_inode(inodeno_t ino
, snapid_t s
=CEPH_NOSNAP
) {
737 return get_inode(vinodeno_t(ino
, s
));
740 CDir
* get_dirfrag(dirfrag_t df
) {
741 CInode
*in
= get_inode(df
.ino
);
744 return in
->get_dirfrag(df
.frag
);
746 CDir
* get_dirfrag(inodeno_t ino
, const string
& dn
) {
747 CInode
*in
= get_inode(ino
);
750 frag_t fg
= in
->pick_dirfrag(dn
);
751 return in
->get_dirfrag(fg
);
753 CDir
* get_force_dirfrag(dirfrag_t df
, bool replay
) {
754 CInode
*diri
= get_inode(df
.ino
);
757 CDir
*dir
= force_dir_fragment(diri
, df
.frag
, replay
);
759 dir
= diri
->get_dirfrag(df
.frag
);
763 MDSCacheObject
*get_object(MDSCacheObjectInfo
&info
);
768 void add_inode(CInode
*in
);
770 void remove_inode(CInode
*in
);
772 void touch_inode(CInode
*in
) {
773 if (in
->get_parent_dn())
774 touch_dentry(in
->get_projected_parent_dn());
777 void touch_dentry(CDentry
*dn
) {
778 if (dn
->state_test(CDentry::STATE_BOTTOMLRU
)) {
779 bottom_lru
.lru_midtouch(dn
);
784 lru
.lru_midtouch(dn
);
787 void touch_dentry_bottom(CDentry
*dn
) {
788 if (dn
->state_test(CDentry::STATE_BOTTOMLRU
))
790 lru
.lru_bottouch(dn
);
794 void inode_remove_replica(CInode
*in
, mds_rank_t rep
, bool rejoin
,
795 set
<SimpleLock
*>& gather_locks
);
796 void dentry_remove_replica(CDentry
*dn
, mds_rank_t rep
, set
<SimpleLock
*>& gather_locks
);
798 void rename_file(CDentry
*srcdn
, CDentry
*destdn
);
802 void truncate_inode(CInode
*in
, LogSegment
*ls
);
803 void _truncate_inode(CInode
*in
, LogSegment
*ls
);
804 void truncate_inode_finish(CInode
*in
, LogSegment
*ls
);
805 void truncate_inode_logged(CInode
*in
, MutationRef
& mut
);
807 void add_recovered_truncate(CInode
*in
, LogSegment
*ls
);
808 void remove_recovered_truncate(CInode
*in
, LogSegment
*ls
);
809 void start_recovered_truncates();
813 CDir
*get_auth_container(CDir
*in
);
814 CDir
*get_export_container(CDir
*dir
);
815 void find_nested_exports(CDir
*dir
, set
<CDir
*>& s
);
816 void find_nested_exports_under(CDir
*import
, CDir
*dir
, set
<CDir
*>& s
);
820 bool opening_root
, open
;
821 list
<MDSInternalContextBase
*> waiting_for_open
;
825 void create_unlinked_system_inode(CInode
*in
, inodeno_t ino
,
827 CInode
*create_system_inode(inodeno_t ino
, int mode
);
828 CInode
*create_root_inode();
830 void create_empty_hierarchy(MDSGather
*gather
);
831 void create_mydir_hierarchy(MDSGather
*gather
);
833 bool is_open() { return open
; }
834 void wait_for_open(MDSInternalContextBase
*c
) {
835 waiting_for_open
.push_back(c
);
838 void open_root_inode(MDSInternalContextBase
*c
);
840 void open_mydir_inode(MDSInternalContextBase
*c
);
841 void populate_mydir();
843 void _create_system_file(CDir
*dir
, const char *name
, CInode
*in
, MDSInternalContextBase
*fin
);
844 void _create_system_file_finish(MutationRef
& mut
, CDentry
*dn
,
845 version_t dpv
, MDSInternalContextBase
*fin
);
847 void open_foreign_mdsdir(inodeno_t ino
, MDSInternalContextBase
*c
);
848 CDir
*get_stray_dir(CInode
*in
);
849 CDentry
*get_or_create_stray_dentry(CInode
*in
);
851 MDSInternalContextBase
*_get_waiter(MDRequestRef
& mdr
, Message
*req
, MDSInternalContextBase
*fin
);
854 * Find the given dentry (and whether it exists or not), its ancestors,
855 * and get them all into memory and usable on this MDS. This function
856 * makes a best-effort attempt to load everything; if it needs to
857 * go away and do something then it will put the request on a waitlist.
858 * It prefers the mdr, then the req, then the fin. (At least one of these
861 * At least one of the params mdr, req, and fin must be non-null.
863 * @param mdr The MDRequest associated with the path. Can be null.
864 * @param req The Message associated with the path. Can be null.
865 * @param fin The Context associated with the path. Can be null.
866 * @param path The path to traverse to.
867 * @param pdnvec Data return parameter -- on success, contains a
868 * vector of dentries. On failure, is either empty or contains the
869 * full trace of traversable dentries.
870 * @param pin Data return parameter -- if successful, points to the inode
871 * associated with filepath. If unsuccessful, is null.
872 * @param onfail Specifies different lookup failure behaviors. If set to
873 * MDS_TRAVERSE_DISCOVERXLOCK, path_traverse will succeed on null
874 * dentries (instead of returning -ENOENT). If set to
875 * MDS_TRAVERSE_FORWARD, it will forward the request to the auth
876 * MDS if that becomes appropriate (ie, if it doesn't know the contents
877 * of a directory). If set to MDS_TRAVERSE_DISCOVER, it
878 * will attempt to look up the path from a different MDS (and bring them
879 * into its cache as replicas).
881 * @returns 0 on success, 1 on "not done yet", 2 on "forwarding", -errno otherwise.
882 * If it returns 1, the requester associated with this call has been placed
883 * on the appropriate waitlist, and it should unwind itself and back out.
884 * If it returns 2 the request has been forwarded, and again the requester
885 * should unwind itself and back out.
887 int path_traverse(MDRequestRef
& mdr
, Message
*req
, MDSInternalContextBase
*fin
, const filepath
& path
,
888 vector
<CDentry
*> *pdnvec
, CInode
**pin
, int onfail
);
890 CInode
*cache_traverse(const filepath
& path
);
892 void open_remote_dirfrag(CInode
*diri
, frag_t fg
, MDSInternalContextBase
*fin
);
893 CInode
*get_dentry_inode(CDentry
*dn
, MDRequestRef
& mdr
, bool projected
=false);
895 bool parallel_fetch(map
<inodeno_t
,filepath
>& pathmap
, set
<inodeno_t
>& missing
);
896 bool parallel_fetch_traverse_dir(inodeno_t ino
, filepath
& path
,
897 set
<CDir
*>& fetch_queue
, set
<inodeno_t
>& missing
,
898 C_GatherBuilder
&gather_bld
);
900 void open_remote_dentry(CDentry
*dn
, bool projected
, MDSInternalContextBase
*fin
,
901 bool want_xlocked
=false);
902 void _open_remote_dentry_finish(CDentry
*dn
, inodeno_t ino
, MDSInternalContextBase
*fin
,
903 bool want_xlocked
, int r
);
905 void make_trace(vector
<CDentry
*>& trace
, CInode
*in
);
908 struct open_ino_info_t
{
909 vector
<inode_backpointer_t
> ancestors
;
910 set
<mds_rank_t
> checked
;
912 mds_rank_t auth_hint
;
914 bool fetch_backtrace
;
921 list
<MDSInternalContextBase
*> waiters
;
922 open_ino_info_t() : checking(MDS_RANK_NONE
), auth_hint(MDS_RANK_NONE
),
923 check_peers(true), fetch_backtrace(true), discover(false),
924 want_replica(false), want_xlocked(false), tid(0), pool(-1),
927 ceph_tid_t open_ino_last_tid
;
928 map
<inodeno_t
,open_ino_info_t
> opening_inodes
;
930 void _open_ino_backtrace_fetched(inodeno_t ino
, bufferlist
& bl
, int err
);
931 void _open_ino_parent_opened(inodeno_t ino
, int ret
);
932 void _open_ino_traverse_dir(inodeno_t ino
, open_ino_info_t
& info
, int err
);
933 void _open_ino_fetch_dir(inodeno_t ino
, MMDSOpenIno
*m
, CDir
*dir
, bool parent
);
934 int open_ino_traverse_dir(inodeno_t ino
, MMDSOpenIno
*m
,
935 vector
<inode_backpointer_t
>& ancestors
,
936 bool discover
, bool want_xlocked
, mds_rank_t
*hint
);
937 void open_ino_finish(inodeno_t ino
, open_ino_info_t
& info
, int err
);
938 void do_open_ino(inodeno_t ino
, open_ino_info_t
& info
, int err
);
939 void do_open_ino_peer(inodeno_t ino
, open_ino_info_t
& info
);
940 void handle_open_ino(MMDSOpenIno
*m
, int err
=0);
941 void handle_open_ino_reply(MMDSOpenInoReply
*m
);
942 friend class C_IO_MDC_OpenInoBacktraceFetched
;
943 friend struct C_MDC_OpenInoTraverseDir
;
944 friend struct C_MDC_OpenInoParentOpened
;
947 void kick_open_ino_peers(mds_rank_t who
);
948 void open_ino(inodeno_t ino
, int64_t pool
, MDSInternalContextBase
*fin
,
949 bool want_replica
=true, bool want_xlocked
=false);
951 // -- find_ino_peer --
952 struct find_ino_peer_info_t
{
955 MDSInternalContextBase
*fin
;
958 set
<mds_rank_t
> checked
;
960 find_ino_peer_info_t() : tid(0), fin(NULL
), hint(MDS_RANK_NONE
), checking(MDS_RANK_NONE
) {}
963 map
<ceph_tid_t
, find_ino_peer_info_t
> find_ino_peer
;
964 ceph_tid_t find_ino_peer_last_tid
;
966 void find_ino_peers(inodeno_t ino
, MDSInternalContextBase
*c
, mds_rank_t hint
=MDS_RANK_NONE
);
967 void _do_find_ino_peer(find_ino_peer_info_t
& fip
);
968 void handle_find_ino(MMDSFindIno
*m
);
969 void handle_find_ino_reply(MMDSFindInoReply
*m
);
970 void kick_find_ino_peers(mds_rank_t who
);
974 void snaprealm_create(MDRequestRef
& mdr
, CInode
*in
);
975 void _snaprealm_create_finish(MDRequestRef
& mdr
, MutationRef
& mut
, CInode
*in
);
979 void fetch_backtrace(inodeno_t ino
, int64_t pool
, bufferlist
& bl
, Context
*fin
);
980 uint64_t get_num_strays() const { return stray_manager
.get_num_strays(); }
983 void scan_stray_dir(dirfrag_t next
=dirfrag_t());
984 StrayManager stray_manager
;
985 friend struct C_MDC_RetryScanStray
;
986 friend class C_IO_MDC_FetchedBacktrace
;
990 void dispatch(Message
*m
);
994 void handle_discover(MDiscover
*dis
);
995 void handle_discover_reply(MDiscoverReply
*m
);
996 friend class C_MDC_Join
;
999 void replicate_dir(CDir
*dir
, mds_rank_t to
, bufferlist
& bl
) {
1000 dirfrag_t df
= dir
->dirfrag();
1002 dir
->encode_replica(to
, bl
);
1004 void replicate_dentry(CDentry
*dn
, mds_rank_t to
, bufferlist
& bl
) {
1005 ::encode(dn
->name
, bl
);
1006 ::encode(dn
->last
, bl
);
1007 dn
->encode_replica(to
, bl
);
1009 void replicate_inode(CInode
*in
, mds_rank_t to
, bufferlist
& bl
,
1010 uint64_t features
) {
1011 ::encode(in
->inode
.ino
, bl
); // bleh, minor assymetry here
1012 ::encode(in
->last
, bl
);
1013 in
->encode_replica(to
, bl
, features
);
1016 CDir
* add_replica_dir(bufferlist::iterator
& p
, CInode
*diri
, mds_rank_t from
, list
<MDSInternalContextBase
*>& finished
);
1017 CDir
* forge_replica_dir(CInode
*diri
, frag_t fg
, mds_rank_t from
);
1018 CDentry
*add_replica_dentry(bufferlist::iterator
& p
, CDir
*dir
, list
<MDSInternalContextBase
*>& finished
);
1019 CInode
*add_replica_inode(bufferlist::iterator
& p
, CDentry
*dn
, list
<MDSInternalContextBase
*>& finished
);
1021 void replicate_stray(CDentry
*straydn
, mds_rank_t who
, bufferlist
& bl
);
1022 CDentry
*add_replica_stray(bufferlist
&bl
, mds_rank_t from
);
1026 void send_dentry_link(CDentry
*dn
, MDRequestRef
& mdr
);
1027 void send_dentry_unlink(CDentry
*dn
, CDentry
*straydn
, MDRequestRef
& mdr
);
1029 void handle_dentry_link(MDentryLink
*m
);
1030 void handle_dentry_unlink(MDentryUnlink
*m
);
1033 // -- fragmenting --
1039 list
<MDSInternalContextBase
*> waiters
;
1040 list
<frag_t
> old_frags
;
1041 bufferlist rollback
;
1042 ufragment() : bits(0), committed(false), ls(NULL
) {}
1044 map
<dirfrag_t
, ufragment
> uncommitted_fragments
;
1046 struct fragment_info_t
{
1049 list
<CDir
*> resultfrags
;
1051 // for deadlock detection
1053 utime_t last_cum_auth_pins_change
;
1054 int last_cum_auth_pins
;
1055 int num_remote_waiters
; // number of remote authpin waiters
1056 fragment_info_t() : bits(0), all_frozen(false), last_cum_auth_pins(0), num_remote_waiters(0) {}
1057 bool is_fragmenting() { return !resultfrags
.empty(); }
1059 map
<dirfrag_t
,fragment_info_t
> fragments
;
1061 void adjust_dir_fragments(CInode
*diri
, frag_t basefrag
, int bits
,
1062 list
<CDir
*>& frags
, list
<MDSInternalContextBase
*>& waiters
, bool replay
);
1063 void adjust_dir_fragments(CInode
*diri
,
1064 list
<CDir
*>& srcfrags
,
1065 frag_t basefrag
, int bits
,
1066 list
<CDir
*>& resultfrags
,
1067 list
<MDSInternalContextBase
*>& waiters
,
1069 CDir
*force_dir_fragment(CInode
*diri
, frag_t fg
, bool replay
=true);
1070 void get_force_dirfrag_bound_set(vector
<dirfrag_t
>& dfs
, set
<CDir
*>& bounds
);
1072 bool can_fragment(CInode
*diri
, list
<CDir
*>& dirs
);
1073 void fragment_freeze_dirs(list
<CDir
*>& dirs
);
1074 void fragment_mark_and_complete(MDRequestRef
& mdr
);
1075 void fragment_frozen(MDRequestRef
& mdr
, int r
);
1076 void fragment_unmark_unfreeze_dirs(list
<CDir
*>& dirs
);
1077 void dispatch_fragment_dir(MDRequestRef
& mdr
);
1078 void _fragment_logged(MDRequestRef
& mdr
);
1079 void _fragment_stored(MDRequestRef
& mdr
);
1080 void _fragment_committed(dirfrag_t f
, list
<CDir
*>& resultfrags
);
1081 void _fragment_finish(dirfrag_t f
, list
<CDir
*>& resultfrags
);
1083 friend class EFragment
;
1084 friend class C_MDC_FragmentFrozen
;
1085 friend class C_MDC_FragmentMarking
;
1086 friend class C_MDC_FragmentPrep
;
1087 friend class C_MDC_FragmentStore
;
1088 friend class C_MDC_FragmentCommit
;
1089 friend class C_IO_MDC_FragmentFinish
;
1091 void handle_fragment_notify(MMDSFragmentNotify
*m
);
1093 void add_uncommitted_fragment(dirfrag_t basedirfrag
, int bits
, list
<frag_t
>& old_frag
,
1094 LogSegment
*ls
, bufferlist
*rollback
=NULL
);
1095 void finish_uncommitted_fragment(dirfrag_t basedirfrag
, int op
);
1096 void rollback_uncommitted_fragment(dirfrag_t basedirfrag
, list
<frag_t
>& old_frags
);
1098 void wait_for_uncommitted_fragment(dirfrag_t dirfrag
, MDSInternalContextBase
*c
) {
1099 assert(uncommitted_fragments
.count(dirfrag
));
1100 uncommitted_fragments
[dirfrag
].waiters
.push_back(c
);
1102 void split_dir(CDir
*dir
, int byn
);
1103 void merge_dir(CInode
*diri
, frag_t fg
);
1104 void rollback_uncommitted_fragments();
1106 void find_stale_fragment_freeze();
1107 void fragment_freeze_inc_num_waiters(CDir
*dir
);
1108 bool fragment_are_all_frozen(CDir
*dir
);
1109 int get_num_fragmenting_dirs() { return fragments
.size(); }
1112 //int send_inode_updates(CInode *in);
1113 //void handle_inode_update(MInodeUpdate *m);
1115 int send_dir_updates(CDir
*in
, bool bcast
=false);
1116 void handle_dir_update(MDirUpdate
*m
);
1118 // -- cache expiration --
1119 void handle_cache_expire(MCacheExpire
*m
);
1120 void process_delayed_expire(CDir
*dir
);
1121 void discard_delayed_expire(CDir
*dir
);
1124 int dump_cache(const char *fn
, Formatter
*f
,
1125 const std::string
& dump_root
= "",
1128 int dump_cache() { return dump_cache(NULL
, NULL
); }
1129 int dump_cache(const std::string
&filename
);
1130 int dump_cache(Formatter
*f
);
1131 int dump_cache(const std::string
& dump_root
, int depth
, Formatter
*f
);
1133 void dump_resolve_status(Formatter
*f
) const;
1134 void dump_rejoin_status(Formatter
*f
) const;
1139 void show_subtrees(int dbl
=10);
1141 CInode
*hack_pick_random_inode() {
1142 assert(!inode_map
.empty());
1143 int n
= rand() % inode_map
.size();
1144 ceph::unordered_map
<vinodeno_t
,CInode
*>::iterator p
= inode_map
.begin();
1150 void flush_dentry_work(MDRequestRef
& mdr
);
1152 * Resolve path to a dentry and pass it onto the ScrubStack.
1154 * TODO: return enough information to the original mdr formatter
1155 * and completion that they can subsequeuntly check the progress of
1156 * this scrub (we won't block them on a whole scrub as it can take a very
1159 void enqueue_scrub_work(MDRequestRef
& mdr
);
1160 void repair_inode_stats_work(MDRequestRef
& mdr
);
1161 void repair_dirfrag_stats_work(MDRequestRef
& mdr
);
1162 friend class C_MDC_RepairDirfragStats
;
1164 void flush_dentry(const string
& path
, Context
*fin
);
1166 * Create and start an OP_ENQUEUE_SCRUB
1168 void enqueue_scrub(const string
& path
, const std::string
&tag
,
1169 bool force
, bool recursive
, bool repair
,
1170 Formatter
*f
, Context
*fin
);
1171 void repair_inode_stats(CInode
*diri
);
1172 void repair_dirfrag_stats(CDir
*dir
);
1175 /* Because exports may fail, this set lets us keep track of inodes that need exporting. */
1176 std::set
<CInode
*> export_pin_queue
;
1179 class C_MDS_RetryRequest
: public MDSInternalContext
{
1183 C_MDS_RetryRequest(MDCache
*c
, MDRequestRef
& r
);
1184 void finish(int r
) override
;