1 #include <linux/ceph/ceph_debug.h>
3 #include <linux/module.h>
4 #include <linux/types.h>
5 #include <linux/slab.h>
6 #include <linux/random.h>
7 #include <linux/sched.h>
9 #include <linux/ceph/mon_client.h>
10 #include <linux/ceph/libceph.h>
11 #include <linux/ceph/debugfs.h>
12 #include <linux/ceph/decode.h>
13 #include <linux/ceph/auth.h>
16 * Interact with Ceph monitor cluster. Handle requests for new map
17 * versions, and periodically resend as needed. Also implement
18 * statfs() and umount().
20 * A small cluster of Ceph "monitors" are responsible for managing critical
21 * cluster configuration and state information. An odd number (e.g., 3, 5)
22 * of cmon daemons use a modified version of the Paxos part-time parliament
23 * algorithm to manage the MDS map (mds cluster membership), OSD map, and
24 * list of clients who have mounted the file system.
26 * We maintain an open, active session with a monitor at all times in order to
27 * receive timely MDSMap updates. We periodically send a keepalive byte on the
28 * TCP socket to ensure we detect a failure. If the connection does break, we
29 * randomly hunt for a new monitor. Once the connection is reestablished, we
30 * resend any outstanding requests.
33 static const struct ceph_connection_operations mon_con_ops
;
35 static int __validate_auth(struct ceph_mon_client
*monc
);
38 * Decode a monmap blob (e.g., during mount).
40 struct ceph_monmap
*ceph_monmap_decode(void *p
, void *end
)
42 struct ceph_monmap
*m
= NULL
;
44 struct ceph_fsid fsid
;
49 ceph_decode_32_safe(&p
, end
, len
, bad
);
50 ceph_decode_need(&p
, end
, len
, bad
);
52 dout("monmap_decode %p %p len %d\n", p
, end
, (int)(end
-p
));
54 ceph_decode_16_safe(&p
, end
, version
, bad
);
56 ceph_decode_need(&p
, end
, sizeof(fsid
) + 2*sizeof(u32
), bad
);
57 ceph_decode_copy(&p
, &fsid
, sizeof(fsid
));
58 epoch
= ceph_decode_32(&p
);
60 num_mon
= ceph_decode_32(&p
);
61 ceph_decode_need(&p
, end
, num_mon
*sizeof(m
->mon_inst
[0]), bad
);
63 if (num_mon
>= CEPH_MAX_MON
)
65 m
= kmalloc(sizeof(*m
) + sizeof(m
->mon_inst
[0])*num_mon
, GFP_NOFS
);
67 return ERR_PTR(-ENOMEM
);
71 ceph_decode_copy(&p
, m
->mon_inst
, num_mon
*sizeof(m
->mon_inst
[0]));
72 for (i
= 0; i
< num_mon
; i
++)
73 ceph_decode_addr(&m
->mon_inst
[i
].addr
);
75 dout("monmap_decode epoch %d, num_mon %d\n", m
->epoch
,
77 for (i
= 0; i
< m
->num_mon
; i
++)
78 dout("monmap_decode mon%d is %s\n", i
,
79 ceph_pr_addr(&m
->mon_inst
[i
].addr
.in_addr
));
83 dout("monmap_decode failed with %d\n", err
);
89 * return true if *addr is included in the monmap.
91 int ceph_monmap_contains(struct ceph_monmap
*m
, struct ceph_entity_addr
*addr
)
95 for (i
= 0; i
< m
->num_mon
; i
++)
96 if (memcmp(addr
, &m
->mon_inst
[i
].addr
, sizeof(*addr
)) == 0)
102 * Send an auth request.
104 static void __send_prepared_auth_request(struct ceph_mon_client
*monc
, int len
)
106 monc
->pending_auth
= 1;
107 monc
->m_auth
->front
.iov_len
= len
;
108 monc
->m_auth
->hdr
.front_len
= cpu_to_le32(len
);
109 ceph_msg_revoke(monc
->m_auth
);
110 ceph_msg_get(monc
->m_auth
); /* keep our ref */
111 ceph_con_send(&monc
->con
, monc
->m_auth
);
115 * Close monitor session, if any.
117 static void __close_session(struct ceph_mon_client
*monc
)
119 dout("__close_session closing mon%d\n", monc
->cur_mon
);
120 ceph_msg_revoke(monc
->m_auth
);
121 ceph_msg_revoke_incoming(monc
->m_auth_reply
);
122 ceph_msg_revoke(monc
->m_subscribe
);
123 ceph_msg_revoke_incoming(monc
->m_subscribe_ack
);
124 ceph_con_close(&monc
->con
);
126 monc
->pending_auth
= 0;
127 ceph_auth_reset(monc
->auth
);
131 * Open a session with a (new) monitor.
133 static int __open_session(struct ceph_mon_client
*monc
)
138 if (monc
->cur_mon
< 0) {
139 get_random_bytes(&r
, 1);
140 monc
->cur_mon
= r
% monc
->monmap
->num_mon
;
141 dout("open_session num=%d r=%d -> mon%d\n",
142 monc
->monmap
->num_mon
, r
, monc
->cur_mon
);
144 monc
->sub_renew_after
= jiffies
; /* i.e., expired */
145 monc
->want_next_osdmap
= !!monc
->want_next_osdmap
;
147 dout("open_session mon%d opening\n", monc
->cur_mon
);
148 ceph_con_open(&monc
->con
,
149 CEPH_ENTITY_TYPE_MON
, monc
->cur_mon
,
150 &monc
->monmap
->mon_inst
[monc
->cur_mon
].addr
);
152 /* send an initial keepalive to ensure our timestamp is
153 * valid by the time we are in an OPENED state */
154 ceph_con_keepalive(&monc
->con
);
156 /* initiatiate authentication handshake */
157 ret
= ceph_auth_build_hello(monc
->auth
,
158 monc
->m_auth
->front
.iov_base
,
159 monc
->m_auth
->front_alloc_len
);
160 __send_prepared_auth_request(monc
, ret
);
162 dout("open_session mon%d already open\n", monc
->cur_mon
);
167 static bool __sub_expired(struct ceph_mon_client
*monc
)
169 return time_after_eq(jiffies
, monc
->sub_renew_after
);
173 * Reschedule delayed work timer.
175 static void __schedule_delayed(struct ceph_mon_client
*monc
)
177 struct ceph_options
*opt
= monc
->client
->options
;
180 if (monc
->cur_mon
< 0 || __sub_expired(monc
)) {
184 if (opt
->monc_ping_timeout
> 0)
185 delay
= min(delay
, opt
->monc_ping_timeout
/ 3);
187 dout("__schedule_delayed after %lu\n", delay
);
188 schedule_delayed_work(&monc
->delayed_work
,
189 round_jiffies_relative(delay
));
193 * Send subscribe request for mdsmap and/or osdmap.
195 static void __send_subscribe(struct ceph_mon_client
*monc
)
197 dout("__send_subscribe sub_sent=%u exp=%u want_osd=%d\n",
198 (unsigned int)monc
->sub_sent
, __sub_expired(monc
),
199 monc
->want_next_osdmap
);
200 if ((__sub_expired(monc
) && !monc
->sub_sent
) ||
201 monc
->want_next_osdmap
== 1) {
202 struct ceph_msg
*msg
= monc
->m_subscribe
;
203 struct ceph_mon_subscribe_item
*i
;
207 p
= msg
->front
.iov_base
;
208 end
= p
+ msg
->front_alloc_len
;
210 num
= 1 + !!monc
->want_next_osdmap
+ !!monc
->want_mdsmap
;
211 ceph_encode_32(&p
, num
);
213 if (monc
->want_next_osdmap
) {
214 dout("__send_subscribe to 'osdmap' %u\n",
215 (unsigned int)monc
->have_osdmap
);
216 ceph_encode_string(&p
, end
, "osdmap", 6);
218 i
->have
= cpu_to_le64(monc
->have_osdmap
);
221 monc
->want_next_osdmap
= 2; /* requested */
223 if (monc
->want_mdsmap
) {
224 dout("__send_subscribe to 'mdsmap' %u+\n",
225 (unsigned int)monc
->have_mdsmap
);
226 ceph_encode_string(&p
, end
, "mdsmap", 6);
228 i
->have
= cpu_to_le64(monc
->have_mdsmap
);
232 ceph_encode_string(&p
, end
, "monmap", 6);
238 msg
->front
.iov_len
= p
- msg
->front
.iov_base
;
239 msg
->hdr
.front_len
= cpu_to_le32(msg
->front
.iov_len
);
240 ceph_msg_revoke(msg
);
241 ceph_con_send(&monc
->con
, ceph_msg_get(msg
));
243 monc
->sub_sent
= jiffies
| 1; /* never 0 */
247 static void handle_subscribe_ack(struct ceph_mon_client
*monc
,
248 struct ceph_msg
*msg
)
250 unsigned int seconds
;
251 struct ceph_mon_subscribe_ack
*h
= msg
->front
.iov_base
;
253 if (msg
->front
.iov_len
< sizeof(*h
))
255 seconds
= le32_to_cpu(h
->duration
);
257 mutex_lock(&monc
->mutex
);
258 dout("handle_subscribe_ack after %d seconds\n", seconds
);
259 monc
->sub_renew_after
= monc
->sub_sent
+ (seconds
>> 1)*HZ
- 1;
261 mutex_unlock(&monc
->mutex
);
264 pr_err("got corrupt subscribe-ack msg\n");
269 * Keep track of which maps we have
271 int ceph_monc_got_mdsmap(struct ceph_mon_client
*monc
, u32 got
)
273 mutex_lock(&monc
->mutex
);
274 monc
->have_mdsmap
= got
;
275 mutex_unlock(&monc
->mutex
);
278 EXPORT_SYMBOL(ceph_monc_got_mdsmap
);
280 int ceph_monc_got_osdmap(struct ceph_mon_client
*monc
, u32 got
)
282 mutex_lock(&monc
->mutex
);
283 monc
->have_osdmap
= got
;
284 monc
->want_next_osdmap
= 0;
285 mutex_unlock(&monc
->mutex
);
290 * Register interest in the next osdmap
292 void ceph_monc_request_next_osdmap(struct ceph_mon_client
*monc
)
294 dout("request_next_osdmap have %u\n", monc
->have_osdmap
);
295 mutex_lock(&monc
->mutex
);
296 if (!monc
->want_next_osdmap
)
297 monc
->want_next_osdmap
= 1;
298 if (monc
->want_next_osdmap
< 2)
299 __send_subscribe(monc
);
300 mutex_unlock(&monc
->mutex
);
302 EXPORT_SYMBOL(ceph_monc_request_next_osdmap
);
305 * Wait for an osdmap with a given epoch.
307 * @epoch: epoch to wait for
308 * @timeout: in jiffies, 0 means "wait forever"
310 int ceph_monc_wait_osdmap(struct ceph_mon_client
*monc
, u32 epoch
,
311 unsigned long timeout
)
313 unsigned long started
= jiffies
;
316 mutex_lock(&monc
->mutex
);
317 while (monc
->have_osdmap
< epoch
) {
318 mutex_unlock(&monc
->mutex
);
320 if (timeout
&& time_after_eq(jiffies
, started
+ timeout
))
323 ret
= wait_event_interruptible_timeout(monc
->client
->auth_wq
,
324 monc
->have_osdmap
>= epoch
,
325 ceph_timeout_jiffies(timeout
));
329 mutex_lock(&monc
->mutex
);
332 mutex_unlock(&monc
->mutex
);
335 EXPORT_SYMBOL(ceph_monc_wait_osdmap
);
340 int ceph_monc_open_session(struct ceph_mon_client
*monc
)
342 mutex_lock(&monc
->mutex
);
343 __open_session(monc
);
344 __schedule_delayed(monc
);
345 mutex_unlock(&monc
->mutex
);
348 EXPORT_SYMBOL(ceph_monc_open_session
);
350 static void ceph_monc_handle_map(struct ceph_mon_client
*monc
,
351 struct ceph_msg
*msg
)
353 struct ceph_client
*client
= monc
->client
;
354 struct ceph_monmap
*monmap
= NULL
, *old
= monc
->monmap
;
357 mutex_lock(&monc
->mutex
);
359 dout("handle_monmap\n");
360 p
= msg
->front
.iov_base
;
361 end
= p
+ msg
->front
.iov_len
;
363 monmap
= ceph_monmap_decode(p
, end
);
364 if (IS_ERR(monmap
)) {
365 pr_err("problem decoding monmap, %d\n",
366 (int)PTR_ERR(monmap
));
370 if (ceph_check_fsid(monc
->client
, &monmap
->fsid
) < 0) {
375 client
->monc
.monmap
= monmap
;
378 client
->have_fsid
= true;
381 mutex_unlock(&monc
->mutex
);
382 wake_up_all(&client
->auth_wq
);
386 * generic requests (currently statfs, mon_get_version)
388 static struct ceph_mon_generic_request
*__lookup_generic_req(
389 struct ceph_mon_client
*monc
, u64 tid
)
391 struct ceph_mon_generic_request
*req
;
392 struct rb_node
*n
= monc
->generic_request_tree
.rb_node
;
395 req
= rb_entry(n
, struct ceph_mon_generic_request
, node
);
398 else if (tid
> req
->tid
)
406 static void __insert_generic_request(struct ceph_mon_client
*monc
,
407 struct ceph_mon_generic_request
*new)
409 struct rb_node
**p
= &monc
->generic_request_tree
.rb_node
;
410 struct rb_node
*parent
= NULL
;
411 struct ceph_mon_generic_request
*req
= NULL
;
415 req
= rb_entry(parent
, struct ceph_mon_generic_request
, node
);
416 if (new->tid
< req
->tid
)
418 else if (new->tid
> req
->tid
)
424 rb_link_node(&new->node
, parent
, p
);
425 rb_insert_color(&new->node
, &monc
->generic_request_tree
);
428 static void release_generic_request(struct kref
*kref
)
430 struct ceph_mon_generic_request
*req
=
431 container_of(kref
, struct ceph_mon_generic_request
, kref
);
434 ceph_msg_put(req
->reply
);
436 ceph_msg_put(req
->request
);
441 static void put_generic_request(struct ceph_mon_generic_request
*req
)
443 kref_put(&req
->kref
, release_generic_request
);
446 static void get_generic_request(struct ceph_mon_generic_request
*req
)
448 kref_get(&req
->kref
);
451 static struct ceph_msg
*get_generic_reply(struct ceph_connection
*con
,
452 struct ceph_msg_header
*hdr
,
455 struct ceph_mon_client
*monc
= con
->private;
456 struct ceph_mon_generic_request
*req
;
457 u64 tid
= le64_to_cpu(hdr
->tid
);
460 mutex_lock(&monc
->mutex
);
461 req
= __lookup_generic_req(monc
, tid
);
463 dout("get_generic_reply %lld dne\n", tid
);
467 dout("get_generic_reply %lld got %p\n", tid
, req
->reply
);
469 m
= ceph_msg_get(req
->reply
);
471 * we don't need to track the connection reading into
472 * this reply because we only have one open connection
476 mutex_unlock(&monc
->mutex
);
480 static int __do_generic_request(struct ceph_mon_client
*monc
, u64 tid
,
481 struct ceph_mon_generic_request
*req
)
485 /* register request */
486 req
->tid
= tid
!= 0 ? tid
: ++monc
->last_tid
;
487 req
->request
->hdr
.tid
= cpu_to_le64(req
->tid
);
488 __insert_generic_request(monc
, req
);
489 monc
->num_generic_requests
++;
490 ceph_con_send(&monc
->con
, ceph_msg_get(req
->request
));
491 mutex_unlock(&monc
->mutex
);
493 err
= wait_for_completion_interruptible(&req
->completion
);
495 mutex_lock(&monc
->mutex
);
496 rb_erase(&req
->node
, &monc
->generic_request_tree
);
497 monc
->num_generic_requests
--;
504 static int do_generic_request(struct ceph_mon_client
*monc
,
505 struct ceph_mon_generic_request
*req
)
509 mutex_lock(&monc
->mutex
);
510 err
= __do_generic_request(monc
, 0, req
);
511 mutex_unlock(&monc
->mutex
);
519 static void handle_statfs_reply(struct ceph_mon_client
*monc
,
520 struct ceph_msg
*msg
)
522 struct ceph_mon_generic_request
*req
;
523 struct ceph_mon_statfs_reply
*reply
= msg
->front
.iov_base
;
524 u64 tid
= le64_to_cpu(msg
->hdr
.tid
);
526 if (msg
->front
.iov_len
!= sizeof(*reply
))
528 dout("handle_statfs_reply %p tid %llu\n", msg
, tid
);
530 mutex_lock(&monc
->mutex
);
531 req
= __lookup_generic_req(monc
, tid
);
533 *(struct ceph_statfs
*)req
->buf
= reply
->st
;
535 get_generic_request(req
);
537 mutex_unlock(&monc
->mutex
);
539 complete_all(&req
->completion
);
540 put_generic_request(req
);
545 pr_err("corrupt statfs reply, tid %llu\n", tid
);
550 * Do a synchronous statfs().
552 int ceph_monc_do_statfs(struct ceph_mon_client
*monc
, struct ceph_statfs
*buf
)
554 struct ceph_mon_generic_request
*req
;
555 struct ceph_mon_statfs
*h
;
558 req
= kzalloc(sizeof(*req
), GFP_NOFS
);
562 kref_init(&req
->kref
);
564 init_completion(&req
->completion
);
567 req
->request
= ceph_msg_new(CEPH_MSG_STATFS
, sizeof(*h
), GFP_NOFS
,
571 req
->reply
= ceph_msg_new(CEPH_MSG_STATFS_REPLY
, 1024, GFP_NOFS
,
576 /* fill out request */
577 h
= req
->request
->front
.iov_base
;
578 h
->monhdr
.have_version
= 0;
579 h
->monhdr
.session_mon
= cpu_to_le16(-1);
580 h
->monhdr
.session_mon_tid
= 0;
581 h
->fsid
= monc
->monmap
->fsid
;
583 err
= do_generic_request(monc
, req
);
586 put_generic_request(req
);
589 EXPORT_SYMBOL(ceph_monc_do_statfs
);
591 static void handle_get_version_reply(struct ceph_mon_client
*monc
,
592 struct ceph_msg
*msg
)
594 struct ceph_mon_generic_request
*req
;
595 u64 tid
= le64_to_cpu(msg
->hdr
.tid
);
596 void *p
= msg
->front
.iov_base
;
597 void *end
= p
+ msg
->front_alloc_len
;
600 dout("%s %p tid %llu\n", __func__
, msg
, tid
);
602 ceph_decode_need(&p
, end
, 2*sizeof(u64
), bad
);
603 handle
= ceph_decode_64(&p
);
604 if (tid
!= 0 && tid
!= handle
)
607 mutex_lock(&monc
->mutex
);
608 req
= __lookup_generic_req(monc
, handle
);
610 *(u64
*)req
->buf
= ceph_decode_64(&p
);
612 get_generic_request(req
);
614 mutex_unlock(&monc
->mutex
);
616 complete_all(&req
->completion
);
617 put_generic_request(req
);
622 pr_err("corrupt mon_get_version reply, tid %llu\n", tid
);
627 * Send MMonGetVersion and wait for the reply.
629 * @what: one of "mdsmap", "osdmap" or "monmap"
631 int ceph_monc_do_get_version(struct ceph_mon_client
*monc
, const char *what
,
634 struct ceph_mon_generic_request
*req
;
639 req
= kzalloc(sizeof(*req
), GFP_NOFS
);
643 kref_init(&req
->kref
);
645 init_completion(&req
->completion
);
647 req
->request
= ceph_msg_new(CEPH_MSG_MON_GET_VERSION
,
648 sizeof(u64
) + sizeof(u32
) + strlen(what
),
655 req
->reply
= ceph_msg_new(CEPH_MSG_MON_GET_VERSION_REPLY
, 1024,
662 p
= req
->request
->front
.iov_base
;
663 end
= p
+ req
->request
->front_alloc_len
;
665 /* fill out request */
666 mutex_lock(&monc
->mutex
);
667 tid
= ++monc
->last_tid
;
668 ceph_encode_64(&p
, tid
); /* handle */
669 ceph_encode_string(&p
, end
, what
, strlen(what
));
671 err
= __do_generic_request(monc
, tid
, req
);
673 mutex_unlock(&monc
->mutex
);
675 put_generic_request(req
);
678 EXPORT_SYMBOL(ceph_monc_do_get_version
);
681 * Resend pending generic requests.
683 static void __resend_generic_request(struct ceph_mon_client
*monc
)
685 struct ceph_mon_generic_request
*req
;
688 for (p
= rb_first(&monc
->generic_request_tree
); p
; p
= rb_next(p
)) {
689 req
= rb_entry(p
, struct ceph_mon_generic_request
, node
);
690 ceph_msg_revoke(req
->request
);
691 ceph_msg_revoke_incoming(req
->reply
);
692 ceph_con_send(&monc
->con
, ceph_msg_get(req
->request
));
697 * Delayed work. If we haven't mounted yet, retry. Otherwise,
698 * renew/retry subscription as needed (in case it is timing out, or we
699 * got an ENOMEM). And keep the monitor connection alive.
701 static void delayed_work(struct work_struct
*work
)
703 struct ceph_mon_client
*monc
=
704 container_of(work
, struct ceph_mon_client
, delayed_work
.work
);
706 dout("monc delayed_work\n");
707 mutex_lock(&monc
->mutex
);
709 __close_session(monc
);
710 __open_session(monc
); /* continue hunting */
712 struct ceph_options
*opt
= monc
->client
->options
;
713 int is_auth
= ceph_auth_is_authenticated(monc
->auth
);
714 if (ceph_con_keepalive_expired(&monc
->con
,
715 opt
->monc_ping_timeout
)) {
716 dout("monc keepalive timeout\n");
718 __close_session(monc
);
719 monc
->hunting
= true;
720 __open_session(monc
);
723 if (!monc
->hunting
) {
724 ceph_con_keepalive(&monc
->con
);
725 __validate_auth(monc
);
729 __send_subscribe(monc
);
731 __schedule_delayed(monc
);
732 mutex_unlock(&monc
->mutex
);
736 * On startup, we build a temporary monmap populated with the IPs
737 * provided by mount(2).
739 static int build_initial_monmap(struct ceph_mon_client
*monc
)
741 struct ceph_options
*opt
= monc
->client
->options
;
742 struct ceph_entity_addr
*mon_addr
= opt
->mon_addr
;
743 int num_mon
= opt
->num_mon
;
746 /* build initial monmap */
747 monc
->monmap
= kzalloc(sizeof(*monc
->monmap
) +
748 num_mon
*sizeof(monc
->monmap
->mon_inst
[0]),
752 for (i
= 0; i
< num_mon
; i
++) {
753 monc
->monmap
->mon_inst
[i
].addr
= mon_addr
[i
];
754 monc
->monmap
->mon_inst
[i
].addr
.nonce
= 0;
755 monc
->monmap
->mon_inst
[i
].name
.type
=
756 CEPH_ENTITY_TYPE_MON
;
757 monc
->monmap
->mon_inst
[i
].name
.num
= cpu_to_le64(i
);
759 monc
->monmap
->num_mon
= num_mon
;
763 int ceph_monc_init(struct ceph_mon_client
*monc
, struct ceph_client
*cl
)
768 memset(monc
, 0, sizeof(*monc
));
771 mutex_init(&monc
->mutex
);
773 err
= build_initial_monmap(monc
);
779 monc
->auth
= ceph_auth_init(cl
->options
->name
,
781 if (IS_ERR(monc
->auth
)) {
782 err
= PTR_ERR(monc
->auth
);
785 monc
->auth
->want_keys
=
786 CEPH_ENTITY_TYPE_AUTH
| CEPH_ENTITY_TYPE_MON
|
787 CEPH_ENTITY_TYPE_OSD
| CEPH_ENTITY_TYPE_MDS
;
791 monc
->m_subscribe_ack
= ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE_ACK
,
792 sizeof(struct ceph_mon_subscribe_ack
),
794 if (!monc
->m_subscribe_ack
)
797 monc
->m_subscribe
= ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE
, 96, GFP_NOFS
,
799 if (!monc
->m_subscribe
)
800 goto out_subscribe_ack
;
802 monc
->m_auth_reply
= ceph_msg_new(CEPH_MSG_AUTH_REPLY
, 4096, GFP_NOFS
,
804 if (!monc
->m_auth_reply
)
807 monc
->m_auth
= ceph_msg_new(CEPH_MSG_AUTH
, 4096, GFP_NOFS
, true);
808 monc
->pending_auth
= 0;
812 ceph_con_init(&monc
->con
, monc
, &mon_con_ops
,
813 &monc
->client
->msgr
);
816 monc
->hunting
= true;
817 monc
->sub_renew_after
= jiffies
;
820 INIT_DELAYED_WORK(&monc
->delayed_work
, delayed_work
);
821 monc
->generic_request_tree
= RB_ROOT
;
822 monc
->num_generic_requests
= 0;
825 monc
->have_mdsmap
= 0;
826 monc
->have_osdmap
= 0;
827 monc
->want_next_osdmap
= 1;
831 ceph_msg_put(monc
->m_auth_reply
);
833 ceph_msg_put(monc
->m_subscribe
);
835 ceph_msg_put(monc
->m_subscribe_ack
);
837 ceph_auth_destroy(monc
->auth
);
843 EXPORT_SYMBOL(ceph_monc_init
);
845 void ceph_monc_stop(struct ceph_mon_client
*monc
)
848 cancel_delayed_work_sync(&monc
->delayed_work
);
850 mutex_lock(&monc
->mutex
);
851 __close_session(monc
);
853 mutex_unlock(&monc
->mutex
);
856 * flush msgr queue before we destroy ourselves to ensure that:
857 * - any work that references our embedded con is finished.
858 * - any osd_client or other work that may reference an authorizer
859 * finishes before we shut down the auth subsystem.
863 ceph_auth_destroy(monc
->auth
);
865 ceph_msg_put(monc
->m_auth
);
866 ceph_msg_put(monc
->m_auth_reply
);
867 ceph_msg_put(monc
->m_subscribe
);
868 ceph_msg_put(monc
->m_subscribe_ack
);
872 EXPORT_SYMBOL(ceph_monc_stop
);
874 static void finish_hunting(struct ceph_mon_client
*monc
)
877 dout("%s found mon%d\n", __func__
, monc
->cur_mon
);
878 monc
->hunting
= false;
882 static void handle_auth_reply(struct ceph_mon_client
*monc
,
883 struct ceph_msg
*msg
)
888 mutex_lock(&monc
->mutex
);
889 was_auth
= ceph_auth_is_authenticated(monc
->auth
);
890 monc
->pending_auth
= 0;
891 ret
= ceph_handle_auth_reply(monc
->auth
, msg
->front
.iov_base
,
893 monc
->m_auth
->front
.iov_base
,
894 monc
->m_auth
->front_alloc_len
);
896 __send_prepared_auth_request(monc
, ret
);
900 finish_hunting(monc
);
903 monc
->client
->auth_err
= ret
;
904 } else if (!was_auth
&& ceph_auth_is_authenticated(monc
->auth
)) {
905 dout("authenticated, starting session\n");
907 monc
->client
->msgr
.inst
.name
.type
= CEPH_ENTITY_TYPE_CLIENT
;
908 monc
->client
->msgr
.inst
.name
.num
=
909 cpu_to_le64(monc
->auth
->global_id
);
911 __send_subscribe(monc
);
912 __resend_generic_request(monc
);
914 pr_info("mon%d %s session established\n", monc
->cur_mon
,
915 ceph_pr_addr(&monc
->con
.peer_addr
.in_addr
));
919 mutex_unlock(&monc
->mutex
);
920 if (monc
->client
->auth_err
< 0)
921 wake_up_all(&monc
->client
->auth_wq
);
924 static int __validate_auth(struct ceph_mon_client
*monc
)
928 if (monc
->pending_auth
)
931 ret
= ceph_build_auth(monc
->auth
, monc
->m_auth
->front
.iov_base
,
932 monc
->m_auth
->front_alloc_len
);
934 return ret
; /* either an error, or no need to authenticate */
935 __send_prepared_auth_request(monc
, ret
);
939 int ceph_monc_validate_auth(struct ceph_mon_client
*monc
)
943 mutex_lock(&monc
->mutex
);
944 ret
= __validate_auth(monc
);
945 mutex_unlock(&monc
->mutex
);
948 EXPORT_SYMBOL(ceph_monc_validate_auth
);
951 * handle incoming message
953 static void dispatch(struct ceph_connection
*con
, struct ceph_msg
*msg
)
955 struct ceph_mon_client
*monc
= con
->private;
956 int type
= le16_to_cpu(msg
->hdr
.type
);
962 case CEPH_MSG_AUTH_REPLY
:
963 handle_auth_reply(monc
, msg
);
966 case CEPH_MSG_MON_SUBSCRIBE_ACK
:
967 handle_subscribe_ack(monc
, msg
);
970 case CEPH_MSG_STATFS_REPLY
:
971 handle_statfs_reply(monc
, msg
);
974 case CEPH_MSG_MON_GET_VERSION_REPLY
:
975 handle_get_version_reply(monc
, msg
);
978 case CEPH_MSG_MON_MAP
:
979 ceph_monc_handle_map(monc
, msg
);
982 case CEPH_MSG_OSD_MAP
:
983 ceph_osdc_handle_map(&monc
->client
->osdc
, msg
);
987 /* can the chained handler handle it? */
988 if (monc
->client
->extra_mon_dispatch
&&
989 monc
->client
->extra_mon_dispatch(monc
->client
, msg
) == 0)
992 pr_err("received unknown message type %d %s\n", type
,
993 ceph_msg_type_name(type
));
999 * Allocate memory for incoming message
1001 static struct ceph_msg
*mon_alloc_msg(struct ceph_connection
*con
,
1002 struct ceph_msg_header
*hdr
,
1005 struct ceph_mon_client
*monc
= con
->private;
1006 int type
= le16_to_cpu(hdr
->type
);
1007 int front_len
= le32_to_cpu(hdr
->front_len
);
1008 struct ceph_msg
*m
= NULL
;
1013 case CEPH_MSG_MON_SUBSCRIBE_ACK
:
1014 m
= ceph_msg_get(monc
->m_subscribe_ack
);
1016 case CEPH_MSG_STATFS_REPLY
:
1017 return get_generic_reply(con
, hdr
, skip
);
1018 case CEPH_MSG_AUTH_REPLY
:
1019 m
= ceph_msg_get(monc
->m_auth_reply
);
1021 case CEPH_MSG_MON_GET_VERSION_REPLY
:
1022 if (le64_to_cpu(hdr
->tid
) != 0)
1023 return get_generic_reply(con
, hdr
, skip
);
1026 * Older OSDs don't set reply tid even if the orignal
1027 * request had a non-zero tid. Workaround this weirdness
1028 * by falling through to the allocate case.
1030 case CEPH_MSG_MON_MAP
:
1031 case CEPH_MSG_MDS_MAP
:
1032 case CEPH_MSG_OSD_MAP
:
1033 m
= ceph_msg_new(type
, front_len
, GFP_NOFS
, false);
1035 return NULL
; /* ENOMEM--return skip == 0 */
1040 pr_info("alloc_msg unknown type %d\n", type
);
1042 } else if (front_len
> m
->front_alloc_len
) {
1043 pr_warn("mon_alloc_msg front %d > prealloc %d (%u#%llu)\n",
1044 front_len
, m
->front_alloc_len
,
1045 (unsigned int)con
->peer_name
.type
,
1046 le64_to_cpu(con
->peer_name
.num
));
1048 m
= ceph_msg_new(type
, front_len
, GFP_NOFS
, false);
1055 * If the monitor connection resets, pick a new monitor and resubmit
1056 * any pending requests.
1058 static void mon_fault(struct ceph_connection
*con
)
1060 struct ceph_mon_client
*monc
= con
->private;
1065 dout("mon_fault\n");
1066 mutex_lock(&monc
->mutex
);
1071 pr_info("mon%d %s session lost, "
1072 "hunting for new mon\n", monc
->cur_mon
,
1073 ceph_pr_addr(&monc
->con
.peer_addr
.in_addr
));
1075 __close_session(monc
);
1076 if (!monc
->hunting
) {
1078 monc
->hunting
= true;
1079 __open_session(monc
);
1081 /* already hunting, let's wait a bit */
1082 __schedule_delayed(monc
);
1085 mutex_unlock(&monc
->mutex
);
1089 * We can ignore refcounting on the connection struct, as all references
1090 * will come from the messenger workqueue, which is drained prior to
1091 * mon_client destruction.
1093 static struct ceph_connection
*con_get(struct ceph_connection
*con
)
1098 static void con_put(struct ceph_connection
*con
)
1102 static const struct ceph_connection_operations mon_con_ops
= {
1105 .dispatch
= dispatch
,
1107 .alloc_msg
= mon_alloc_msg
,