1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2011 New Dream Network
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
17 #include "common/admin_socket.h"
18 #include "common/admin_socket_client.h"
19 #include "common/dout.h"
20 #include "common/errno.h"
21 #include "common/safe_io.h"
22 #include "common/Thread.h"
23 #include "common/version.h"
24 #include "common/ceph_mutex.h"
27 #include "common/Cond.h"
30 #include "messages/MCommand.h"
31 #include "messages/MCommandReply.h"
32 #include "messages/MMonCommand.h"
33 #include "messages/MMonCommandAck.h"
35 // re-include our assert to clobber the system one; fix dout:
36 #include "include/ceph_assert.h"
37 #include "include/compat.h"
38 #include "include/sock_compat.h"
40 #define dout_subsys ceph_subsys_asok
42 #define dout_prefix *_dout << "asok(" << (void*)m_cct << ") "
45 using std::ostringstream
;
46 using namespace TOPNSPC::common
;
49 * UNIX domain sockets created by an application persist even after that
50 * application closes, unless they're explicitly unlinked. This is because the
51 * directory containing the socket keeps a reference to the socket.
53 * This code makes things a little nicer by unlinking those dead sockets when
54 * the application exits normally.
57 template<typename F
, typename
... Args
>
58 inline int retry_sys_call(F f
, Args
... args
) {
62 } while (r
< 0 && errno
== EINTR
);
67 static std::mutex cleanup_lock
;
68 static std::vector
<std::string
> cleanup_files
;
69 static bool cleanup_atexit
= false;
71 static void remove_cleanup_file(std::string_view file
) {
72 std::unique_lock
l(cleanup_lock
);
74 if (auto i
= std::find(cleanup_files
.cbegin(), cleanup_files
.cend(), file
);
75 i
!= cleanup_files
.cend()) {
76 retry_sys_call(::unlink
, i
->c_str());
77 cleanup_files
.erase(i
);
81 void remove_all_cleanup_files() {
82 std::unique_lock
l(cleanup_lock
);
83 for (const auto& s
: cleanup_files
) {
84 retry_sys_call(::unlink
, s
.c_str());
86 cleanup_files
.clear();
89 static void add_cleanup_file(std::string file
) {
90 std::unique_lock
l(cleanup_lock
);
91 cleanup_files
.push_back(std::move(file
));
92 if (!cleanup_atexit
) {
93 atexit(remove_all_cleanup_files
);
94 cleanup_atexit
= true;
98 AdminSocket::AdminSocket(CephContext
*cct
)
102 AdminSocket::~AdminSocket()
108 * This thread listens on the UNIX domain socket for incoming connections.
109 * It only handles one connection at a time at the moment. All I/O is nonblocking,
110 * so that we can implement sensible timeouts. [TODO: make all I/O nonblocking]
112 * This thread also listens to m_wakeup_rd_fd. If there is any data sent to this
113 * pipe, the thread wakes up. If m_shutdown is set, the thread terminates
114 * itself gracefully, allowing the AdminSocketConfigObs class to join() it.
117 std::string
AdminSocket::create_wakeup_pipe(int *pipe_rd
, int *pipe_wr
)
120 if (pipe_cloexec(pipefd
, O_NONBLOCK
) < 0) {
123 oss
<< "AdminSocket::create_wakeup_pipe error: " << cpp_strerror(e
);
127 *pipe_rd
= pipefd
[0];
128 *pipe_wr
= pipefd
[1];
132 std::string
AdminSocket::destroy_wakeup_pipe()
134 // Send a byte to the wakeup pipe that the thread is listening to
135 char buf
[1] = { 0x0 };
136 int ret
= safe_write(m_wakeup_wr_fd
, buf
, sizeof(buf
));
139 retry_sys_call(::close
, m_wakeup_wr_fd
);
144 oss
<< "AdminSocket::destroy_shutdown_pipe error: failed to write"
145 "to thread shutdown pipe: error " << ret
;
151 // Close read end. Doing this before join() blocks the listenter and prevents
153 retry_sys_call(::close
, m_wakeup_rd_fd
);
159 std::string
AdminSocket::bind_and_listen(const std::string
&sock_path
, int *fd
)
161 ldout(m_cct
, 5) << "bind_and_listen " << sock_path
<< dendl
;
163 struct sockaddr_un address
;
164 if (sock_path
.size() > sizeof(address
.sun_path
) - 1) {
166 oss
<< "AdminSocket::bind_and_listen: "
167 << "The UNIX domain socket path " << sock_path
<< " is too long! The "
168 << "maximum length on this system is "
169 << (sizeof(address
.sun_path
) - 1);
172 int sock_fd
= socket_cloexec(PF_UNIX
, SOCK_STREAM
, 0);
176 oss
<< "AdminSocket::bind_and_listen: "
177 << "failed to create socket: " << cpp_strerror(err
);
180 // FIPS zeroization audit 20191115: this memset is fine.
181 memset(&address
, 0, sizeof(struct sockaddr_un
));
182 address
.sun_family
= AF_UNIX
;
183 snprintf(address
.sun_path
, sizeof(address
.sun_path
),
184 "%s", sock_path
.c_str());
185 if (::bind(sock_fd
, (struct sockaddr
*)&address
,
186 sizeof(struct sockaddr_un
)) != 0) {
188 if (err
== EADDRINUSE
) {
189 AdminSocketClient
client(sock_path
);
193 ldout(m_cct
, 20) << "socket " << sock_path
<< " is in use" << dendl
;
196 ldout(m_cct
, 20) << "unlink stale file " << sock_path
<< dendl
;
197 retry_sys_call(::unlink
, sock_path
.c_str());
198 if (::bind(sock_fd
, (struct sockaddr
*)&address
,
199 sizeof(struct sockaddr_un
)) == 0) {
208 oss
<< "AdminSocket::bind_and_listen: "
209 << "failed to bind the UNIX domain socket to '" << sock_path
210 << "': " << cpp_strerror(err
);
215 if (listen(sock_fd
, 5) != 0) {
218 oss
<< "AdminSocket::bind_and_listen: "
219 << "failed to listen to socket: " << cpp_strerror(err
);
221 retry_sys_call(::unlink
, sock_path
.c_str());
228 void AdminSocket::entry() noexcept
230 ldout(m_cct
, 5) << "entry start" << dendl
;
232 struct pollfd fds
[2];
233 // FIPS zeroization audit 20191115: this memset is fine.
234 memset(fds
, 0, sizeof(fds
));
235 fds
[0].fd
= m_sock_fd
;
236 fds
[0].events
= POLLIN
| POLLRDBAND
;
237 fds
[1].fd
= m_wakeup_rd_fd
;
238 fds
[1].events
= POLLIN
| POLLRDBAND
;
240 ldout(m_cct
,20) << __func__
<< " waiting" << dendl
;
241 int ret
= poll(fds
, 2, -1);
247 lderr(m_cct
) << "AdminSocket: poll(2) error: '"
248 << cpp_strerror(err
) << dendl
;
251 ldout(m_cct
,20) << __func__
<< " awake" << dendl
;
253 if (fds
[0].revents
& POLLIN
) {
254 // Send out some data
257 if (fds
[1].revents
& POLLIN
) {
260 auto s
= ::read(m_wakeup_rd_fd
, &buf
, 1);
263 ldout(m_cct
, 5) << "AdminSocket: (ignoring) read(2) error: '"
264 << cpp_strerror(e
) << dendl
;
269 // Parent wants us to shut down
273 ldout(m_cct
, 5) << "entry exit" << dendl
;
276 void AdminSocket::chown(uid_t uid
, gid_t gid
)
278 if (m_sock_fd
>= 0) {
279 int r
= ::chown(m_path
.c_str(), uid
, gid
);
282 lderr(m_cct
) << "AdminSocket: failed to chown socket: "
283 << cpp_strerror(r
) << dendl
;
288 void AdminSocket::chmod(mode_t mode
)
290 if (m_sock_fd
>= 0) {
291 int r
= ::chmod(m_path
.c_str(), mode
);
294 lderr(m_cct
) << "AdminSocket: failed to chmod socket: "
295 << cpp_strerror(r
) << dendl
;
300 void AdminSocket::do_accept()
302 struct sockaddr_un address
;
303 socklen_t address_length
= sizeof(address
);
304 ldout(m_cct
, 30) << "AdminSocket: calling accept" << dendl
;
305 int connection_fd
= accept_cloexec(m_sock_fd
, (struct sockaddr
*) &address
,
307 if (connection_fd
< 0) {
309 lderr(m_cct
) << "AdminSocket: do_accept error: '"
310 << cpp_strerror(err
) << dendl
;
313 ldout(m_cct
, 30) << "AdminSocket: finished accept" << dendl
;
319 int ret
= safe_read(connection_fd
, &cmd
[pos
], 1);
322 lderr(m_cct
) << "AdminSocket: error reading request code: "
323 << cpp_strerror(ret
) << dendl
;
325 retry_sys_call(::close
, connection_fd
);
328 if (cmd
[0] == '\0') {
329 // old protocol: __be32
330 if (pos
== 3 && cmd
[0] == '\0') {
336 c
= "perfcounters_dump";
339 c
= "perfcounters_schema";
345 //wrap command with new protocol
346 c
= "{\"prefix\": \"" + c
+ "\"}";
350 // new protocol: null or \n terminated string
351 if (cmd
[pos
] == '\n' || cmd
[pos
] == '\0') {
357 if (++pos
>= sizeof(cmd
)) {
358 lderr(m_cct
) << "AdminSocket: error reading request too long" << dendl
;
359 retry_sys_call(::close
, connection_fd
);
364 std::vector
<std::string
> cmdvec
= { c
};
365 bufferlist empty
, out
;
367 int rval
= execute_command(cmdvec
, empty
/* inbl */, err
, &out
);
369 // Unfortunately, the asok wire protocol does not let us pass an error code,
370 // and many asok command implementations return helpful error strings. So,
371 // let's prepend an error string to the output if there is an error code.
374 ss
<< "ERROR: " << cpp_strerror(rval
) << "\n";
375 ss
<< err
.str() << "\n";
381 uint32_t len
= htonl(out
.length());
382 int ret
= safe_write(connection_fd
, &len
, sizeof(len
));
384 lderr(m_cct
) << "AdminSocket: error writing response length "
385 << cpp_strerror(ret
) << dendl
;
387 if (out
.write_fd(connection_fd
) < 0) {
388 lderr(m_cct
) << "AdminSocket: error writing response payload "
389 << cpp_strerror(ret
) << dendl
;
392 retry_sys_call(::close
, connection_fd
);
395 void AdminSocket::do_tell_queue()
397 ldout(m_cct
,10) << __func__
<< dendl
;
398 std::list
<cref_t
<MCommand
>> q
;
399 std::list
<cref_t
<MMonCommand
>> lq
;
401 std::lock_guard
l(tell_lock
);
403 lq
.swap(tell_legacy_queue
);
410 [m
](int r
, const std::string
& err
, bufferlist
& outbl
) {
411 auto reply
= new MCommandReply(r
, err
);
412 reply
->set_tid(m
->get_tid());
413 reply
->set_data(outbl
);
415 #warning "fix message send with crimson"
417 m
->get_connection()->send_message(reply
);
426 [m
](int r
, const std::string
& err
, bufferlist
& outbl
) {
427 auto reply
= new MMonCommandAck(m
->cmd
, r
, err
, 0);
428 reply
->set_tid(m
->get_tid());
429 reply
->set_data(outbl
);
431 #warning "fix message send with crimson"
433 m
->get_connection()->send_message(reply
);
439 int AdminSocket::execute_command(
440 const std::vector
<std::string
>& cmd
,
441 const bufferlist
& inbl
,
446 #warning "must implement admin socket blocking execute_command() for crimson"
451 ceph::mutex mylock
= ceph::make_mutex("admin_socket::excute_command::mylock");
452 ceph::condition_variable mycond
;
453 C_SafeCond
fin(mylock
, mycond
, &done
, &rval
);
457 [&errss
, outbl
, &fin
](int r
, const std::string
& err
, bufferlist
& out
) {
463 std::unique_lock l
{mylock
};
464 mycond
.wait(l
, [&done
] { return done
;});
470 void AdminSocket::execute_command(
471 const std::vector
<std::string
>& cmdvec
,
472 const bufferlist
& inbl
,
473 std::function
<void(int,const std::string
&,bufferlist
&)> on_finish
)
479 ldout(m_cct
,10) << __func__
<< " cmdvec='" << cmdvec
<< "'" << dendl
;
480 if (!cmdmap_from_json(cmdvec
, &cmdmap
, errss
)) {
481 ldout(m_cct
, 0) << "AdminSocket: " << errss
.str() << dendl
;
482 return on_finish(-EINVAL
, "invalid json", empty
);
486 cmd_getval(cmdmap
, "format", format
);
487 cmd_getval(cmdmap
, "prefix", prefix
);
488 } catch (const bad_cmd_get
& e
) {
489 return on_finish(-EINVAL
, "invalid json, missing format and/or prefix",
493 Formatter
*f
= Formatter::create(format
, "json-pretty", "json-pretty");
495 auto [retval
, hook
] = find_matched_hook(prefix
, cmdmap
);
498 lderr(m_cct
) << "AdminSocket: request '" << cmdvec
499 << "' not defined" << dendl
;
501 return on_finish(-EINVAL
, "unknown command prefix "s
+ prefix
, empty
);
504 return on_finish(-EINVAL
, "invalid command json", empty
);
510 prefix
, cmdmap
, f
, inbl
,
511 [f
, on_finish
](int r
, const std::string
& err
, bufferlist
& out
) {
512 // handle either existing output in bufferlist *or* via formatter
513 if (r
>= 0 && out
.length() == 0) {
517 on_finish(r
, err
, out
);
520 std::unique_lock
l(lock
);
522 in_hook_cond
.notify_all();
525 std::pair
<int, AdminSocketHook
*>
526 AdminSocket::find_matched_hook(std::string
& prefix
,
527 const cmdmap_t
& cmdmap
)
529 std::unique_lock
l(lock
);
530 // Drop lock after done with the lookup to avoid cycles in cases where the
531 // hook takes the same lock that was held during calls to
532 // register/unregister, and set in_hook to allow unregister to wait for us
533 // before removing this hook.
534 auto [hooks_begin
, hooks_end
] = hooks
.equal_range(prefix
);
535 if (hooks_begin
== hooks_end
) {
536 return {ENOENT
, nullptr};
538 // make sure one of the registered commands with this prefix validates.
540 for (auto hook
= hooks_begin
; hook
!= hooks_end
; ++hook
) {
541 if (validate_cmd(m_cct
, hook
->second
.desc
, cmdmap
, errss
)) {
543 return {0, hook
->second
.hook
};
546 return {EINVAL
, nullptr};
549 void AdminSocket::queue_tell_command(cref_t
<MCommand
> m
)
551 ldout(m_cct
,10) << __func__
<< " " << *m
<< dendl
;
552 std::lock_guard
l(tell_lock
);
553 tell_queue
.push_back(std::move(m
));
556 void AdminSocket::queue_tell_command(cref_t
<MMonCommand
> m
)
558 ldout(m_cct
,10) << __func__
<< " " << *m
<< dendl
;
559 std::lock_guard
l(tell_lock
);
560 tell_legacy_queue
.push_back(std::move(m
));
564 int AdminSocket::register_command(std::string_view cmddesc
,
565 AdminSocketHook
*hook
,
566 std::string_view help
)
569 std::unique_lock
l(lock
);
570 string prefix
= cmddesc_get_prefix(cmddesc
);
571 auto i
= hooks
.find(prefix
);
572 if (i
!= hooks
.cend() &&
573 i
->second
.desc
== cmddesc
) {
574 ldout(m_cct
, 5) << "register_command " << prefix
575 << " cmddesc " << cmddesc
<< " hook " << hook
576 << " EEXIST" << dendl
;
579 ldout(m_cct
, 5) << "register_command " << prefix
<< " hook " << hook
581 hooks
.emplace_hint(i
,
582 std::piecewise_construct
,
583 std::forward_as_tuple(prefix
),
584 std::forward_as_tuple(hook
, cmddesc
, help
));
590 void AdminSocket::unregister_commands(const AdminSocketHook
*hook
)
592 std::unique_lock
l(lock
);
593 auto i
= hooks
.begin();
594 while (i
!= hooks
.end()) {
595 if (i
->second
.hook
== hook
) {
596 ldout(m_cct
, 5) << __func__
<< " " << i
->first
<< dendl
;
598 // If we are currently processing a command, wait for it to
599 // complete in case it referenced the hook that we are
601 in_hook_cond
.wait(l
, [this]() { return !in_hook
; });
609 class VersionHook
: public AdminSocketHook
{
611 int call(std::string_view command
, const cmdmap_t
& cmdmap
,
614 bufferlist
& out
) override
{
615 if (command
== "0"sv
) {
616 out
.append(CEPH_ADMIN_SOCK_VERSION
);
618 f
->open_object_section("version");
619 if (command
== "version") {
620 f
->dump_string("version", ceph_version_to_str());
621 f
->dump_string("release", ceph_release_to_str());
622 f
->dump_string("release_type", ceph_release_type());
623 } else if (command
== "git_version") {
624 f
->dump_string("git_version", git_version_to_str());
633 class HelpHook
: public AdminSocketHook
{
636 explicit HelpHook(AdminSocket
*as
) : m_as(as
) {}
637 int call(std::string_view command
, const cmdmap_t
& cmdmap
,
640 bufferlist
& out
) override
{
641 f
->open_object_section("help");
642 for (const auto& [command
, info
] : m_as
->hooks
) {
643 if (info
.help
.length())
644 f
->dump_string(command
.c_str(), info
.help
);
651 class GetdescsHook
: public AdminSocketHook
{
654 explicit GetdescsHook(AdminSocket
*as
) : m_as(as
) {}
655 int call(std::string_view command
, const cmdmap_t
& cmdmap
,
658 bufferlist
& out
) override
{
660 f
->open_object_section("command_descriptions");
661 for (const auto& [command
, info
] : m_as
->hooks
) {
662 // GCC 8 actually has [[maybe_unused]] on a structured binding
663 // do what you'd expect. GCC 7 does not.
665 ostringstream secname
;
666 secname
<< "cmd" << setfill('0') << std::setw(3) << cmdnum
;
667 dump_cmd_and_help_to_json(f
,
669 secname
.str().c_str(),
674 f
->close_section(); // command_descriptions
679 bool AdminSocket::init(const std::string
& path
)
681 ldout(m_cct
, 5) << "init " << path
<< dendl
;
683 /* Set up things for the new thread */
685 int pipe_rd
= -1, pipe_wr
= -1;
686 err
= create_wakeup_pipe(&pipe_rd
, &pipe_wr
);
688 lderr(m_cct
) << "AdminSocketConfigObs::init: error: " << err
<< dendl
;
692 err
= bind_and_listen(path
, &sock_fd
);
694 lderr(m_cct
) << "AdminSocketConfigObs::init: failed: " << err
<< dendl
;
700 /* Create new thread */
702 m_wakeup_rd_fd
= pipe_rd
;
703 m_wakeup_wr_fd
= pipe_wr
;
706 version_hook
= std::make_unique
<VersionHook
>();
707 register_command("0", version_hook
.get(), "");
708 register_command("version", version_hook
.get(), "get ceph version");
709 register_command("git_version", version_hook
.get(),
711 help_hook
= std::make_unique
<HelpHook
>(this);
712 register_command("help", help_hook
.get(),
713 "list available commands");
714 getdescs_hook
= std::make_unique
<GetdescsHook
>(this);
715 register_command("get_command_descriptions",
716 getdescs_hook
.get(), "list available commands");
718 th
= make_named_thread("admin_socket", &AdminSocket::entry
, this);
719 add_cleanup_file(m_path
.c_str());
723 void AdminSocket::shutdown()
725 // Under normal operation this is unlikely to occur. However for some unit
726 // tests, some object members are not initialized and so cannot be deleted
728 if (m_wakeup_wr_fd
< 0)
731 ldout(m_cct
, 5) << "shutdown" << dendl
;
734 auto err
= destroy_wakeup_pipe();
736 lderr(m_cct
) << "AdminSocket::shutdown: error: " << err
<< dendl
;
739 retry_sys_call(::close
, m_sock_fd
);
741 unregister_commands(version_hook
.get());
742 version_hook
.reset();
744 unregister_commands(help_hook
.get());
747 unregister_commands(getdescs_hook
.get());
748 getdescs_hook
.reset();
750 remove_cleanup_file(m_path
);
754 void AdminSocket::wakeup()
756 // Send a byte to the wakeup pipe that the thread is listening to
757 char buf
[1] = { 0x0 };
758 int r
= safe_write(m_wakeup_wr_fd
, buf
, sizeof(buf
));