1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2017 John Spray <john.spray@redhat.com>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #include "include/stringify.h"
16 #include "common/errno.h"
17 #include "common/backport14.h"
19 #include "BaseMgrModule.h"
21 #include "BaseMgrStandbyModule.h"
24 #include "ActivePyModules.h"
26 #include "PyModuleRegistry.h"
28 // definition for non-const static member
29 std::string
PyModuleRegistry::config_prefix
;
33 #define dout_context g_ceph_context
34 #define dout_subsys ceph_subsys_mgr
37 #define dout_prefix *_dout << "mgr[py] "
40 PyObject
* log_write(PyObject
*, PyObject
* args
) {
42 if (PyArg_ParseTuple(args
, "s", &m
)) {
44 if (len
&& m
[len
-1] == '\n') {
47 dout(4) << m
<< dendl
;
52 PyObject
* log_flush(PyObject
*, PyObject
*){
56 static PyMethodDef log_methods
[] = {
57 {"write", log_write
, METH_VARARGS
, "write stdout and stderr"},
58 {"flush", log_flush
, METH_VARARGS
, "flush"},
59 {nullptr, nullptr, 0, nullptr}
64 #define dout_prefix *_dout << "mgr " << __func__ << " "
68 std::string
PyModule::get_site_packages()
70 std::stringstream site_packages
;
72 // CPython doesn't auto-add site-packages dirs to sys.path for us,
73 // but it does provide a module that we can ask for them.
74 auto site_module
= PyImport_ImportModule("site");
77 auto site_packages_fn
= PyObject_GetAttrString(site_module
, "getsitepackages");
78 if (site_packages_fn
!= nullptr) {
79 auto site_packages_list
= PyObject_CallObject(site_packages_fn
, nullptr);
80 assert(site_packages_list
);
82 auto n
= PyList_Size(site_packages_list
);
83 for (Py_ssize_t i
= 0; i
< n
; ++i
) {
87 site_packages
<< PyString_AsString(PyList_GetItem(site_packages_list
, i
));
90 Py_DECREF(site_packages_list
);
91 Py_DECREF(site_packages_fn
);
93 // Fall back to generating our own site-packages paths by imitating
94 // what the standard site.py does. This is annoying but it lets us
95 // run inside virtualenvs :-/
97 auto site_packages_fn
= PyObject_GetAttrString(site_module
, "addsitepackages");
98 assert(site_packages_fn
);
100 auto known_paths
= PySet_New(nullptr);
101 auto pArgs
= PyTuple_Pack(1, known_paths
);
102 PyObject_CallObject(site_packages_fn
, pArgs
);
104 Py_DECREF(known_paths
);
105 Py_DECREF(site_packages_fn
);
107 auto sys_module
= PyImport_ImportModule("sys");
109 auto sys_path
= PyObject_GetAttrString(sys_module
, "path");
112 dout(1) << "sys.path:" << dendl
;
113 auto n
= PyList_Size(sys_path
);
115 for (Py_ssize_t i
= 0; i
< n
; ++i
) {
116 dout(1) << " " << PyString_AsString(PyList_GetItem(sys_path
, i
)) << dendl
;
120 site_packages
<< ":";
122 site_packages
<< PyString_AsString(PyList_GetItem(sys_path
, i
));
126 Py_DECREF(sys_module
);
129 Py_DECREF(site_module
);
131 return site_packages
.str();
134 int PyModuleRegistry::init(const MgrMap
&map
)
136 Mutex::Locker
locker(lock
);
138 // Don't try and init me if you don't really have a map
139 assert(map
.epoch
> 0);
143 // namespace in config-key prefixed by "mgr/"
144 config_prefix
= std::string(g_conf
->name
.get_type_str()) + "/";
146 // Set up global python interpreter
147 Py_SetProgramName(const_cast<char*>(PYTHON_EXECUTABLE
));
150 // Let CPython know that we will be calling it back from other
151 // threads in future.
152 if (! PyEval_ThreadsInitialized()) {
153 PyEval_InitThreads();
156 // Drop the GIL and remember the main thread state (current
157 // thread state becomes NULL)
158 pMainThreadState
= PyEval_SaveThread();
159 assert(pMainThreadState
!= nullptr);
161 std::list
<std::string
> failed_modules
;
164 for (const auto& module_name
: mgr_map
.modules
) {
165 dout(1) << "Loading python module '" << module_name
<< "'" << dendl
;
166 auto mod
= ceph::make_unique
<PyModule
>(module_name
);
167 int r
= mod
->load(pMainThreadState
);
169 // Don't use handle_pyerror() here; we don't have the GIL
170 // or the right thread state (this is deliberate).
171 derr
<< "Error loading module '" << module_name
<< "': "
172 << cpp_strerror(r
) << dendl
;
173 failed_modules
.push_back(module_name
);
174 // Don't drop out here, load the other modules
177 modules
[module_name
] = std::move(mod
);
181 if (!failed_modules
.empty()) {
182 clog
->error() << "Failed to load ceph-mgr modules: " << joinify(
183 failed_modules
.begin(), failed_modules
.end(), std::string(", "));
190 int PyModule::load(PyThreadState
*pMainThreadState
)
192 assert(pMainThreadState
!= nullptr);
194 // Configure sub-interpreter and construct C++-generated python classes
196 SafeThreadState
sts(pMainThreadState
);
199 auto thread_state
= Py_NewInterpreter();
200 if (thread_state
== nullptr) {
201 derr
<< "Failed to create python sub-interpreter for '" << module_name
<< '"' << dendl
;
204 pMyThreadState
.set(thread_state
);
205 // Some python modules do not cope with an unpopulated argv, so lets
206 // fake one. This step also picks up site-packages into sys.path.
207 const char *argv
[] = {"ceph-mgr"};
208 PySys_SetArgv(1, (char**)argv
);
210 if (g_conf
->daemonize
) {
211 auto py_logger
= Py_InitModule("ceph_logger", log_methods
);
212 #if PY_MAJOR_VERSION >= 3
213 PySys_SetObject("stderr", py_logger
);
214 PySys_SetObject("stdout", py_logger
);
216 PySys_SetObject(const_cast<char*>("stderr"), py_logger
);
217 PySys_SetObject(const_cast<char*>("stdout"), py_logger
);
221 // Configure sys.path to include mgr_module_path
222 std::string sys_path
= std::string(Py_GetPath()) + ":" + get_site_packages()
223 + ":" + g_conf
->get_val
<std::string
>("mgr_module_path");
224 dout(10) << "Computed sys.path '" << sys_path
<< "'" << dendl
;
226 PySys_SetPath(const_cast<char*>(sys_path
.c_str()));
229 PyMethodDef ModuleMethods
[] = {
234 PyObject
*ceph_module
= Py_InitModule("ceph_module", ModuleMethods
);
235 assert(ceph_module
!= nullptr);
237 auto load_class
= [ceph_module
](const char *name
, PyTypeObject
*type
)
239 type
->tp_new
= PyType_GenericNew
;
240 if (PyType_Ready(type
) < 0) {
245 PyModule_AddObject(ceph_module
, name
, (PyObject
*)type
);
248 load_class("BaseMgrModule", &BaseMgrModuleType
);
249 load_class("BaseMgrStandbyModule", &BaseMgrStandbyModuleType
);
250 load_class("BasePyOSDMap", &BasePyOSDMapType
);
251 load_class("BasePyOSDMapIncremental", &BasePyOSDMapIncrementalType
);
252 load_class("BasePyCRUSH", &BasePyCRUSHType
);
255 // Environment is all good, import the external module
257 Gil
gil(pMyThreadState
);
260 PyObject
*pName
= PyString_FromString(module_name
.c_str());
261 auto pModule
= PyImport_Import(pName
);
263 if (pModule
== nullptr) {
264 derr
<< "Module not found: '" << module_name
<< "'" << dendl
;
265 derr
<< handle_pyerror() << dendl
;
270 // TODO: let them call it what they want instead of just 'Module'
271 pClass
= PyObject_GetAttrString(pModule
, (const char*)"Module");
272 if (pClass
== nullptr) {
273 derr
<< "Class not found in module '" << module_name
<< "'" << dendl
;
274 derr
<< handle_pyerror() << dendl
;
278 pStandbyClass
= PyObject_GetAttrString(pModule
,
279 (const char*)"StandbyModule");
281 dout(4) << "Standby mode available in module '" << module_name
284 dout(4) << "Standby mode not provided by module '" << module_name
295 PyModule::~PyModule()
297 if (pMyThreadState
.ts
!= nullptr) {
298 Gil
gil(pMyThreadState
, true);
300 Py_XDECREF(pStandbyClass
);
304 void PyModuleRegistry::standby_start(MonClient
*monc
)
306 Mutex::Locker
l(lock
);
307 assert(active_modules
== nullptr);
308 assert(standby_modules
== nullptr);
309 assert(is_initialized());
311 dout(4) << "Starting modules in standby mode" << dendl
;
313 standby_modules
.reset(new StandbyPyModules(monc
, mgr_map
));
315 std::set
<std::string
> failed_modules
;
316 for (const auto &i
: modules
) {
317 if (i
.second
->pStandbyClass
) {
318 dout(4) << "starting module " << i
.second
->get_name() << dendl
;
319 int r
= standby_modules
->start_one(i
.first
,
320 i
.second
->pStandbyClass
,
321 i
.second
->pMyThreadState
);
323 derr
<< "failed to start module '" << i
.second
->get_name()
325 failed_modules
.insert(i
.second
->get_name());
326 // Continue trying to load any other modules
329 dout(4) << "skipping module '" << i
.second
->get_name() << "' because "
330 "it does not implement a standby mode" << dendl
;
334 if (!failed_modules
.empty()) {
335 clog
->error() << "Failed to execute ceph-mgr module(s) in standby mode: "
336 << joinify(failed_modules
.begin(), failed_modules
.end(),
341 void PyModuleRegistry::active_start(
342 PyModuleConfig
&config_
,
343 DaemonStateIndex
&ds
, ClusterState
&cs
, MonClient
&mc
,
344 LogChannelRef clog_
, Objecter
&objecter_
, Client
&client_
,
347 Mutex::Locker
locker(lock
);
349 dout(4) << "Starting modules in active mode" << dendl
;
351 assert(active_modules
== nullptr);
352 assert(is_initialized());
354 if (standby_modules
!= nullptr) {
355 standby_modules
->shutdown();
356 standby_modules
.reset();
359 active_modules
.reset(new ActivePyModules(
360 config_
, ds
, cs
, mc
, clog_
, objecter_
, client_
, f
));
362 for (const auto &i
: modules
) {
363 dout(4) << "Starting " << i
.first
<< dendl
;
364 int r
= active_modules
->start_one(i
.first
,
366 i
.second
->pMyThreadState
);
368 derr
<< "Failed to run module in active mode ('" << i
.first
<< "')"
374 void PyModuleRegistry::active_shutdown()
376 Mutex::Locker
locker(lock
);
378 if (active_modules
!= nullptr) {
379 active_modules
->shutdown();
380 active_modules
.reset();
384 void PyModuleRegistry::shutdown()
386 Mutex::Locker
locker(lock
);
388 if (standby_modules
!= nullptr) {
389 standby_modules
->shutdown();
390 standby_modules
.reset();
393 // Ideally, now, we'd be able to do this for all modules:
395 // Py_EndInterpreter(pMyThreadState);
396 // PyThreadState_Swap(pMainThreadState);
398 // Unfortunately, if the module has any other *python* threads active
399 // at this point, Py_EndInterpreter() will abort with:
401 // Fatal Python error: Py_EndInterpreter: not the last thread
403 // This can happen when using CherryPy in a module, becuase CherryPy
404 // runs an extra thread as a timeout monitor, which spends most of its
405 // life inside a time.sleep(60). Unless you are very, very lucky with
406 // the timing calling this destructor, that thread will still be stuck
407 // in a sleep, and Py_EndInterpreter() will abort.
409 // This could of course also happen with a poorly written module which
410 // made no attempt to clean up any additional threads it created.
412 // The safest thing to do is just not call Py_EndInterpreter(), and
413 // let Py_Finalize() kill everything after all modules are shut down.
417 PyEval_RestoreThread(pMainThreadState
);
421 static void _list_modules(
422 const std::string path
,
423 std::set
<std::string
> *modules
)
425 DIR *dir
= opendir(path
.c_str());
429 struct dirent
*entry
= NULL
;
430 while ((entry
= readdir(dir
)) != NULL
) {
431 string
n(entry
->d_name
);
432 string fn
= path
+ "/" + n
;
434 int r
= ::stat(fn
.c_str(), &st
);
435 if (r
== 0 && S_ISDIR(st
.st_mode
)) {
436 string initfn
= fn
+ "/module.py";
437 r
= ::stat(initfn
.c_str(), &st
);
446 void PyModuleRegistry::list_modules(std::set
<std::string
> *modules
)
448 _list_modules(g_conf
->get_val
<std::string
>("mgr_module_path"), modules
);