ceph/src/boost/boost/mpi/python/serialize.hpp

   1 // Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
   2
   3 // Use, modification and distribution is subject to the Boost Software
   4 // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
   5 // http://www.boost.org/LICENSE_1_0.txt)
   6
   7 //  Authors: Douglas Gregor
   8
   9 /** @file serialize.hpp
  10  *
  11  *  This file provides Boost.Serialization support for Python objects
  12  *  within Boost.MPI. Python objects can be serialized in one of two
  13  *  ways. The default serialization method involves using the Python
  14  *  "pickle" module to pickle the Python objects, transmits the
  15  *  pickled representation, and unpickles the result when
  16  *  received. For C++ types that have been exposed to Python and
  17  *  registered with register_serialized(), objects are directly
  18  *  serialized for transmissing, skipping the pickling step.
  19  */
  20 #ifndef BOOST_MPI_PYTHON_SERIALIZE_HPP
  21 #define BOOST_MPI_PYTHON_SERIALIZE_HPP
  22
  23 #include <boost/mpi/python/config.hpp>
  24
  25 #include <boost/python/object.hpp>
  26 #include <boost/python/str.hpp>
  27 #include <boost/python/extract.hpp>
  28
  29 #include <memory>
  30 #include <map>
  31
  32 #include <boost/function/function3.hpp>
  33
  34 #include <boost/mpl/bool.hpp>
  35 #include <boost/mpl/if.hpp>
  36
  37 #include <boost/serialization/split_free.hpp>
  38 #include <boost/serialization/array.hpp>
  39 #include <boost/serialization/array_wrapper.hpp>
  40
  41 #include <boost/assert.hpp>
  42
  43 #include <boost/type_traits/is_fundamental.hpp>
  44
  45 #define BOOST_MPI_PYTHON_FORWARD_ONLY
  46 #include <boost/mpi/python.hpp>
  47
  48 /************************************************************************
  49  * Boost.Python Serialization Section                                   *
  50  ************************************************************************/
  51 #if !defined(BOOST_NO_SFINAE) && !defined(BOOST_NO_IS_CONVERTIBLE)
  52 /**
  53  * @brief Declare IArchive and OArchive as a Boost.Serialization
  54  * archives that can be used for Python objects.
  55  *
  56  * This macro can only be expanded from the global namespace. It only
  57  * requires that Archiver be forward-declared. IArchiver and OArchiver
  58  * will only support Serialization of Python objects by pickling
  59  * them. If the Archiver type should also support "direct"
  60  * serialization (for C++ types), use
  61  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE instead.
  62  */
  63 #  define BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver)        \
  64 namespace boost { namespace python { namespace api {    \
  65   template<typename R, typename T>                      \
  66   struct enable_binary< IArchiver , R, T> {};           \
  67                                                         \
  68   template<typename R, typename T>                      \
  69   struct enable_binary< OArchiver , R, T> {};           \
  70 } } }
  71 # else
  72 #  define BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver)
  73 #endif
  74
  75 /**
  76  * @brief Declare IArchiver and OArchiver as a Boost.Serialization
  77  * archives that can be used for Python objects and C++ objects
  78  * wrapped in Python.
  79  *
  80  * This macro can only be expanded from the global namespace. It only
  81  * requires that IArchiver and OArchiver be forward-declared. However,
  82  * note that you will also need to write
  83  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(IArchiver,
  84  * OArchiver) in one of your translation units.
  85
  86 DPG PICK UP HERE
  87  */
  88 #define BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(IArchiver, OArchiver) \
  89 BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver)                \
  90 namespace boost { namespace python { namespace detail {                 \
  91 template<>                                                              \
  92 BOOST_MPI_PYTHON_DECL direct_serialization_table< IArchiver , OArchiver >& \
  93  get_direct_serialization_table< IArchiver , OArchiver >();             \
  94 }                                                                       \
  95                                                                         \
  96 template<>                                                              \
  97 struct has_direct_serialization< IArchiver , OArchiver> : mpl::true_ { }; \
  98                                                                         \
  99 template<>                                                              \
 100 struct output_archiver< IArchiver > { typedef OArchiver type; };        \
 101                                                                         \
 102 template<>                                                              \
 103 struct input_archiver< OArchiver > { typedef IArchiver type; };         \
 104 } }
 105
 106 /**
 107  * @brief Define the implementation for Boost.Serialization archivers
 108  * that can be used for Python objects and C++ objects wrapped in
 109  * Python.
 110  *
 111  * This macro can only be expanded from the global namespace. It only
 112  * requires that IArchiver and OArchiver be forward-declared. Before
 113  * using this macro, you will need to declare IArchiver and OArchiver
 114  * as direct serialization archives with
 115  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(IArchiver, OArchiver).
 116  */
 117 #define BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(IArchiver, OArchiver) \
 118 namespace boost { namespace python { namespace detail {                 \
 119 template                                                                \
 120   class BOOST_MPI_PYTHON_DECL direct_serialization_table< IArchiver , OArchiver >; \
 121                                                                         \
 122 template<>                                                              \
 123  BOOST_MPI_PYTHON_DECL                                                  \
 124  direct_serialization_table< IArchiver , OArchiver >&                   \
 125  get_direct_serialization_table< IArchiver , OArchiver >( )             \
 126 {                                                                       \
 127   static direct_serialization_table< IArchiver, OArchiver > table;      \
 128   return table;                                                         \
 129 }                                                                       \
 130 } } }
 131
 132 namespace boost { namespace python {
 133
 134 /**
 135  * INTERNAL ONLY
 136  *
 137  * Provides access to the Python "pickle" module from within C++.
 138  */
 139 class BOOST_MPI_PYTHON_DECL pickle {
 140   struct data_t;
 141
 142 public:
 143   static str dumps(object obj, int protocol = -1);
 144   static object loads(str s);
 145
 146 private:
 147   static void initialize_data();
 148
 149   static data_t* data;
 150 };
 151
 152 /**
 153  * @brief Whether the input/output archiver pair has "direct"
 154  * serialization for C++ objects exposed in Python.
 155  *
 156  * Users do not typically need to specialize this trait, as it will be
 157  * specialized as part of the macro
 158  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
 159  */
 160 template<typename IArchiver, typename OArchiver>
 161 struct has_direct_serialization : mpl::false_ { };
 162
 163 /**
 164  *  @brief A metafunction that determines the output archiver for the
 165  *  given input archiver.
 166  *
 167  * Users do not typically need to specialize this trait, as it will be
 168  * specialized as part of the macro
 169  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
 170  */
 171 template<typename IArchiver> struct output_archiver { };
 172
 173 /**
 174  *  @brief A metafunction that determines the input archiver for the
 175  *  given output archiver.
 176  *
 177  * Users do not typically need to specialize this trait, as it will be
 178  * specialized as part of the macro
 179  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
 180  *
 181  */
 182 template<typename OArchiver> struct input_archiver { };
 183
 184 namespace detail {
 185
 186   /**
 187    * INTERNAL ONLY
 188    *
 189    * This class contains the direct-serialization code for the given
 190    * IArchiver/OArchiver pair. It is intended to be used as a
 191    * singleton class, and will be accessed when (de-)serializing a
 192    * Boost.Python object with an archiver that supports direct
 193    * serializations. Do not create instances of this class directly:
 194    * instead, use get_direct_serialization_table.
 195    */
 196   template<typename IArchiver, typename OArchiver>
 197   class BOOST_MPI_PYTHON_DECL direct_serialization_table
 198   {
 199   public:
 200     typedef boost::function3<void, OArchiver&, const object&, const unsigned int>
 201       saver_t;
 202     typedef boost::function3<void, IArchiver&, object&, const unsigned int>
 203       loader_t;
 204
 205     typedef std::map<PyTypeObject*, std::pair<int, saver_t> > savers_t;
 206     typedef std::map<int, loader_t> loaders_t;
 207
 208     /**
 209      * Retrieve the saver (serializer) associated with the Python
 210      * object @p obj.
 211      *
 212      *   @param obj The object we want to save. Only its (Python) type
 213      *   is important.
 214      *
 215      *   @param descriptor The value of the descriptor associated to
 216      *   the returned saver. Will be set to zero if no saver was found
 217      *   for @p obj.
 218      *
 219      *   @returns a function object that can be used to serialize this
 220      *   object (and other objects of the same type), if possible. If
 221      *   no saver can be found, returns an empty function object..
 222      */
 223     saver_t saver(const object& obj, int& descriptor)
 224     {
 225       typename savers_t::iterator pos = savers.find(obj.ptr()->ob_type);
 226       if (pos != savers.end()) {
 227         descriptor = pos->second.first;
 228         return pos->second.second;
 229       }
 230       else {
 231         descriptor = 0;
 232         return saver_t();
 233       }
 234     }
 235
 236     /**
 237      * Retrieve the loader (deserializer) associated with the given
 238      * descriptor.
 239      *
 240      *  @param descriptor The descriptor number provided by saver()
 241      *  when determining the saver for this type.
 242      *
 243      *  @returns a function object that can be used to deserialize an
 244      *  object whose type is the same as that corresponding to the
 245      *  descriptor. If the descriptor is unknown, the return value
 246      *  will be an empty function object.
 247      */
 248     loader_t loader(int descriptor)
 249     {
 250       typename loaders_t::iterator pos = loaders.find(descriptor);
 251       if (pos != loaders.end())
 252         return pos->second;
 253       else
 254         return loader_t();
 255     }
 256
 257     /**
 258      * Register the type T for direct serialization.
 259      *
 260      *  @param value A sample value of the type @c T. This may be used
 261      *  to compute the Python type associated with the C++ type @c T.
 262      *
 263      *  @param type The Python type associated with the C++ type @c
 264      *  T. If not provided, it will be computed from the same value @p
 265      *  value.
 266      */
 267     template<typename T>
 268     void register_type(const T& value = T(), PyTypeObject* type = 0)
 269     {
 270       // If the user did not provide us with a Python type, figure it
 271       // out for ourselves.
 272       if (!type) {
 273         object obj(value);
 274         type = obj.ptr()->ob_type;
 275       }
 276
 277       register_type(default_saver<T>(), default_loader<T>(type), value, type);
 278     }
 279
 280     /**
 281      * Register the type T for direct serialization.
 282      *
 283      *  @param saver A function object that will serialize a
 284      *  Boost.Python object (that represents a C++ object of type @c
 285      *  T) to an @c OArchive.
 286      *
 287      *  @param loader A function object that will deserialize from an
 288      *  @c IArchive into a Boost.Python object that represents a C++
 289      *  object of type @c T.
 290      *
 291      *  @param value A sample value of the type @c T. This may be used
 292      *  to compute the Python type associated with the C++ type @c T.
 293      *
 294      *  @param type The Python type associated with the C++ type @c
 295      *  T. If not provided, it will be computed from the same value @p
 296      *  value.
 297      */
 298     template<typename T>
 299     void register_type(const saver_t& saver, const loader_t& loader,
 300                        const T& value = T(), PyTypeObject* type = 0)
 301     {
 302       // If the user did not provide us with a Python type, figure it
 303       // out for ourselves.
 304       if (!type) {
 305         object obj(value);
 306         type = obj.ptr()->ob_type;
 307       }
 308
 309       int descriptor = savers.size() + 1;
 310       if (savers.find(type) != savers.end())
 311         return;
 312
 313       savers[type] = std::make_pair(descriptor, saver);
 314       loaders[descriptor] = loader;
 315     }
 316
 317   protected:
 318     template<typename T>
 319     struct default_saver {
 320       void operator()(OArchiver& ar, const object& obj, const unsigned int) {
 321         T value = extract<T>(obj)();
 322         ar << value;
 323       }
 324     };
 325
 326     template<typename T>
 327     struct default_loader {
 328       default_loader(PyTypeObject* type) : type(type) { }
 329
 330       void operator()(IArchiver& ar, object& obj, const unsigned int) {
 331         // If we can, extract the object in place.
 332         if (!is_fundamental<T>::value && obj && obj.ptr()->ob_type == type) {
 333           ar >> extract<T&>(obj)();
 334         } else {
 335           T value;
 336           ar >> value;
 337           obj = object(value);
 338         }
 339       }
 340
 341     private:
 342       PyTypeObject* type;
 343     };
 344
 345     savers_t savers;
 346     loaders_t loaders;
 347   };
 348
 349   /**
 350    * @brief Retrieve the direct-serialization table for an
 351    * IArchiver/OArchiver pair.
 352    *
 353    * This function is responsible for returning a reference to the
 354    * singleton direct-serialization table. Its primary template is
 355    * left undefined, to force the use of an explicit specialization
 356    * with a definition in a single translation unit. Use the macro
 357    * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL to define this
 358    * explicit specialization.
 359    */
 360   template<typename IArchiver, typename OArchiver>
 361   direct_serialization_table<IArchiver, OArchiver>&
 362   get_direct_serialization_table();
 363 } // end namespace detail
 364
 365 /**
 366  * @brief Register the type T for direct serialization.
 367  *
 368  * The @c register_serialized function registers a C++ type for direct
 369  * serialization with the given @c IArchiver/@c OArchiver pair. Direct
 370  * serialization elides the use of the Python @c pickle package when
 371  * serializing Python objects that represent C++ values. Direct
 372  * serialization can be beneficial both to improve serialization
 373  * performance (Python pickling can be very inefficient) and to permit
 374  * serialization for Python-wrapped C++ objects that do not support
 375  * pickling.
 376  *
 377  *  @param value A sample value of the type @c T. This may be used
 378  *  to compute the Python type associated with the C++ type @c T.
 379  *
 380  *  @param type The Python type associated with the C++ type @c
 381  *  T. If not provided, it will be computed from the same value @p
 382  *  value.
 383  */
 384 template<typename IArchiver, typename OArchiver, typename T>
 385 void
 386 register_serialized(const T& value = T(), PyTypeObject* type = 0)
 387 {
 388   detail::direct_serialization_table<IArchiver, OArchiver>& table =
 389     detail::get_direct_serialization_table<IArchiver, OArchiver>();
 390   table.register_type(value, type);
 391 }
 392
 393 namespace detail {
 394
 395 /// Save a Python object by pickling it.
 396 template<typename Archiver>
 397 void
 398 save_impl(Archiver& ar, const boost::python::object& obj,
 399           const unsigned int /*version*/,
 400           mpl::false_ /*has_direct_serialization*/)
 401 {
 402   boost::python::str py_string = boost::python::pickle::dumps(obj);
 403   int len = boost::python::extract<int>(py_string.attr("__len__")());
 404   const char* string = boost::python::extract<const char*>(py_string);
 405   ar << len << boost::serialization::make_array(string, len);
 406 }
 407
 408 /// Try to save a Python object by directly serializing it; fall back
 409 /// on pickling if required.
 410 template<typename Archiver>
 411 void
 412 save_impl(Archiver& ar, const boost::python::object& obj,
 413           const unsigned int version,
 414           mpl::true_ /*has_direct_serialization*/)
 415 {
 416   typedef Archiver OArchiver;
 417   typedef typename input_archiver<OArchiver>::type IArchiver;
 418   typedef typename direct_serialization_table<IArchiver, OArchiver>::saver_t
 419     saver_t;
 420
 421   direct_serialization_table<IArchiver, OArchiver>& table =
 422     get_direct_serialization_table<IArchiver, OArchiver>();
 423
 424   int descriptor = 0;
 425   if (saver_t saver = table.saver(obj, descriptor)) {
 426     ar << descriptor;
 427     saver(ar, obj, version);
 428   } else {
 429     // Pickle it
 430     ar << descriptor;
 431     detail::save_impl(ar, obj, version, mpl::false_());
 432   }
 433 }
 434
 435 /// Load a Python object by unpickling it
 436 template<typename Archiver>
 437 void
 438 load_impl(Archiver& ar, boost::python::object& obj,
 439           const unsigned int /*version*/,
 440           mpl::false_ /*has_direct_serialization*/)
 441 {
 442   int len;
 443   ar >> len;
 444
 445   std::auto_ptr<char> string(new char[len]);
 446   ar >> boost::serialization::make_array(string.get(), len);
 447   boost::python::str py_string(string.get(), len);
 448   obj = boost::python::pickle::loads(py_string);
 449 }
 450
 451 /// Try to load a Python object by directly deserializing it; fall back
 452 /// on unpickling if required.
 453 template<typename Archiver>
 454 void
 455 load_impl(Archiver& ar, boost::python::object& obj,
 456           const unsigned int version,
 457           mpl::true_ /*has_direct_serialization*/)
 458 {
 459   typedef Archiver IArchiver;
 460   typedef typename output_archiver<IArchiver>::type OArchiver;
 461   typedef typename direct_serialization_table<IArchiver, OArchiver>::loader_t
 462     loader_t;
 463
 464   direct_serialization_table<IArchiver, OArchiver>& table =
 465     get_direct_serialization_table<IArchiver, OArchiver>();
 466
 467   int descriptor;
 468   ar >> descriptor;
 469
 470   if (descriptor) {
 471     loader_t loader = table.loader(descriptor);
 472     BOOST_ASSERT(loader);
 473
 474     loader(ar, obj, version);
 475   } else {
 476     // Unpickle it
 477     detail::load_impl(ar, obj, version, mpl::false_());
 478   }
 479 }
 480
 481 } // end namespace detail
 482
 483 template<typename Archiver>
 484 void
 485 save(Archiver& ar, const boost::python::object& obj,
 486      const unsigned int version)
 487 {
 488   typedef Archiver OArchiver;
 489   typedef typename input_archiver<OArchiver>::type IArchiver;
 490
 491   detail::save_impl(ar, obj, version,
 492                     has_direct_serialization<IArchiver, OArchiver>());
 493 }
 494
 495 template<typename Archiver>
 496 void
 497 load(Archiver& ar, boost::python::object& obj,
 498      const unsigned int version)
 499 {
 500   typedef Archiver IArchiver;
 501   typedef typename output_archiver<IArchiver>::type OArchiver;
 502
 503   detail::load_impl(ar, obj, version,
 504                     has_direct_serialization<IArchiver, OArchiver>());
 505 }
 506
 507 template<typename Archive>
 508 inline void
 509 serialize(Archive& ar, boost::python::object& obj, const unsigned int version)
 510 {
 511   boost::serialization::split_free(ar, obj, version);
 512 }
 513
 514 } } // end namespace boost::python
 515
 516 /************************************************************************
 517  * Boost.MPI-Specific Section                                           *
 518  ************************************************************************/
 519 namespace boost { namespace mpi {
 520  class packed_iarchive;
 521  class packed_oarchive;
 522 } } // end namespace boost::mpi
 523
 524 BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(
 525   ::boost::mpi::packed_iarchive,
 526   ::boost::mpi::packed_oarchive)
 527
 528 namespace boost { namespace mpi { namespace python {
 529
 530 template<typename T>
 531 void
 532 register_serialized(const T& value, PyTypeObject* type)
 533 {
 534   using boost::python::register_serialized;
 535   register_serialized<packed_iarchive, packed_oarchive>(value, type);
 536 }
 537
 538 } } } // end namespace boost::mpi::python
 539
 540 #endif // BOOST_MPI_PYTHON_SERIALIZE_HPP