ceph/src/boost/libs/mpi/include/boost/mpi/python/serialize.hpp

   1 // Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
   2
   3 // Use, modification and distribution is subject to the Boost Software
   4 // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
   5 // http://www.boost.org/LICENSE_1_0.txt)
   6
   7 //  Authors: Douglas Gregor
   8
   9 /** @file serialize.hpp
  10  *
  11  *  This file provides Boost.Serialization support for Python objects
  12  *  within Boost.MPI. Python objects can be serialized in one of two
  13  *  ways. The default serialization method involves using the Python
  14  *  "pickle" module to pickle the Python objects, transmits the
  15  *  pickled representation, and unpickles the result when
  16  *  received. For C++ types that have been exposed to Python and
  17  *  registered with register_serialized(), objects are directly
  18  *  serialized for transmissing, skipping the pickling step.
  19  */
  20 #ifndef BOOST_MPI_PYTHON_SERIALIZE_HPP
  21 #define BOOST_MPI_PYTHON_SERIALIZE_HPP
  22
  23 #include <boost/mpi/python/config.hpp>
  24
  25 #include <boost/python/object.hpp>
  26 #include <boost/python/str.hpp>
  27 #include <boost/python/extract.hpp>
  28
  29 #include <memory>
  30 #include <map>
  31
  32 #include <boost/function/function3.hpp>
  33
  34 #include <boost/mpl/bool.hpp>
  35 #include <boost/mpl/if.hpp>
  36
  37 #include <boost/serialization/split_free.hpp>
  38 #include <boost/serialization/array.hpp>
  39
  40 #include <boost/assert.hpp>
  41
  42 #include <boost/type_traits/is_fundamental.hpp>
  43
  44 #define BOOST_MPI_PYTHON_FORWARD_ONLY
  45 #include <boost/mpi/python.hpp>
  46
  47 /************************************************************************
  48  * Boost.Python Serialization Section                                   *
  49  ************************************************************************/
  50 #if !defined(BOOST_NO_SFINAE) && !defined(BOOST_NO_IS_CONVERTIBLE)
  51 /**
  52  * @brief Declare IArchive and OArchive as a Boost.Serialization
  53  * archives that can be used for Python objects.
  54  *
  55  * This macro can only be expanded from the global namespace. It only
  56  * requires that Archiver be forward-declared. IArchiver and OArchiver
  57  * will only support Serialization of Python objects by pickling
  58  * them. If the Archiver type should also support "direct"
  59  * serialization (for C++ types), use
  60  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE instead.
  61  */
  62 #  define BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver)        \
  63 namespace boost { namespace python { namespace api {    \
  64   template<typename R, typename T>                      \
  65   struct enable_binary< IArchiver , R, T> {};           \
  66                                                         \
  67   template<typename R, typename T>                      \
  68   struct enable_binary< OArchiver , R, T> {};           \
  69 } } }
  70 # else
  71 #  define BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver)
  72 #endif
  73
  74 /**
  75  * @brief Declare IArchiver and OArchiver as a Boost.Serialization
  76  * archives that can be used for Python objects and C++ objects
  77  * wrapped in Python.
  78  *
  79  * This macro can only be expanded from the global namespace. It only
  80  * requires that IArchiver and OArchiver be forward-declared. However,
  81  * note that you will also need to write
  82  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(IArchiver,
  83  * OArchiver) in one of your translation units.
  84
  85 DPG PICK UP HERE
  86  */
  87 #define BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(IArchiver, OArchiver) \
  88 BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver)                \
  89 namespace boost { namespace python { namespace detail {                 \
  90 template<>                                                              \
  91 BOOST_MPI_PYTHON_DECL direct_serialization_table< IArchiver , OArchiver >& \
  92  get_direct_serialization_table< IArchiver , OArchiver >();             \
  93 }                                                                       \
  94                                                                         \
  95 template<>                                                              \
  96 struct has_direct_serialization< IArchiver , OArchiver> : mpl::true_ { }; \
  97                                                                         \
  98 template<>                                                              \
  99 struct output_archiver< IArchiver > { typedef OArchiver type; };        \
 100                                                                         \
 101 template<>                                                              \
 102 struct input_archiver< OArchiver > { typedef IArchiver type; };         \
 103 } }
 104
 105 /**
 106  * @brief Define the implementation for Boost.Serialization archivers
 107  * that can be used for Python objects and C++ objects wrapped in
 108  * Python.
 109  *
 110  * This macro can only be expanded from the global namespace. It only
 111  * requires that IArchiver and OArchiver be forward-declared. Before
 112  * using this macro, you will need to declare IArchiver and OArchiver
 113  * as direct serialization archives with
 114  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(IArchiver, OArchiver).
 115  */
 116 #define BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(IArchiver, OArchiver) \
 117 namespace boost { namespace python { namespace detail {                 \
 118 template                                                                \
 119   class BOOST_MPI_PYTHON_DECL direct_serialization_table< IArchiver , OArchiver >; \
 120                                                                         \
 121 template<>                                                              \
 122  BOOST_MPI_PYTHON_DECL                                                  \
 123  direct_serialization_table< IArchiver , OArchiver >&                   \
 124  get_direct_serialization_table< IArchiver , OArchiver >( )             \
 125 {                                                                       \
 126   static direct_serialization_table< IArchiver, OArchiver > table;      \
 127   return table;                                                         \
 128 }                                                                       \
 129 } } }
 130
 131 namespace boost { namespace python {
 132
 133 /**
 134  * INTERNAL ONLY
 135  *
 136  * Provides access to the Python "pickle" module from within C++.
 137  */
 138 class BOOST_MPI_PYTHON_DECL pickle {
 139   struct data_t;
 140
 141 public:
 142   static str dumps(object obj, int protocol = -1);
 143   static object loads(str s);
 144
 145 private:
 146   static void initialize_data();
 147
 148   static data_t* data;
 149 };
 150
 151 /**
 152  * @brief Whether the input/output archiver pair has "direct"
 153  * serialization for C++ objects exposed in Python.
 154  *
 155  * Users do not typically need to specialize this trait, as it will be
 156  * specialized as part of the macro
 157  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
 158  */
 159 template<typename IArchiver, typename OArchiver>
 160 struct has_direct_serialization : mpl::false_ { };
 161
 162 /**
 163  *  @brief A metafunction that determines the output archiver for the
 164  *  given input archiver.
 165  *
 166  * Users do not typically need to specialize this trait, as it will be
 167  * specialized as part of the macro
 168  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
 169  */
 170 template<typename IArchiver> struct output_archiver { };
 171
 172 /**
 173  *  @brief A metafunction that determines the input archiver for the
 174  *  given output archiver.
 175  *
 176  * Users do not typically need to specialize this trait, as it will be
 177  * specialized as part of the macro
 178  * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
 179  *
 180  */
 181 template<typename OArchiver> struct input_archiver { };
 182
 183 namespace detail {
 184
 185   /**
 186    * INTERNAL ONLY
 187    *
 188    * This class contains the direct-serialization code for the given
 189    * IArchiver/OArchiver pair. It is intended to be used as a
 190    * singleton class, and will be accessed when (de-)serializing a
 191    * Boost.Python object with an archiver that supports direct
 192    * serializations. Do not create instances of this class directly:
 193    * instead, use get_direct_serialization_table.
 194    */
 195   template<typename IArchiver, typename OArchiver>
 196   class BOOST_MPI_PYTHON_DECL direct_serialization_table
 197   {
 198   public:
 199     typedef boost::function3<void, OArchiver&, const object&, const unsigned int>
 200       saver_t;
 201     typedef boost::function3<void, IArchiver&, object&, const unsigned int>
 202       loader_t;
 203
 204     typedef std::map<PyTypeObject*, std::pair<int, saver_t> > savers_t;
 205     typedef std::map<int, loader_t> loaders_t;
 206
 207     /**
 208      * Retrieve the saver (serializer) associated with the Python
 209      * object @p obj.
 210      *
 211      *   @param obj The object we want to save. Only its (Python) type
 212      *   is important.
 213      *
 214      *   @param descriptor The value of the descriptor associated to
 215      *   the returned saver. Will be set to zero if no saver was found
 216      *   for @p obj.
 217      *
 218      *   @returns a function object that can be used to serialize this
 219      *   object (and other objects of the same type), if possible. If
 220      *   no saver can be found, returns an empty function object..
 221      */
 222     saver_t saver(const object& obj, int& descriptor)
 223     {
 224       typename savers_t::iterator pos = savers.find(obj.ptr()->ob_type);
 225       if (pos != savers.end()) {
 226         descriptor = pos->second.first;
 227         return pos->second.second;
 228       }
 229       else {
 230         descriptor = 0;
 231         return saver_t();
 232       }
 233     }
 234
 235     /**
 236      * Retrieve the loader (deserializer) associated with the given
 237      * descriptor.
 238      *
 239      *  @param descriptor The descriptor number provided by saver()
 240      *  when determining the saver for this type.
 241      *
 242      *  @returns a function object that can be used to deserialize an
 243      *  object whose type is the same as that corresponding to the
 244      *  descriptor. If the descriptor is unknown, the return value
 245      *  will be an empty function object.
 246      */
 247     loader_t loader(int descriptor)
 248     {
 249       typename loaders_t::iterator pos = loaders.find(descriptor);
 250       if (pos != loaders.end())
 251         return pos->second;
 252       else
 253         return loader_t();
 254     }
 255
 256     /**
 257      * Register the type T for direct serialization.
 258      *
 259      *  @param value A sample value of the type @c T. This may be used
 260      *  to compute the Python type associated with the C++ type @c T.
 261      *
 262      *  @param type The Python type associated with the C++ type @c
 263      *  T. If not provided, it will be computed from the same value @p
 264      *  value.
 265      */
 266     template<typename T>
 267     void register_type(const T& value = T(), PyTypeObject* type = 0)
 268     {
 269       // If the user did not provide us with a Python type, figure it
 270       // out for ourselves.
 271       if (!type) {
 272         object obj(value);
 273         type = obj.ptr()->ob_type;
 274       }
 275
 276       register_type(default_saver<T>(), default_loader<T>(type), value, type);
 277     }
 278
 279     /**
 280      * Register the type T for direct serialization.
 281      *
 282      *  @param saver A function object that will serialize a
 283      *  Boost.Python object (that represents a C++ object of type @c
 284      *  T) to an @c OArchive.
 285      *
 286      *  @param loader A function object that will deserialize from an
 287      *  @c IArchive into a Boost.Python object that represents a C++
 288      *  object of type @c T.
 289      *
 290      *  @param value A sample value of the type @c T. This may be used
 291      *  to compute the Python type associated with the C++ type @c T.
 292      *
 293      *  @param type The Python type associated with the C++ type @c
 294      *  T. If not provided, it will be computed from the same value @p
 295      *  value.
 296      */
 297     template<typename T>
 298     void register_type(const saver_t& saver, const loader_t& loader,
 299                        const T& value = T(), PyTypeObject* type = 0)
 300     {
 301       // If the user did not provide us with a Python type, figure it
 302       // out for ourselves.
 303       if (!type) {
 304         object obj(value);
 305         type = obj.ptr()->ob_type;
 306       }
 307
 308       int descriptor = savers.size() + 1;
 309       if (savers.find(type) != savers.end())
 310         return;
 311
 312       savers[type] = std::make_pair(descriptor, saver);
 313       loaders[descriptor] = loader;
 314     }
 315
 316   protected:
 317     template<typename T>
 318     struct default_saver {
 319       void operator()(OArchiver& ar, const object& obj, const unsigned int) {
 320         T value = extract<T>(obj)();
 321         ar << value;
 322       }
 323     };
 324
 325     template<typename T>
 326     struct default_loader {
 327       default_loader(PyTypeObject* type) : type(type) { }
 328
 329       void operator()(IArchiver& ar, object& obj, const unsigned int) {
 330         // If we can, extract the object in place.
 331         if (!is_fundamental<T>::value && obj && obj.ptr()->ob_type == type) {
 332           ar >> extract<T&>(obj)();
 333         } else {
 334           T value;
 335           ar >> value;
 336           obj = object(value);
 337         }
 338       }
 339
 340     private:
 341       PyTypeObject* type;
 342     };
 343
 344     savers_t savers;
 345     loaders_t loaders;
 346   };
 347
 348   /**
 349    * @brief Retrieve the direct-serialization table for an
 350    * IArchiver/OArchiver pair.
 351    *
 352    * This function is responsible for returning a reference to the
 353    * singleton direct-serialization table. Its primary template is
 354    * left undefined, to force the use of an explicit specialization
 355    * with a definition in a single translation unit. Use the macro
 356    * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL to define this
 357    * explicit specialization.
 358    */
 359   template<typename IArchiver, typename OArchiver>
 360   direct_serialization_table<IArchiver, OArchiver>&
 361   get_direct_serialization_table();
 362 } // end namespace detail
 363
 364 /**
 365  * @brief Register the type T for direct serialization.
 366  *
 367  * The @c register_serialized function registers a C++ type for direct
 368  * serialization with the given @c IArchiver/@c OArchiver pair. Direct
 369  * serialization elides the use of the Python @c pickle package when
 370  * serializing Python objects that represent C++ values. Direct
 371  * serialization can be beneficial both to improve serialization
 372  * performance (Python pickling can be very inefficient) and to permit
 373  * serialization for Python-wrapped C++ objects that do not support
 374  * pickling.
 375  *
 376  *  @param value A sample value of the type @c T. This may be used
 377  *  to compute the Python type associated with the C++ type @c T.
 378  *
 379  *  @param type The Python type associated with the C++ type @c
 380  *  T. If not provided, it will be computed from the same value @p
 381  *  value.
 382  */
 383 template<typename IArchiver, typename OArchiver, typename T>
 384 void
 385 register_serialized(const T& value = T(), PyTypeObject* type = 0)
 386 {
 387   detail::direct_serialization_table<IArchiver, OArchiver>& table =
 388     detail::get_direct_serialization_table<IArchiver, OArchiver>();
 389   table.register_type(value, type);
 390 }
 391
 392 namespace detail {
 393
 394 /// Save a Python object by pickling it.
 395 template<typename Archiver>
 396 void
 397 save_impl(Archiver& ar, const boost::python::object& obj,
 398           const unsigned int /*version*/,
 399           mpl::false_ /*has_direct_serialization*/)
 400 {
 401   boost::python::str py_string = boost::python::pickle::dumps(obj);
 402   int len = boost::python::extract<int>(py_string.attr("__len__")());
 403   const char* string = boost::python::extract<const char*>(py_string);
 404   ar << len << boost::serialization::make_array(string, len);
 405 }
 406
 407 /// Try to save a Python object by directly serializing it; fall back
 408 /// on pickling if required.
 409 template<typename Archiver>
 410 void
 411 save_impl(Archiver& ar, const boost::python::object& obj,
 412           const unsigned int version,
 413           mpl::true_ /*has_direct_serialization*/)
 414 {
 415   typedef Archiver OArchiver;
 416   typedef typename input_archiver<OArchiver>::type IArchiver;
 417   typedef typename direct_serialization_table<IArchiver, OArchiver>::saver_t
 418     saver_t;
 419
 420   direct_serialization_table<IArchiver, OArchiver>& table =
 421     get_direct_serialization_table<IArchiver, OArchiver>();
 422
 423   int descriptor = 0;
 424   if (saver_t saver = table.saver(obj, descriptor)) {
 425     ar << descriptor;
 426     saver(ar, obj, version);
 427   } else {
 428     // Pickle it
 429     ar << descriptor;
 430     detail::save_impl(ar, obj, version, mpl::false_());
 431   }
 432 }
 433
 434 /// Load a Python object by unpickling it
 435 template<typename Archiver>
 436 void
 437 load_impl(Archiver& ar, boost::python::object& obj,
 438           const unsigned int /*version*/,
 439           mpl::false_ /*has_direct_serialization*/)
 440 {
 441   int len;
 442   ar >> len;
 443
 444   std::auto_ptr<char> string(new char[len]);
 445   ar >> boost::serialization::make_array(string.get(), len);
 446   boost::python::str py_string(string.get(), len);
 447   obj = boost::python::pickle::loads(py_string);
 448 }
 449
 450 /// Try to load a Python object by directly deserializing it; fall back
 451 /// on unpickling if required.
 452 template<typename Archiver>
 453 void
 454 load_impl(Archiver& ar, boost::python::object& obj,
 455           const unsigned int version,
 456           mpl::true_ /*has_direct_serialization*/)
 457 {
 458   typedef Archiver IArchiver;
 459   typedef typename output_archiver<IArchiver>::type OArchiver;
 460   typedef typename direct_serialization_table<IArchiver, OArchiver>::loader_t
 461     loader_t;
 462
 463   direct_serialization_table<IArchiver, OArchiver>& table =
 464     get_direct_serialization_table<IArchiver, OArchiver>();
 465
 466   int descriptor;
 467   ar >> descriptor;
 468
 469   if (descriptor) {
 470     loader_t loader = table.loader(descriptor);
 471     BOOST_ASSERT(loader);
 472
 473     loader(ar, obj, version);
 474   } else {
 475     // Unpickle it
 476     detail::load_impl(ar, obj, version, mpl::false_());
 477   }
 478 }
 479
 480 } // end namespace detail
 481
 482 template<typename Archiver>
 483 void
 484 save(Archiver& ar, const boost::python::object& obj,
 485      const unsigned int version)
 486 {
 487   typedef Archiver OArchiver;
 488   typedef typename input_archiver<OArchiver>::type IArchiver;
 489
 490   detail::save_impl(ar, obj, version,
 491                     has_direct_serialization<IArchiver, OArchiver>());
 492 }
 493
 494 template<typename Archiver>
 495 void
 496 load(Archiver& ar, boost::python::object& obj,
 497      const unsigned int version)
 498 {
 499   typedef Archiver IArchiver;
 500   typedef typename output_archiver<IArchiver>::type OArchiver;
 501
 502   detail::load_impl(ar, obj, version,
 503                     has_direct_serialization<IArchiver, OArchiver>());
 504 }
 505
 506 template<typename Archive>
 507 inline void
 508 serialize(Archive& ar, boost::python::object& obj, const unsigned int version)
 509 {
 510   boost::serialization::split_free(ar, obj, version);
 511 }
 512
 513 } } // end namespace boost::python
 514
 515 /************************************************************************
 516  * Boost.MPI-Specific Section                                           *
 517  ************************************************************************/
 518 namespace boost { namespace mpi {
 519  class packed_iarchive;
 520  class packed_oarchive;
 521 } } // end namespace boost::mpi
 522
 523 BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(
 524   ::boost::mpi::packed_iarchive,
 525   ::boost::mpi::packed_oarchive)
 526
 527 namespace boost { namespace mpi { namespace python {
 528
 529 template<typename T>
 530 void
 531 register_serialized(const T& value, PyTypeObject* type)
 532 {
 533   using boost::python::register_serialized;
 534   register_serialized<packed_iarchive, packed_oarchive>(value, type);
 535 }
 536
 537 } } } // end namespace boost::mpi::python
 538
 539 #endif // BOOST_MPI_PYTHON_SERIALIZE_HPP