ceph/src/boost/libs/spirit/include/boost/spirit/home/lex/lexer/lexertl/position_token.hpp

   1 //  Copyright (c) 2001-2011 Hartmut Kaiser
   2 //
   3 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
   4 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
   5
   6 #if !defined(BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM)
   7 #define BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM
   8
   9 #if defined(_MSC_VER)
  10 #pragma once
  11 #endif
  12
  13 #include <boost/config.hpp>
  14 #include <boost/detail/workaround.hpp>
  15 #include <boost/spirit/home/qi/detail/assign_to.hpp>
  16 #include <boost/spirit/home/support/attributes.hpp>
  17 #include <boost/spirit/home/support/argument.hpp>
  18 #include <boost/spirit/home/support/detail/lexer/generator.hpp>
  19 #include <boost/spirit/home/support/detail/lexer/rules.hpp>
  20 #include <boost/spirit/home/support/detail/lexer/consts.hpp>
  21 #include <boost/spirit/home/support/utree/utree_traits_fwd.hpp>
  22 #include <boost/spirit/home/lex/lexer/terminals.hpp>
  23 #include <boost/fusion/include/vector.hpp>
  24 #include <boost/fusion/include/at.hpp>
  25 #include <boost/fusion/include/value_at.hpp>
  26 #include <boost/detail/iterator.hpp>
  27 #include <boost/variant.hpp>
  28 #include <boost/mpl/vector.hpp>
  29 #include <boost/mpl/bool.hpp>
  30 #include <boost/mpl/is_sequence.hpp>
  31 #include <boost/mpl/begin.hpp>
  32 #include <boost/mpl/insert.hpp>
  33 #include <boost/mpl/vector.hpp>
  34 #include <boost/mpl/if.hpp>
  35 #include <boost/mpl/or.hpp>
  36 #include <boost/type_traits/is_same.hpp>
  37 #include <boost/range/iterator_range.hpp>
  38 #include <boost/static_assert.hpp>
  39
  40 #if defined(BOOST_SPIRIT_DEBUG)
  41 #include <iosfwd>
  42 #endif
  43
  44 namespace boost { namespace spirit { namespace lex { namespace lexertl
  45 {
  46     ///////////////////////////////////////////////////////////////////////////
  47     //
  48     //  The position_token is the type of the objects returned by the
  49     //  iterator if it has been specified while instantiating the lexer object.
  50     //
  51     //    template parameters:
  52     //        Iterator        The type of the iterator used to access the
  53     //                        underlying character stream.
  54     //        AttributeTypes  A mpl sequence containing the types of all
  55     //                        required different token values to be supported
  56     //                        by this token type.
  57     //        HasState        A mpl::bool_ indicating, whether this token type
  58     //                        should support lexer states.
  59     //        Idtype          The type to use for the token id (defaults to
  60     //                        std::size_t).
  61     //
  62     //  It is possible to use other token types with the spirit::lex
  63     //  framework as well. If you plan to use a different type as your token
  64     //  type, you'll need to expose the following things from your token type
  65     //  to make it compatible with spirit::lex:
  66     //
  67     //    typedefs
  68     //        iterator_type   The type of the iterator used to access the
  69     //                        underlying character stream.
  70     //
  71     //        id_type         The type of the token id used.
  72     //
  73     //    methods
  74     //        default constructor
  75     //                        This should initialize the token as an end of
  76     //                        input token.
  77     //        constructors    The prototype of the other required
  78     //                        constructors should be:
  79     //
  80     //              token(int)
  81     //                        This constructor should initialize the token as
  82     //                        an invalid token (not carrying any specific
  83     //                        values)
  84     //
  85     //              where:  the int is used as a tag only and its value is
  86     //                      ignored
  87     //
  88     //                        and:
  89     //
  90     //              token(Idtype id, std::size_t state,
  91     //                    iterator_type first, iterator_type last);
  92     //
  93     //              where:  id:           token id
  94     //                      state:        lexer state this token was matched in
  95     //                      first, last:  pair of iterators marking the matched
  96     //                                    range in the underlying input stream
  97     //
  98     //        accessors
  99     //              id()      return the token id of the matched input sequence
 100     //              id(newid) set the token id of the token instance
 101     //
 102     //              state()   return the lexer state this token was matched in
 103     //
 104     //              value()   return the token value
 105     //
 106     //  Additionally, you will have to implement a couple of helper functions
 107     //  in the same namespace as the token type: a comparison operator==() to
 108     //  compare your token instances, a token_is_valid() function and different
 109     //  specializations of the Spirit customization point
 110     //  assign_to_attribute_from_value as shown below.
 111     //
 112     ///////////////////////////////////////////////////////////////////////////
 113     template <typename Iterator = char const*
 114       , typename AttributeTypes = mpl::vector0<>
 115       , typename HasState = mpl::true_
 116       , typename Idtype = std::size_t>
 117     struct position_token;
 118
 119     ///////////////////////////////////////////////////////////////////////////
 120     //  This specialization of the token type doesn't contain any item data and
 121     //  doesn't support working with lexer states. Although, like all other
 122     //  variants of position_token, it carries a pair of iterators marking the
 123     //  begin and the end of the matched character sequence.
 124     ///////////////////////////////////////////////////////////////////////////
 125     template <typename Iterator, typename Idtype>
 126     struct position_token<Iterator, lex::omit, mpl::false_, Idtype>
 127     {
 128         typedef Iterator iterator_type;
 129         typedef iterator_range<iterator_type> iterpair_type;
 130         typedef mpl::false_ has_state;
 131         typedef Idtype id_type;
 132         typedef unused_type token_value_type;
 133
 134         //  default constructed tokens correspond to EOI tokens
 135         position_token()
 136           : id_(id_type(boost::lexer::npos)) {}
 137
 138         //  construct an invalid token
 139         explicit position_token(int)
 140           : id_(id_type(0)) {}
 141
 142         position_token(id_type id, std::size_t)
 143           : id_(id) {}
 144
 145         position_token(id_type id, std::size_t, token_value_type)
 146           : id_(id) {}
 147
 148         position_token(id_type id, std::size_t, Iterator const& first
 149               , Iterator const& last)
 150           : id_(id), matched_(first, last) {}
 151
 152         //  this default conversion operator is needed to allow the direct
 153         //  usage of tokens in conjunction with the primitive parsers defined
 154         //  in Qi
 155         operator id_type() const { return id_; }
 156
 157         //  Retrieve or set the token id of this token instance.
 158         id_type id() const { return id_; }
 159         void id(id_type newid) { id_ = newid; }
 160
 161         std::size_t state() const { return 0; }   // always '0' (INITIAL state)
 162
 163         bool is_valid() const
 164         {
 165             return 0 != id_ && id_type(boost::lexer::npos) != id_;
 166         }
 167
 168         // access the stored iterator range of the matched input sequence
 169         iterator_type begin() const { return matched_.begin(); }
 170         iterator_type end() const { return matched_.end(); }
 171
 172         iterpair_type& matched() { return matched_; }
 173         iterpair_type const& matched() const { return matched_; }
 174
 175         token_value_type& value() { static token_value_type u; return u; }
 176         token_value_type const& value() const { return unused; }
 177
 178 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 179         // workaround for MSVC10 which has problems copying a default
 180         // constructed iterator_range
 181         position_token& operator= (position_token const& rhs)
 182         {
 183             if (this != &rhs)
 184             {
 185                 id_ = rhs.id_;
 186                 if (is_valid())
 187                     matched_ = rhs.matched_;
 188             }
 189             return *this;
 190         }
 191 #endif
 192
 193     protected:
 194         id_type id_;              // token id, 0 if nothing has been matched
 195         iterpair_type matched_;   // matched input sequence
 196     };
 197
 198 #if defined(BOOST_SPIRIT_DEBUG)
 199     template <typename Char, typename Traits, typename Iterator
 200       , typename AttributeTypes, typename HasState, typename Idtype>
 201     inline std::basic_ostream<Char, Traits>&
 202     operator<< (std::basic_ostream<Char, Traits>& os
 203       , position_token<Iterator, AttributeTypes, HasState, Idtype> const& t)
 204     {
 205         if (t.is_valid()) {
 206             Iterator end = t.end();
 207             for (Iterator it = t.begin(); it != end; ++it)
 208                 os << *it;
 209         }
 210         else {
 211             os << "<invalid token>";
 212         }
 213         return os;
 214     }
 215 #endif
 216
 217     ///////////////////////////////////////////////////////////////////////////
 218     //  This specialization of the token type doesn't contain any item data but
 219     //  supports working with lexer states.
 220     ///////////////////////////////////////////////////////////////////////////
 221     template <typename Iterator, typename Idtype>
 222     struct position_token<Iterator, lex::omit, mpl::true_, Idtype>
 223       : position_token<Iterator, lex::omit, mpl::false_, Idtype>
 224     {
 225     private:
 226         typedef position_token<Iterator, lex::omit, mpl::false_, Idtype>
 227             base_type;
 228
 229     public:
 230         typedef typename base_type::id_type id_type;
 231         typedef Iterator iterator_type;
 232         typedef mpl::true_ has_state;
 233         typedef unused_type token_value_type;
 234
 235         //  default constructed tokens correspond to EOI tokens
 236         position_token() : state_(boost::lexer::npos) {}
 237
 238         //  construct an invalid token
 239         explicit position_token(int)
 240           : base_type(0), state_(boost::lexer::npos) {}
 241
 242         position_token(id_type id, std::size_t state)
 243           : base_type(id, boost::lexer::npos), state_(state) {}
 244
 245         position_token(id_type id, std::size_t state, token_value_type)
 246           : base_type(id, boost::lexer::npos, unused)
 247           , state_(state) {}
 248
 249         position_token(id_type id, std::size_t state
 250               , Iterator const& first, Iterator const& last)
 251           : base_type(id, boost::lexer::npos, first, last)
 252           , state_(state) {}
 253
 254         std::size_t state() const { return state_; }
 255
 256 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 257         // workaround for MSVC10 which has problems copying a default
 258         // constructed iterator_range
 259         position_token& operator= (position_token const& rhs)
 260         {
 261             if (this != &rhs)
 262             {
 263                 this->base_type::operator=(static_cast<base_type const&>(rhs));
 264                 state_ = rhs.state_;
 265             }
 266             return *this;
 267         }
 268 #endif
 269
 270     protected:
 271         std::size_t state_;      // lexer state this token was matched in
 272     };
 273
 274     ///////////////////////////////////////////////////////////////////////////
 275     // These specializations for an empty attribute list cause all token
 276     // instances to expose as it attribute the iterator_range pointing to the
 277     // matched input sequence.
 278     ///////////////////////////////////////////////////////////////////////////
 279     template <typename Iterator, typename HasState, typename Idtype>
 280     struct position_token<Iterator, mpl::vector<>, HasState, Idtype>
 281       : position_token<Iterator, lex::omit, HasState, Idtype>
 282     {
 283     private:
 284         typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
 285
 286     public:
 287         typedef typename base_type::id_type id_type;
 288         typedef typename base_type::iterator_type iterator_type;
 289         typedef typename base_type::iterpair_type iterpair_type;
 290         typedef HasState has_state;
 291         typedef iterpair_type token_value_type;
 292
 293         //  default constructed tokens correspond to EOI tokens
 294         position_token() {}
 295
 296         //  construct an invalid token
 297         explicit position_token(int)
 298           : base_type(0) {}
 299
 300         position_token(id_type id, std::size_t state)
 301           : base_type(id, state) {}
 302
 303         position_token(id_type id, std::size_t state, token_value_type)
 304           : base_type(id, state, unused) {}
 305
 306         position_token(id_type id, std::size_t state
 307               , Iterator const& first, Iterator const& last)
 308           : base_type(id, state, first, last) {}
 309
 310         token_value_type& value() { return this->base_type::matched(); }
 311         token_value_type const& value() const { return this->base_type::matched(); }
 312     };
 313
 314     template <typename Iterator, typename HasState, typename Idtype>
 315     struct position_token<Iterator, mpl::vector0<>, HasState, Idtype>
 316       : position_token<Iterator, lex::omit, HasState, Idtype>
 317     {
 318     private:
 319         typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
 320
 321     public:
 322         typedef typename base_type::id_type id_type;
 323         typedef typename base_type::iterator_type iterator_type;
 324         typedef typename base_type::iterpair_type iterpair_type;
 325         typedef HasState has_state;
 326         typedef iterpair_type token_value_type;
 327
 328         //  default constructed tokens correspond to EOI tokens
 329         position_token() {}
 330
 331         //  construct an invalid token
 332         explicit position_token(int)
 333           : base_type(0) {}
 334
 335         position_token(id_type id, std::size_t state)
 336           : base_type(id, state) {}
 337
 338         position_token(id_type id, std::size_t state, token_value_type)
 339           : base_type(id, state, unused) {}
 340
 341         position_token(id_type id, std::size_t state
 342               , Iterator const& first, Iterator const& last)
 343           : base_type(id, state, first, last) {}
 344
 345         token_value_type& value() { return this->base_type::matched(); }
 346         token_value_type const& value() const { return this->base_type::matched(); }
 347     };
 348
 349     ///////////////////////////////////////////////////////////////////////////
 350     // These specializations for an attribute list of length one cause all token
 351     // instances to expose the specified type as its attribute.
 352     ///////////////////////////////////////////////////////////////////////////
 353     template <typename Iterator, typename Attribute, typename HasState
 354       , typename Idtype>
 355     struct position_token<Iterator, mpl::vector<Attribute>, HasState, Idtype>
 356       : position_token<Iterator, lex::omit, HasState, Idtype>
 357     {
 358     private:
 359         typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
 360
 361     public:
 362         typedef typename base_type::id_type id_type;
 363         typedef typename base_type::iterator_type iterator_type;
 364         typedef typename base_type::iterpair_type iterpair_type;
 365         typedef HasState has_state;
 366         typedef boost::optional<Attribute> token_value_type;
 367
 368         //  default constructed tokens correspond to EOI tokens
 369         position_token() {}
 370
 371         //  construct an invalid token
 372         explicit position_token(int)
 373           : base_type(0) {}
 374
 375         position_token(id_type id, std::size_t state)
 376           : base_type(id, state) {}
 377
 378         position_token(id_type id, std::size_t state, token_value_type const& v)
 379           : base_type(id, state, unused), value_(v) {}
 380
 381         position_token(id_type id, std::size_t state
 382               , Iterator const& first, Iterator const& last)
 383           : base_type(id, state, first, last) {}
 384
 385         token_value_type& value() { return value_; }
 386         token_value_type const& value() const { return value_; }
 387
 388         bool has_value() const { return !!value_; }
 389
 390 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 391         // workaround for MSVC10 which has problems copying a default
 392         // constructed iterator_range
 393         position_token& operator= (position_token const& rhs)
 394         {
 395             if (this != &rhs)
 396             {
 397                 this->base_type::operator=(static_cast<base_type const&>(rhs));
 398                 if (this->is_valid())
 399                     value_ = rhs.value_;
 400             }
 401             return *this;
 402         }
 403 #endif
 404
 405     protected:
 406         token_value_type value_; // token value
 407     };
 408
 409     template <typename Iterator, typename Attribute, typename HasState
 410       , typename Idtype>
 411     struct position_token<Iterator, mpl::vector1<Attribute>, HasState, Idtype>
 412       : position_token<Iterator, lex::omit, HasState, Idtype>
 413     {
 414     private:
 415         typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
 416
 417     public:
 418         typedef typename base_type::id_type id_type;
 419         typedef typename base_type::iterator_type iterator_type;
 420         typedef typename base_type::iterpair_type iterpair_type;
 421         typedef HasState has_state;
 422         typedef boost::optional<Attribute> token_value_type;
 423
 424         //  default constructed tokens correspond to EOI tokens
 425         position_token() {}
 426
 427         //  construct an invalid token
 428         explicit position_token(int)
 429           : base_type(0) {}
 430
 431         position_token(id_type id, std::size_t state)
 432           : base_type(id, state) {}
 433
 434         position_token(id_type id, std::size_t state, token_value_type const& v)
 435           : base_type(id, state, unused), value_(v) {}
 436
 437         position_token(id_type id, std::size_t state
 438               , Iterator const& first, Iterator const& last)
 439           : base_type(id, state, first, last) {}
 440
 441         token_value_type& value() { return value_; }
 442         token_value_type const& value() const { return value_; }
 443
 444         bool has_value() const { return value_; }
 445
 446 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 447         // workaround for MSVC10 which has problems copying a default
 448         // constructed iterator_range
 449         position_token& operator= (position_token const& rhs)
 450         {
 451             if (this != &rhs)
 452             {
 453                 this->base_type::operator=(static_cast<base_type const&>(rhs));
 454                 if (this->is_valid())
 455                     value_ = rhs.value_;
 456             }
 457             return *this;
 458         }
 459 #endif
 460
 461     protected:
 462         token_value_type value_; // token value
 463     };
 464
 465     ///////////////////////////////////////////////////////////////////////////
 466     //  The generic version of the position_token type derives from the
 467     //  specialization above and adds a single data member holding the item
 468     //  data carried by the token instance.
 469     ///////////////////////////////////////////////////////////////////////////
 470     namespace detail
 471     {
 472         ///////////////////////////////////////////////////////////////////////
 473         //  Meta-function to calculate the type of the variant data item to be
 474         //  stored with each token instance.
 475         //
 476         //  Note: The iterator pair needs to be the first type in the list of
 477         //        types supported by the generated variant type (this is being
 478         //        used to identify whether the stored data item in a particular
 479         //        token instance needs to be converted from the pair of
 480         //        iterators (see the first of the assign_to_attribute_from_value
 481         //        specializations below).
 482         ///////////////////////////////////////////////////////////////////////
 483         template <typename IteratorPair, typename AttributeTypes>
 484         struct position_token_value_typesequence
 485         {
 486             typedef typename mpl::insert<
 487                 AttributeTypes
 488               , typename mpl::begin<AttributeTypes>::type
 489               , IteratorPair
 490             >::type sequence_type;
 491             typedef typename make_variant_over<sequence_type>::type type;
 492         };
 493
 494         ///////////////////////////////////////////////////////////////////////
 495         //  The type of the data item stored with a token instance is defined
 496         //  by the template parameter 'AttributeTypes' and may be:
 497         //
 498         //     lex::omit:         no data item is stored with the token
 499         //                        instance (this is handled by the
 500         //                        specializations of the token class
 501         //                        below)
 502         //     mpl::vector0<>:    each token instance stores a pair of
 503         //                        iterators pointing to the matched input
 504         //                        sequence
 505         //     mpl::vector<...>:  each token instance stores a variant being
 506         //                        able to store the pair of iterators pointing
 507         //                        to the matched input sequence, or any of the
 508         //                        types a specified in the mpl::vector<>
 509         //
 510         //  All this is done to ensure the token type is as small (in terms
 511         //  of its byte-size) as possible.
 512         ///////////////////////////////////////////////////////////////////////
 513         template <typename IteratorPair, typename AttributeTypes>
 514         struct position_token_value
 515           : mpl::eval_if<
 516                 mpl::or_<
 517                     is_same<AttributeTypes, mpl::vector0<> >
 518                   , is_same<AttributeTypes, mpl::vector<> > >
 519               , mpl::identity<IteratorPair>
 520               , position_token_value_typesequence<IteratorPair, AttributeTypes> >
 521         {};
 522     }
 523
 524     template <typename Iterator, typename AttributeTypes, typename HasState
 525       , typename Idtype>
 526     struct position_token
 527       : position_token<Iterator, lex::omit, HasState, Idtype>
 528     {
 529     private: // precondition assertions
 530         BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value ||
 531                             is_same<AttributeTypes, lex::omit>::value));
 532         typedef position_token<Iterator, lex::omit, HasState, Idtype>
 533             base_type;
 534
 535     protected:
 536         //  If no additional token value types are given, the token will
 537         //  hold no token value at all as the base class already has the
 538         //  iterator pair of the matched range in the underlying input sequence.
 539         //  Otherwise the token value is stored as a variant and will
 540         //  initially hold an unused_type but is able to hold any of
 541         //  the given data types as well. The conversion from the iterator pair
 542         //  to the required data type is done when it is accessed for the first
 543         //  time.
 544         typedef iterator_range<Iterator> iterpair_type;
 545
 546     public:
 547         typedef typename base_type::id_type id_type;
 548         typedef typename detail::position_token_value<
 549             iterpair_type, AttributeTypes>::type token_value_type;
 550
 551         typedef Iterator iterator_type;
 552
 553         //  default constructed tokens correspond to EOI tokens
 554         position_token() {}
 555
 556         //  construct an invalid token
 557         explicit position_token(int)
 558           : base_type(0) {}
 559
 560         position_token(id_type id, std::size_t state, token_value_type const& value)
 561           : base_type(id, state, value), value_(value) {}
 562
 563         position_token(id_type id, std::size_t state, Iterator const& first
 564               , Iterator const& last)
 565           : base_type(id, state, first, last)
 566           , value_(iterpair_type(first, last))
 567         {}
 568
 569         token_value_type& value() { return value_; }
 570         token_value_type const& value() const { return value_; }
 571
 572 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 573         // workaround for MSVC10 which has problems copying a default
 574         // constructed iterator_range
 575         position_token& operator= (position_token const& rhs)
 576         {
 577             if (this != &rhs)
 578             {
 579                 this->base_type::operator=(static_cast<base_type const&>(rhs));
 580                 if (this->is_valid())
 581                     value_ = rhs.value_;
 582             }
 583             return *this;
 584         }
 585 #endif
 586
 587     protected:
 588         token_value_type value_; // token value, by default a pair of iterators
 589     };
 590
 591     ///////////////////////////////////////////////////////////////////////////
 592     //  tokens are considered equal, if their id's match (these are unique)
 593     template <typename Iterator, typename AttributeTypes, typename HasState
 594       , typename Idtype>
 595     inline bool
 596     operator== (position_token<Iterator, AttributeTypes, HasState, Idtype> const& lhs,
 597                 position_token<Iterator, AttributeTypes, HasState, Idtype> const& rhs)
 598     {
 599         return lhs.id() == rhs.id();
 600     }
 601
 602     ///////////////////////////////////////////////////////////////////////////
 603     //  This overload is needed by the multi_pass/functor_input_policy to
 604     //  validate a token instance. It has to be defined in the same namespace
 605     //  as the token class itself to allow ADL to find it.
 606     ///////////////////////////////////////////////////////////////////////////
 607     template <typename Iterator, typename AttributeTypes, typename HasState
 608       , typename Idtype>
 609     inline bool
 610     token_is_valid(position_token<Iterator, AttributeTypes, HasState, Idtype> const& t)
 611     {
 612         return t.is_valid();
 613     }
 614 }}}}
 615
 616 namespace boost { namespace spirit { namespace traits
 617 {
 618     ///////////////////////////////////////////////////////////////////////////
 619     //  We have to provide specializations for the customization point
 620     //  assign_to_attribute_from_value allowing to extract the needed value
 621     //  from the token.
 622     ///////////////////////////////////////////////////////////////////////////
 623
 624     //  This is called from the parse function of token_def if the token_def
 625     //  has been defined to carry a special attribute type
 626     template <typename Attribute, typename Iterator, typename AttributeTypes
 627       , typename HasState, typename Idtype>
 628     struct assign_to_attribute_from_value<Attribute
 629       , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 630     {
 631         static void
 632         call(lex::lexertl::position_token<
 633                 Iterator, AttributeTypes, HasState, Idtype> const& t
 634           , Attribute& attr)
 635         {
 636         //  The goal of this function is to avoid the conversion of the pair of
 637         //  iterators (to the matched character sequence) into the token value
 638         //  of the required type being done more than once. For this purpose it
 639         //  checks whether the stored value type is still the default one (pair
 640         //  of iterators) and if yes, replaces the pair of iterators with the
 641         //  converted value to be returned from subsequent calls.
 642
 643             if (0 == t.value().which()) {
 644             //  first access to the token value
 645                 typedef iterator_range<Iterator> iterpair_type;
 646                 iterpair_type const& ip = t.matched();
 647
 648             // Interestingly enough we use the assign_to() framework defined in
 649             // Spirit.Qi allowing to convert the pair of iterators to almost any
 650             // required type (assign_to(), if available, uses the standard Spirit
 651             // parsers to do the conversion).
 652                 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
 653
 654             //  If you get an error during the compilation of the following
 655             //  assignment expression, you probably forgot to list one or more
 656             //  types used as token value types (in your token_def<...>
 657             //  definitions) in your definition of the token class. I.e. any token
 658             //  value type used for a token_def<...> definition has to be listed
 659             //  during the declaration of the token type to use. For instance let's
 660             //  assume we have two token_def's:
 661             //
 662             //      token_def<int> number; number = "...";
 663             //      token_def<std::string> identifier; identifier = "...";
 664             //
 665             //  Then you'll have to use the following token type definition
 666             //  (assuming you are using the token class):
 667             //
 668             //      typedef mpl::vector<int, std::string> token_values;
 669             //      typedef token<base_iter_type, token_values> token_type;
 670             //
 671             //  where: base_iter_type is the iterator type used to expose the
 672             //         underlying input stream.
 673             //
 674             //  This token_type has to be used as the second template parameter
 675             //  to the lexer class:
 676             //
 677             //      typedef lexer<base_iter_type, token_type> lexer_type;
 678             //
 679             //  again, assuming you're using the lexer<> template for your
 680             //  tokenization.
 681
 682                 typedef lex::lexertl::position_token<
 683                     Iterator, AttributeTypes, HasState, Idtype> token_type;
 684                 spirit::traits::assign_to(
 685                     attr, const_cast<token_type&>(t).value());   // re-assign value
 686             }
 687             else {
 688             // reuse the already assigned value
 689                 spirit::traits::assign_to(get<Attribute>(t.value()), attr);
 690             }
 691         }
 692     };
 693
 694     template <typename Attribute, typename Iterator, typename AttributeTypes
 695       , typename HasState, typename Idtype>
 696     struct assign_to_container_from_value<Attribute
 697           , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 698       : assign_to_attribute_from_value<Attribute
 699           , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 700     {};
 701
 702     ///////////////////////////////////////////////////////////////////////////
 703     //  These are called from the parse function of token_def if the token type
 704     //  has no special attribute type assigned
 705     template <typename Attribute, typename Iterator, typename HasState
 706       , typename Idtype>
 707     struct assign_to_attribute_from_value<Attribute
 708       , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> >
 709     {
 710         static void
 711         call(lex::lexertl::position_token<
 712                 Iterator, mpl::vector0<>, HasState, Idtype> const& t
 713           , Attribute& attr)
 714         {
 715             //  The default type returned by the token_def parser component (if
 716             //  it has no token value type assigned) is the pair of iterators
 717             //  to the matched character sequence.
 718             spirit::traits::assign_to(t.begin(), t.end(), attr);
 719         }
 720     };
 721
 722 //     template <typename Attribute, typename Iterator, typename HasState
 723 //       , typename Idtype>
 724 //     struct assign_to_container_from_value<Attribute
 725 //           , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> >
 726 //       : assign_to_attribute_from_value<Attribute
 727 //           , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> >
 728 //     {};
 729
 730     // same as above but using mpl::vector<> instead of mpl::vector0<>
 731     template <typename Attribute, typename Iterator, typename HasState
 732       , typename Idtype>
 733     struct assign_to_attribute_from_value<Attribute
 734       , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> >
 735     {
 736         static void
 737         call(lex::lexertl::position_token<
 738                 Iterator, mpl::vector<>, HasState, Idtype> const& t
 739           , Attribute& attr)
 740         {
 741             //  The default type returned by the token_def parser component (if
 742             //  it has no token value type assigned) is the pair of iterators
 743             //  to the matched character sequence.
 744             spirit::traits::assign_to(t.begin(), t.end(), attr);
 745         }
 746     };
 747
 748 //     template <typename Attribute, typename Iterator, typename HasState
 749 //       , typename Idtype>
 750 //     struct assign_to_container_from_value<Attribute
 751 //           , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> >
 752 //       : assign_to_attribute_from_value<Attribute
 753 //           , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> >
 754 //     {};
 755
 756     ///////////////////////////////////////////////////////////////////////////
 757     //  These are called from the parse function of token_def if the token type
 758     //  has no special attribute type assigned
 759     template <typename Attribute, typename Iterator, typename Attr
 760       , typename HasState, typename Idtype>
 761     struct assign_to_attribute_from_value<Attribute
 762       , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> >
 763     {
 764         static void
 765         call(lex::lexertl::position_token<
 766                 Iterator, mpl::vector1<Attr>, HasState, Idtype> const& t
 767           , Attribute& attr)
 768         {
 769         //  The goal of this function is to avoid the conversion of the pair of
 770         //  iterators (to the matched character sequence) into the token value
 771         //  of the required type being done more than once.
 772
 773             if (!t.has_value()) {
 774             //  first access to the token value
 775                 typedef iterator_range<Iterator> iterpair_type;
 776                 iterpair_type const& ip = t.matched();
 777
 778             // Interestingly enough we use the assign_to() framework defined in
 779             // Spirit.Qi allowing to convert the pair of iterators to almost any
 780             // required type (assign_to(), if available, uses the standard Spirit
 781             // parsers to do the conversion).
 782                 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
 783
 784             // Re-assign the attribute to the stored value
 785                 typedef lex::lexertl::position_token<
 786                     Iterator, mpl::vector1<Attr>, HasState, Idtype> token_type;
 787                 spirit::traits::assign_to(
 788                     attr, const_cast<token_type&>(t).value());
 789             }
 790             else {
 791             // reuse the already assigned value
 792                 spirit::traits::assign_to(t.value(), attr);
 793             }
 794         }
 795     };
 796
 797 //     template <typename Attribute, typename Iterator, typename Attr
 798 //       , typename HasState, typename Idtype>
 799 //     struct assign_to_container_from_value<Attribute
 800 //           , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> >
 801 //       : assign_to_attribute_from_value<Attribute
 802 //           , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> >
 803 //     {};
 804
 805     // same as above but using mpl::vector<Attr> instead of mpl::vector1<Attr>
 806     template <typename Attribute, typename Iterator, typename Attr
 807       , typename HasState, typename Idtype>
 808     struct assign_to_attribute_from_value<Attribute
 809       , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> >
 810     {
 811         static void
 812         call(lex::lexertl::position_token<
 813                 Iterator, mpl::vector<Attr>, HasState, Idtype> const& t
 814           , Attribute& attr)
 815         {
 816         //  The goal of this function is to avoid the conversion of the pair of
 817         //  iterators (to the matched character sequence) into the token value
 818         //  of the required type being done more than once.
 819
 820             if (!t.has_value()) {
 821             //  first access to the token value
 822                 typedef iterator_range<Iterator> iterpair_type;
 823                 iterpair_type const& ip = t.matched();
 824
 825             // Interestingly enough we use the assign_to() framework defined in
 826             // Spirit.Qi allowing to convert the pair of iterators to almost any
 827             // required type (assign_to(), if available, uses the standard Spirit
 828             // parsers to do the conversion).
 829                 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
 830
 831             // Re-assign the attribute to the stored value
 832                 typedef lex::lexertl::position_token<
 833                     Iterator, mpl::vector<Attr>, HasState, Idtype> token_type;
 834                 spirit::traits::assign_to(
 835                     attr, const_cast<token_type&>(t).value());
 836             }
 837             else {
 838             // reuse the already assigned value
 839                 spirit::traits::assign_to(t.value(), attr);
 840             }
 841         }
 842     };
 843
 844 //     template <typename Attribute, typename Iterator, typename Attr
 845 //       , typename HasState, typename Idtype>
 846 //     struct assign_to_container_from_value<Attribute
 847 //           , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> >
 848 //       : assign_to_attribute_from_value<Attribute
 849 //           , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> >
 850 //     {};
 851
 852     //  This is called from the parse function of token_def if the token type
 853     //  has been explicitly omitted (i.e. no attribute value is used), which
 854     //  essentially means that every attribute gets initialized using default
 855     //  constructed values.
 856     template <typename Attribute, typename Iterator, typename HasState
 857       , typename Idtype>
 858     struct assign_to_attribute_from_value<Attribute
 859       , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> >
 860     {
 861         static void
 862         call(lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> const& t
 863           , Attribute& attr)
 864         {
 865             // do nothing
 866         }
 867     };
 868
 869     template <typename Attribute, typename Iterator, typename HasState
 870       , typename Idtype>
 871     struct assign_to_container_from_value<Attribute
 872           , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> >
 873       : assign_to_attribute_from_value<Attribute
 874           , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> >
 875     {};
 876
 877     //  This is called from the parse function of lexer_def_
 878     template <typename Iterator, typename AttributeTypes, typename HasState
 879       , typename Idtype_, typename Idtype>
 880     struct assign_to_attribute_from_value<
 881         fusion::vector2<Idtype_, iterator_range<Iterator> >
 882       , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 883     {
 884         static void
 885         call(lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> const& t
 886           , fusion::vector2<Idtype_, iterator_range<Iterator> >& attr)
 887         {
 888             //  The type returned by the lexer_def_ parser components is a
 889             //  fusion::vector containing the token id of the matched token
 890             //  and the pair of iterators to the matched character sequence.
 891             typedef iterator_range<Iterator> iterpair_type;
 892             typedef fusion::vector2<Idtype_, iterator_range<Iterator> >
 893                 attribute_type;
 894
 895             iterpair_type const& ip = t.matched();
 896             attr = attribute_type(t.id(), ip);
 897         }
 898     };
 899
 900     template <typename Iterator, typename AttributeTypes, typename HasState
 901       , typename Idtype_, typename Idtype>
 902     struct assign_to_container_from_value<
 903             fusion::vector2<Idtype_, iterator_range<Iterator> >
 904           , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 905       : assign_to_attribute_from_value<
 906             fusion::vector2<Idtype_, iterator_range<Iterator> >
 907           , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 908     {};
 909
 910     ///////////////////////////////////////////////////////////////////////////
 911     // Overload debug output for a single token, this integrates lexer tokens
 912     // with Qi's simple_trace debug facilities
 913     template <typename Iterator, typename Attribute, typename HasState
 914       , typename Idtype>
 915     struct token_printer_debug<
 916         lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> >
 917     {
 918         typedef lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> token_type;
 919
 920         template <typename Out>
 921         static void print(Out& out, token_type const& val)
 922         {
 923             out << '[';
 924             spirit::traits::print_token(out, val.value());
 925             out << ']';
 926         }
 927     };
 928 }}}
 929
 930 #endif