ceph/src/boost/boost/spirit/home/lex/lexer/lexertl/position_token.hpp

   1 //  Copyright (c) 2001-2011 Hartmut Kaiser
   2 //
   3 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
   4 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
   5
   6 #if !defined(BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM)
   7 #define BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM
   8
   9 #if defined(_MSC_VER)
  10 #pragma once
  11 #endif
  12
  13 #include <boost/config.hpp>
  14 #include <boost/detail/workaround.hpp>
  15 #include <boost/spirit/home/qi/detail/assign_to.hpp>
  16 #include <boost/spirit/home/support/attributes.hpp>
  17 #include <boost/spirit/home/support/argument.hpp>
  18 #include <boost/spirit/home/support/detail/lexer/generator.hpp>
  19 #include <boost/spirit/home/support/detail/lexer/rules.hpp>
  20 #include <boost/spirit/home/support/detail/lexer/consts.hpp>
  21 #include <boost/spirit/home/support/utree/utree_traits_fwd.hpp>
  22 #include <boost/spirit/home/lex/lexer/terminals.hpp>
  23 #include <boost/fusion/include/vector.hpp>
  24 #include <boost/fusion/include/at.hpp>
  25 #include <boost/fusion/include/value_at.hpp>
  26 #include <boost/variant.hpp>
  27 #include <boost/mpl/vector.hpp>
  28 #include <boost/mpl/bool.hpp>
  29 #include <boost/mpl/is_sequence.hpp>
  30 #include <boost/mpl/begin.hpp>
  31 #include <boost/mpl/insert.hpp>
  32 #include <boost/mpl/vector.hpp>
  33 #include <boost/mpl/if.hpp>
  34 #include <boost/mpl/or.hpp>
  35 #include <boost/type_traits/is_same.hpp>
  36 #include <boost/range/iterator_range.hpp>
  37 #include <boost/static_assert.hpp>
  38
  39 #if defined(BOOST_SPIRIT_DEBUG)
  40 #include <iosfwd>
  41 #endif
  42
  43 namespace boost { namespace spirit { namespace lex { namespace lexertl
  44 {
  45     ///////////////////////////////////////////////////////////////////////////
  46     //
  47     //  The position_token is the type of the objects returned by the
  48     //  iterator if it has been specified while instantiating the lexer object.
  49     //
  50     //    template parameters:
  51     //        Iterator        The type of the iterator used to access the
  52     //                        underlying character stream.
  53     //        AttributeTypes  A mpl sequence containing the types of all
  54     //                        required different token values to be supported
  55     //                        by this token type.
  56     //        HasState        A mpl::bool_ indicating, whether this token type
  57     //                        should support lexer states.
  58     //        Idtype          The type to use for the token id (defaults to
  59     //                        std::size_t).
  60     //
  61     //  It is possible to use other token types with the spirit::lex
  62     //  framework as well. If you plan to use a different type as your token
  63     //  type, you'll need to expose the following things from your token type
  64     //  to make it compatible with spirit::lex:
  65     //
  66     //    typedefs
  67     //        iterator_type   The type of the iterator used to access the
  68     //                        underlying character stream.
  69     //
  70     //        id_type         The type of the token id used.
  71     //
  72     //    methods
  73     //        default constructor
  74     //                        This should initialize the token as an end of
  75     //                        input token.
  76     //        constructors    The prototype of the other required
  77     //                        constructors should be:
  78     //
  79     //              token(int)
  80     //                        This constructor should initialize the token as
  81     //                        an invalid token (not carrying any specific
  82     //                        values)
  83     //
  84     //              where:  the int is used as a tag only and its value is
  85     //                      ignored
  86     //
  87     //                        and:
  88     //
  89     //              token(Idtype id, std::size_t state,
  90     //                    iterator_type first, iterator_type last);
  91     //
  92     //              where:  id:           token id
  93     //                      state:        lexer state this token was matched in
  94     //                      first, last:  pair of iterators marking the matched
  95     //                                    range in the underlying input stream
  96     //
  97     //        accessors
  98     //              id()      return the token id of the matched input sequence
  99     //              id(newid) set the token id of the token instance
 100     //
 101     //              state()   return the lexer state this token was matched in
 102     //
 103     //              value()   return the token value
 104     //
 105     //  Additionally, you will have to implement a couple of helper functions
 106     //  in the same namespace as the token type: a comparison operator==() to
 107     //  compare your token instances, a token_is_valid() function and different
 108     //  specializations of the Spirit customization point
 109     //  assign_to_attribute_from_value as shown below.
 110     //
 111     ///////////////////////////////////////////////////////////////////////////
 112     template <typename Iterator = char const*
 113       , typename AttributeTypes = mpl::vector0<>
 114       , typename HasState = mpl::true_
 115       , typename Idtype = std::size_t>
 116     struct position_token;
 117
 118     ///////////////////////////////////////////////////////////////////////////
 119     //  This specialization of the token type doesn't contain any item data and
 120     //  doesn't support working with lexer states. Although, like all other
 121     //  variants of position_token, it carries a pair of iterators marking the
 122     //  begin and the end of the matched character sequence.
 123     ///////////////////////////////////////////////////////////////////////////
 124     template <typename Iterator, typename Idtype>
 125     struct position_token<Iterator, lex::omit, mpl::false_, Idtype>
 126     {
 127         typedef Iterator iterator_type;
 128         typedef iterator_range<iterator_type> iterpair_type;
 129         typedef mpl::false_ has_state;
 130         typedef Idtype id_type;
 131         typedef unused_type token_value_type;
 132
 133         //  default constructed tokens correspond to EOI tokens
 134         position_token()
 135           : id_(id_type(boost::lexer::npos)) {}
 136
 137         //  construct an invalid token
 138         explicit position_token(int)
 139           : id_(id_type(0)) {}
 140
 141         position_token(id_type id, std::size_t)
 142           : id_(id) {}
 143
 144         position_token(id_type id, std::size_t, token_value_type)
 145           : id_(id) {}
 146
 147         position_token(id_type id, std::size_t, Iterator const& first
 148               , Iterator const& last)
 149           : id_(id), matched_(first, last) {}
 150
 151         //  this default conversion operator is needed to allow the direct
 152         //  usage of tokens in conjunction with the primitive parsers defined
 153         //  in Qi
 154         operator id_type() const { return id_; }
 155
 156         //  Retrieve or set the token id of this token instance.
 157         id_type id() const { return id_; }
 158         void id(id_type newid) { id_ = newid; }
 159
 160         std::size_t state() const { return 0; }   // always '0' (INITIAL state)
 161
 162         bool is_valid() const
 163         {
 164             return 0 != id_ && id_type(boost::lexer::npos) != id_;
 165         }
 166
 167         // access the stored iterator range of the matched input sequence
 168         iterator_type begin() const { return matched_.begin(); }
 169         iterator_type end() const { return matched_.end(); }
 170
 171         iterpair_type& matched() { return matched_; }
 172         iterpair_type const& matched() const { return matched_; }
 173
 174         token_value_type& value() { static token_value_type u; return u; }
 175         token_value_type const& value() const { return unused; }
 176
 177 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 178         // workaround for MSVC10 which has problems copying a default
 179         // constructed iterator_range
 180         position_token& operator= (position_token const& rhs)
 181         {
 182             if (this != &rhs)
 183             {
 184                 id_ = rhs.id_;
 185                 if (is_valid())
 186                     matched_ = rhs.matched_;
 187             }
 188             return *this;
 189         }
 190 #endif
 191
 192     protected:
 193         id_type id_;              // token id, 0 if nothing has been matched
 194         iterpair_type matched_;   // matched input sequence
 195     };
 196
 197 #if defined(BOOST_SPIRIT_DEBUG)
 198     template <typename Char, typename Traits, typename Iterator
 199       , typename AttributeTypes, typename HasState, typename Idtype>
 200     inline std::basic_ostream<Char, Traits>&
 201     operator<< (std::basic_ostream<Char, Traits>& os
 202       , position_token<Iterator, AttributeTypes, HasState, Idtype> const& t)
 203     {
 204         if (t.is_valid()) {
 205             Iterator end = t.end();
 206             for (Iterator it = t.begin(); it != end; ++it)
 207                 os << *it;
 208         }
 209         else {
 210             os << "<invalid token>";
 211         }
 212         return os;
 213     }
 214 #endif
 215
 216     ///////////////////////////////////////////////////////////////////////////
 217     //  This specialization of the token type doesn't contain any item data but
 218     //  supports working with lexer states.
 219     ///////////////////////////////////////////////////////////////////////////
 220     template <typename Iterator, typename Idtype>
 221     struct position_token<Iterator, lex::omit, mpl::true_, Idtype>
 222       : position_token<Iterator, lex::omit, mpl::false_, Idtype>
 223     {
 224     private:
 225         typedef position_token<Iterator, lex::omit, mpl::false_, Idtype>
 226             base_type;
 227
 228     public:
 229         typedef typename base_type::id_type id_type;
 230         typedef Iterator iterator_type;
 231         typedef mpl::true_ has_state;
 232         typedef unused_type token_value_type;
 233
 234         //  default constructed tokens correspond to EOI tokens
 235         position_token() : state_(boost::lexer::npos) {}
 236
 237         //  construct an invalid token
 238         explicit position_token(int)
 239           : base_type(0), state_(boost::lexer::npos) {}
 240
 241         position_token(id_type id, std::size_t state)
 242           : base_type(id, boost::lexer::npos), state_(state) {}
 243
 244         position_token(id_type id, std::size_t state, token_value_type)
 245           : base_type(id, boost::lexer::npos, unused)
 246           , state_(state) {}
 247
 248         position_token(id_type id, std::size_t state
 249               , Iterator const& first, Iterator const& last)
 250           : base_type(id, boost::lexer::npos, first, last)
 251           , state_(state) {}
 252
 253         std::size_t state() const { return state_; }
 254
 255 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 256         // workaround for MSVC10 which has problems copying a default
 257         // constructed iterator_range
 258         position_token& operator= (position_token const& rhs)
 259         {
 260             if (this != &rhs)
 261             {
 262                 this->base_type::operator=(static_cast<base_type const&>(rhs));
 263                 state_ = rhs.state_;
 264             }
 265             return *this;
 266         }
 267 #endif
 268
 269     protected:
 270         std::size_t state_;      // lexer state this token was matched in
 271     };
 272
 273     ///////////////////////////////////////////////////////////////////////////
 274     // These specializations for an empty attribute list cause all token
 275     // instances to expose as it attribute the iterator_range pointing to the
 276     // matched input sequence.
 277     ///////////////////////////////////////////////////////////////////////////
 278     template <typename Iterator, typename HasState, typename Idtype>
 279     struct position_token<Iterator, mpl::vector<>, HasState, Idtype>
 280       : position_token<Iterator, lex::omit, HasState, Idtype>
 281     {
 282     private:
 283         typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
 284
 285     public:
 286         typedef typename base_type::id_type id_type;
 287         typedef typename base_type::iterator_type iterator_type;
 288         typedef typename base_type::iterpair_type iterpair_type;
 289         typedef HasState has_state;
 290         typedef iterpair_type token_value_type;
 291
 292         //  default constructed tokens correspond to EOI tokens
 293         position_token() {}
 294
 295         //  construct an invalid token
 296         explicit position_token(int)
 297           : base_type(0) {}
 298
 299         position_token(id_type id, std::size_t state)
 300           : base_type(id, state) {}
 301
 302         position_token(id_type id, std::size_t state, token_value_type)
 303           : base_type(id, state, unused) {}
 304
 305         position_token(id_type id, std::size_t state
 306               , Iterator const& first, Iterator const& last)
 307           : base_type(id, state, first, last) {}
 308
 309         token_value_type& value() { return this->base_type::matched(); }
 310         token_value_type const& value() const { return this->base_type::matched(); }
 311     };
 312
 313     template <typename Iterator, typename HasState, typename Idtype>
 314     struct position_token<Iterator, mpl::vector0<>, HasState, Idtype>
 315       : position_token<Iterator, lex::omit, HasState, Idtype>
 316     {
 317     private:
 318         typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
 319
 320     public:
 321         typedef typename base_type::id_type id_type;
 322         typedef typename base_type::iterator_type iterator_type;
 323         typedef typename base_type::iterpair_type iterpair_type;
 324         typedef HasState has_state;
 325         typedef iterpair_type token_value_type;
 326
 327         //  default constructed tokens correspond to EOI tokens
 328         position_token() {}
 329
 330         //  construct an invalid token
 331         explicit position_token(int)
 332           : base_type(0) {}
 333
 334         position_token(id_type id, std::size_t state)
 335           : base_type(id, state) {}
 336
 337         position_token(id_type id, std::size_t state, token_value_type)
 338           : base_type(id, state, unused) {}
 339
 340         position_token(id_type id, std::size_t state
 341               , Iterator const& first, Iterator const& last)
 342           : base_type(id, state, first, last) {}
 343
 344         token_value_type& value() { return this->base_type::matched(); }
 345         token_value_type const& value() const { return this->base_type::matched(); }
 346     };
 347
 348     ///////////////////////////////////////////////////////////////////////////
 349     // These specializations for an attribute list of length one cause all token
 350     // instances to expose the specified type as its attribute.
 351     ///////////////////////////////////////////////////////////////////////////
 352     template <typename Iterator, typename Attribute, typename HasState
 353       , typename Idtype>
 354     struct position_token<Iterator, mpl::vector<Attribute>, HasState, Idtype>
 355       : position_token<Iterator, lex::omit, HasState, Idtype>
 356     {
 357     private:
 358         typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
 359
 360     public:
 361         typedef typename base_type::id_type id_type;
 362         typedef typename base_type::iterator_type iterator_type;
 363         typedef typename base_type::iterpair_type iterpair_type;
 364         typedef HasState has_state;
 365         typedef boost::optional<Attribute> token_value_type;
 366
 367         //  default constructed tokens correspond to EOI tokens
 368         position_token() {}
 369
 370         //  construct an invalid token
 371         explicit position_token(int)
 372           : base_type(0) {}
 373
 374         position_token(id_type id, std::size_t state)
 375           : base_type(id, state) {}
 376
 377         position_token(id_type id, std::size_t state, token_value_type const& v)
 378           : base_type(id, state, unused), value_(v) {}
 379
 380         position_token(id_type id, std::size_t state
 381               , Iterator const& first, Iterator const& last)
 382           : base_type(id, state, first, last) {}
 383
 384         token_value_type& value() { return value_; }
 385         token_value_type const& value() const { return value_; }
 386
 387         bool has_value() const { return !!value_; }
 388
 389 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 390         // workaround for MSVC10 which has problems copying a default
 391         // constructed iterator_range
 392         position_token& operator= (position_token const& rhs)
 393         {
 394             if (this != &rhs)
 395             {
 396                 this->base_type::operator=(static_cast<base_type const&>(rhs));
 397                 if (this->is_valid())
 398                     value_ = rhs.value_;
 399             }
 400             return *this;
 401         }
 402 #endif
 403
 404     protected:
 405         token_value_type value_; // token value
 406     };
 407
 408     template <typename Iterator, typename Attribute, typename HasState
 409       , typename Idtype>
 410     struct position_token<Iterator, mpl::vector1<Attribute>, HasState, Idtype>
 411       : position_token<Iterator, lex::omit, HasState, Idtype>
 412     {
 413     private:
 414         typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
 415
 416     public:
 417         typedef typename base_type::id_type id_type;
 418         typedef typename base_type::iterator_type iterator_type;
 419         typedef typename base_type::iterpair_type iterpair_type;
 420         typedef HasState has_state;
 421         typedef boost::optional<Attribute> token_value_type;
 422
 423         //  default constructed tokens correspond to EOI tokens
 424         position_token() {}
 425
 426         //  construct an invalid token
 427         explicit position_token(int)
 428           : base_type(0) {}
 429
 430         position_token(id_type id, std::size_t state)
 431           : base_type(id, state) {}
 432
 433         position_token(id_type id, std::size_t state, token_value_type const& v)
 434           : base_type(id, state, unused), value_(v) {}
 435
 436         position_token(id_type id, std::size_t state
 437               , Iterator const& first, Iterator const& last)
 438           : base_type(id, state, first, last) {}
 439
 440         token_value_type& value() { return value_; }
 441         token_value_type const& value() const { return value_; }
 442
 443         bool has_value() const { return value_; }
 444
 445 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 446         // workaround for MSVC10 which has problems copying a default
 447         // constructed iterator_range
 448         position_token& operator= (position_token const& rhs)
 449         {
 450             if (this != &rhs)
 451             {
 452                 this->base_type::operator=(static_cast<base_type const&>(rhs));
 453                 if (this->is_valid())
 454                     value_ = rhs.value_;
 455             }
 456             return *this;
 457         }
 458 #endif
 459
 460     protected:
 461         token_value_type value_; // token value
 462     };
 463
 464     ///////////////////////////////////////////////////////////////////////////
 465     //  The generic version of the position_token type derives from the
 466     //  specialization above and adds a single data member holding the item
 467     //  data carried by the token instance.
 468     ///////////////////////////////////////////////////////////////////////////
 469     namespace detail
 470     {
 471         ///////////////////////////////////////////////////////////////////////
 472         //  Meta-function to calculate the type of the variant data item to be
 473         //  stored with each token instance.
 474         //
 475         //  Note: The iterator pair needs to be the first type in the list of
 476         //        types supported by the generated variant type (this is being
 477         //        used to identify whether the stored data item in a particular
 478         //        token instance needs to be converted from the pair of
 479         //        iterators (see the first of the assign_to_attribute_from_value
 480         //        specializations below).
 481         ///////////////////////////////////////////////////////////////////////
 482         template <typename IteratorPair, typename AttributeTypes>
 483         struct position_token_value_typesequence
 484         {
 485             typedef typename mpl::insert<
 486                 AttributeTypes
 487               , typename mpl::begin<AttributeTypes>::type
 488               , IteratorPair
 489             >::type sequence_type;
 490             typedef typename make_variant_over<sequence_type>::type type;
 491         };
 492
 493         ///////////////////////////////////////////////////////////////////////
 494         //  The type of the data item stored with a token instance is defined
 495         //  by the template parameter 'AttributeTypes' and may be:
 496         //
 497         //     lex::omit:         no data item is stored with the token
 498         //                        instance (this is handled by the
 499         //                        specializations of the token class
 500         //                        below)
 501         //     mpl::vector0<>:    each token instance stores a pair of
 502         //                        iterators pointing to the matched input
 503         //                        sequence
 504         //     mpl::vector<...>:  each token instance stores a variant being
 505         //                        able to store the pair of iterators pointing
 506         //                        to the matched input sequence, or any of the
 507         //                        types a specified in the mpl::vector<>
 508         //
 509         //  All this is done to ensure the token type is as small (in terms
 510         //  of its byte-size) as possible.
 511         ///////////////////////////////////////////////////////////////////////
 512         template <typename IteratorPair, typename AttributeTypes>
 513         struct position_token_value
 514           : mpl::eval_if<
 515                 mpl::or_<
 516                     is_same<AttributeTypes, mpl::vector0<> >
 517                   , is_same<AttributeTypes, mpl::vector<> > >
 518               , mpl::identity<IteratorPair>
 519               , position_token_value_typesequence<IteratorPair, AttributeTypes> >
 520         {};
 521     }
 522
 523     template <typename Iterator, typename AttributeTypes, typename HasState
 524       , typename Idtype>
 525     struct position_token
 526       : position_token<Iterator, lex::omit, HasState, Idtype>
 527     {
 528     private: // precondition assertions
 529         BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value ||
 530                             is_same<AttributeTypes, lex::omit>::value));
 531         typedef position_token<Iterator, lex::omit, HasState, Idtype>
 532             base_type;
 533
 534     protected:
 535         //  If no additional token value types are given, the token will
 536         //  hold no token value at all as the base class already has the
 537         //  iterator pair of the matched range in the underlying input sequence.
 538         //  Otherwise the token value is stored as a variant and will
 539         //  initially hold an unused_type but is able to hold any of
 540         //  the given data types as well. The conversion from the iterator pair
 541         //  to the required data type is done when it is accessed for the first
 542         //  time.
 543         typedef iterator_range<Iterator> iterpair_type;
 544
 545     public:
 546         typedef typename base_type::id_type id_type;
 547         typedef typename detail::position_token_value<
 548             iterpair_type, AttributeTypes>::type token_value_type;
 549
 550         typedef Iterator iterator_type;
 551
 552         //  default constructed tokens correspond to EOI tokens
 553         position_token() {}
 554
 555         //  construct an invalid token
 556         explicit position_token(int)
 557           : base_type(0) {}
 558
 559         position_token(id_type id, std::size_t state, token_value_type const& value)
 560           : base_type(id, state, value), value_(value) {}
 561
 562         position_token(id_type id, std::size_t state, Iterator const& first
 563               , Iterator const& last)
 564           : base_type(id, state, first, last)
 565           , value_(iterpair_type(first, last))
 566         {}
 567
 568         token_value_type& value() { return value_; }
 569         token_value_type const& value() const { return value_; }
 570
 571 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 572         // workaround for MSVC10 which has problems copying a default
 573         // constructed iterator_range
 574         position_token& operator= (position_token const& rhs)
 575         {
 576             if (this != &rhs)
 577             {
 578                 this->base_type::operator=(static_cast<base_type const&>(rhs));
 579                 if (this->is_valid())
 580                     value_ = rhs.value_;
 581             }
 582             return *this;
 583         }
 584 #endif
 585
 586     protected:
 587         token_value_type value_; // token value, by default a pair of iterators
 588     };
 589
 590     ///////////////////////////////////////////////////////////////////////////
 591     //  tokens are considered equal, if their id's match (these are unique)
 592     template <typename Iterator, typename AttributeTypes, typename HasState
 593       , typename Idtype>
 594     inline bool
 595     operator== (position_token<Iterator, AttributeTypes, HasState, Idtype> const& lhs,
 596                 position_token<Iterator, AttributeTypes, HasState, Idtype> const& rhs)
 597     {
 598         return lhs.id() == rhs.id();
 599     }
 600
 601     ///////////////////////////////////////////////////////////////////////////
 602     //  This overload is needed by the multi_pass/functor_input_policy to
 603     //  validate a token instance. It has to be defined in the same namespace
 604     //  as the token class itself to allow ADL to find it.
 605     ///////////////////////////////////////////////////////////////////////////
 606     template <typename Iterator, typename AttributeTypes, typename HasState
 607       , typename Idtype>
 608     inline bool
 609     token_is_valid(position_token<Iterator, AttributeTypes, HasState, Idtype> const& t)
 610     {
 611         return t.is_valid();
 612     }
 613 }}}}
 614
 615 namespace boost { namespace spirit { namespace traits
 616 {
 617     ///////////////////////////////////////////////////////////////////////////
 618     //  We have to provide specializations for the customization point
 619     //  assign_to_attribute_from_value allowing to extract the needed value
 620     //  from the token.
 621     ///////////////////////////////////////////////////////////////////////////
 622
 623     //  This is called from the parse function of token_def if the token_def
 624     //  has been defined to carry a special attribute type
 625     template <typename Attribute, typename Iterator, typename AttributeTypes
 626       , typename HasState, typename Idtype>
 627     struct assign_to_attribute_from_value<Attribute
 628       , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 629     {
 630         static void
 631         call(lex::lexertl::position_token<
 632                 Iterator, AttributeTypes, HasState, Idtype> const& t
 633           , Attribute& attr)
 634         {
 635         //  The goal of this function is to avoid the conversion of the pair of
 636         //  iterators (to the matched character sequence) into the token value
 637         //  of the required type being done more than once. For this purpose it
 638         //  checks whether the stored value type is still the default one (pair
 639         //  of iterators) and if yes, replaces the pair of iterators with the
 640         //  converted value to be returned from subsequent calls.
 641
 642             if (0 == t.value().which()) {
 643             //  first access to the token value
 644                 typedef iterator_range<Iterator> iterpair_type;
 645                 iterpair_type const& ip = t.matched();
 646
 647             // Interestingly enough we use the assign_to() framework defined in
 648             // Spirit.Qi allowing to convert the pair of iterators to almost any
 649             // required type (assign_to(), if available, uses the standard Spirit
 650             // parsers to do the conversion).
 651                 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
 652
 653             //  If you get an error during the compilation of the following
 654             //  assignment expression, you probably forgot to list one or more
 655             //  types used as token value types (in your token_def<...>
 656             //  definitions) in your definition of the token class. I.e. any token
 657             //  value type used for a token_def<...> definition has to be listed
 658             //  during the declaration of the token type to use. For instance let's
 659             //  assume we have two token_def's:
 660             //
 661             //      token_def<int> number; number = "...";
 662             //      token_def<std::string> identifier; identifier = "...";
 663             //
 664             //  Then you'll have to use the following token type definition
 665             //  (assuming you are using the token class):
 666             //
 667             //      typedef mpl::vector<int, std::string> token_values;
 668             //      typedef token<base_iter_type, token_values> token_type;
 669             //
 670             //  where: base_iter_type is the iterator type used to expose the
 671             //         underlying input stream.
 672             //
 673             //  This token_type has to be used as the second template parameter
 674             //  to the lexer class:
 675             //
 676             //      typedef lexer<base_iter_type, token_type> lexer_type;
 677             //
 678             //  again, assuming you're using the lexer<> template for your
 679             //  tokenization.
 680
 681                 typedef lex::lexertl::position_token<
 682                     Iterator, AttributeTypes, HasState, Idtype> token_type;
 683                 spirit::traits::assign_to(
 684                     attr, const_cast<token_type&>(t).value());   // re-assign value
 685             }
 686             else {
 687             // reuse the already assigned value
 688                 spirit::traits::assign_to(get<Attribute>(t.value()), attr);
 689             }
 690         }
 691     };
 692
 693     template <typename Attribute, typename Iterator, typename AttributeTypes
 694       , typename HasState, typename Idtype>
 695     struct assign_to_container_from_value<Attribute
 696           , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 697       : assign_to_attribute_from_value<Attribute
 698           , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 699     {};
 700
 701     ///////////////////////////////////////////////////////////////////////////
 702     //  These are called from the parse function of token_def if the token type
 703     //  has no special attribute type assigned
 704     template <typename Attribute, typename Iterator, typename HasState
 705       , typename Idtype>
 706     struct assign_to_attribute_from_value<Attribute
 707       , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> >
 708     {
 709         static void
 710         call(lex::lexertl::position_token<
 711                 Iterator, mpl::vector0<>, HasState, Idtype> const& t
 712           , Attribute& attr)
 713         {
 714             //  The default type returned by the token_def parser component (if
 715             //  it has no token value type assigned) is the pair of iterators
 716             //  to the matched character sequence.
 717             spirit::traits::assign_to(t.begin(), t.end(), attr);
 718         }
 719     };
 720
 721 //     template <typename Attribute, typename Iterator, typename HasState
 722 //       , typename Idtype>
 723 //     struct assign_to_container_from_value<Attribute
 724 //           , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> >
 725 //       : assign_to_attribute_from_value<Attribute
 726 //           , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> >
 727 //     {};
 728
 729     // same as above but using mpl::vector<> instead of mpl::vector0<>
 730     template <typename Attribute, typename Iterator, typename HasState
 731       , typename Idtype>
 732     struct assign_to_attribute_from_value<Attribute
 733       , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> >
 734     {
 735         static void
 736         call(lex::lexertl::position_token<
 737                 Iterator, mpl::vector<>, HasState, Idtype> const& t
 738           , Attribute& attr)
 739         {
 740             //  The default type returned by the token_def parser component (if
 741             //  it has no token value type assigned) is the pair of iterators
 742             //  to the matched character sequence.
 743             spirit::traits::assign_to(t.begin(), t.end(), attr);
 744         }
 745     };
 746
 747 //     template <typename Attribute, typename Iterator, typename HasState
 748 //       , typename Idtype>
 749 //     struct assign_to_container_from_value<Attribute
 750 //           , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> >
 751 //       : assign_to_attribute_from_value<Attribute
 752 //           , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> >
 753 //     {};
 754
 755     ///////////////////////////////////////////////////////////////////////////
 756     //  These are called from the parse function of token_def if the token type
 757     //  has no special attribute type assigned
 758     template <typename Attribute, typename Iterator, typename Attr
 759       , typename HasState, typename Idtype>
 760     struct assign_to_attribute_from_value<Attribute
 761       , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> >
 762     {
 763         static void
 764         call(lex::lexertl::position_token<
 765                 Iterator, mpl::vector1<Attr>, HasState, Idtype> const& t
 766           , Attribute& attr)
 767         {
 768         //  The goal of this function is to avoid the conversion of the pair of
 769         //  iterators (to the matched character sequence) into the token value
 770         //  of the required type being done more than once.
 771
 772             if (!t.has_value()) {
 773             //  first access to the token value
 774                 typedef iterator_range<Iterator> iterpair_type;
 775                 iterpair_type const& ip = t.matched();
 776
 777             // Interestingly enough we use the assign_to() framework defined in
 778             // Spirit.Qi allowing to convert the pair of iterators to almost any
 779             // required type (assign_to(), if available, uses the standard Spirit
 780             // parsers to do the conversion).
 781                 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
 782
 783             // Re-assign the attribute to the stored value
 784                 typedef lex::lexertl::position_token<
 785                     Iterator, mpl::vector1<Attr>, HasState, Idtype> token_type;
 786                 spirit::traits::assign_to(
 787                     attr, const_cast<token_type&>(t).value());
 788             }
 789             else {
 790             // reuse the already assigned value
 791                 spirit::traits::assign_to(t.value(), attr);
 792             }
 793         }
 794     };
 795
 796 //     template <typename Attribute, typename Iterator, typename Attr
 797 //       , typename HasState, typename Idtype>
 798 //     struct assign_to_container_from_value<Attribute
 799 //           , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> >
 800 //       : assign_to_attribute_from_value<Attribute
 801 //           , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> >
 802 //     {};
 803
 804     // same as above but using mpl::vector<Attr> instead of mpl::vector1<Attr>
 805     template <typename Attribute, typename Iterator, typename Attr
 806       , typename HasState, typename Idtype>
 807     struct assign_to_attribute_from_value<Attribute
 808       , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> >
 809     {
 810         static void
 811         call(lex::lexertl::position_token<
 812                 Iterator, mpl::vector<Attr>, HasState, Idtype> const& t
 813           , Attribute& attr)
 814         {
 815         //  The goal of this function is to avoid the conversion of the pair of
 816         //  iterators (to the matched character sequence) into the token value
 817         //  of the required type being done more than once.
 818
 819             if (!t.has_value()) {
 820             //  first access to the token value
 821                 typedef iterator_range<Iterator> iterpair_type;
 822                 iterpair_type const& ip = t.matched();
 823
 824             // Interestingly enough we use the assign_to() framework defined in
 825             // Spirit.Qi allowing to convert the pair of iterators to almost any
 826             // required type (assign_to(), if available, uses the standard Spirit
 827             // parsers to do the conversion).
 828                 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
 829
 830             // Re-assign the attribute to the stored value
 831                 typedef lex::lexertl::position_token<
 832                     Iterator, mpl::vector<Attr>, HasState, Idtype> token_type;
 833                 spirit::traits::assign_to(
 834                     attr, const_cast<token_type&>(t).value());
 835             }
 836             else {
 837             // reuse the already assigned value
 838                 spirit::traits::assign_to(t.value(), attr);
 839             }
 840         }
 841     };
 842
 843 //     template <typename Attribute, typename Iterator, typename Attr
 844 //       , typename HasState, typename Idtype>
 845 //     struct assign_to_container_from_value<Attribute
 846 //           , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> >
 847 //       : assign_to_attribute_from_value<Attribute
 848 //           , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> >
 849 //     {};
 850
 851     //  This is called from the parse function of token_def if the token type
 852     //  has been explicitly omitted (i.e. no attribute value is used), which
 853     //  essentially means that every attribute gets initialized using default
 854     //  constructed values.
 855     template <typename Attribute, typename Iterator, typename HasState
 856       , typename Idtype>
 857     struct assign_to_attribute_from_value<Attribute
 858       , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> >
 859     {
 860         static void
 861         call(lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> const& t
 862           , Attribute& attr)
 863         {
 864             // do nothing
 865         }
 866     };
 867
 868     template <typename Attribute, typename Iterator, typename HasState
 869       , typename Idtype>
 870     struct assign_to_container_from_value<Attribute
 871           , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> >
 872       : assign_to_attribute_from_value<Attribute
 873           , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> >
 874     {};
 875
 876     //  This is called from the parse function of lexer_def_
 877     template <typename Iterator, typename AttributeTypes, typename HasState
 878       , typename Idtype_, typename Idtype>
 879     struct assign_to_attribute_from_value<
 880         fusion::vector2<Idtype_, iterator_range<Iterator> >
 881       , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 882     {
 883         static void
 884         call(lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> const& t
 885           , fusion::vector2<Idtype_, iterator_range<Iterator> >& attr)
 886         {
 887             //  The type returned by the lexer_def_ parser components is a
 888             //  fusion::vector containing the token id of the matched token
 889             //  and the pair of iterators to the matched character sequence.
 890             typedef iterator_range<Iterator> iterpair_type;
 891             typedef fusion::vector2<Idtype_, iterator_range<Iterator> >
 892                 attribute_type;
 893
 894             iterpair_type const& ip = t.matched();
 895             attr = attribute_type(t.id(), ip);
 896         }
 897     };
 898
 899     template <typename Iterator, typename AttributeTypes, typename HasState
 900       , typename Idtype_, typename Idtype>
 901     struct assign_to_container_from_value<
 902             fusion::vector2<Idtype_, iterator_range<Iterator> >
 903           , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 904       : assign_to_attribute_from_value<
 905             fusion::vector2<Idtype_, iterator_range<Iterator> >
 906           , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
 907     {};
 908
 909     ///////////////////////////////////////////////////////////////////////////
 910     // Overload debug output for a single token, this integrates lexer tokens
 911     // with Qi's simple_trace debug facilities
 912     template <typename Iterator, typename Attribute, typename HasState
 913       , typename Idtype>
 914     struct token_printer_debug<
 915         lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> >
 916     {
 917         typedef lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> token_type;
 918
 919         template <typename Out>
 920         static void print(Out& out, token_type const& val)
 921         {
 922             out << '[';
 923             spirit::traits::print_token(out, val.value());
 924             out << ']';
 925         }
 926     };
 927 }}}
 928
 929 #endif