ceph/src/boost/boost/spirit/home/lex/lexer/lexertl/token.hpp

   1 //  Copyright (c) 2001-2011 Hartmut Kaiser
   2 //
   3 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
   4 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
   5
   6 #if !defined(BOOST_SPIRIT_LEX_TOKEN_FEB_10_2008_0751PM)
   7 #define BOOST_SPIRIT_LEX_TOKEN_FEB_10_2008_0751PM
   8
   9 #if defined(_MSC_VER)
  10 #pragma once
  11 #endif
  12
  13 #include <boost/config.hpp>
  14 #include <boost/detail/workaround.hpp>
  15 #include <boost/spirit/home/qi/detail/assign_to.hpp>
  16 #include <boost/spirit/home/support/attributes.hpp>
  17 #include <boost/spirit/home/support/argument.hpp>
  18 #include <boost/spirit/home/support/detail/lexer/generator.hpp>
  19 #include <boost/spirit/home/support/detail/lexer/rules.hpp>
  20 #include <boost/spirit/home/support/detail/lexer/consts.hpp>
  21 #include <boost/spirit/home/support/utree/utree_traits_fwd.hpp>
  22 #include <boost/spirit/home/lex/lexer/terminals.hpp>
  23 #include <boost/fusion/include/vector.hpp>
  24 #include <boost/fusion/include/at.hpp>
  25 #include <boost/fusion/include/value_at.hpp>
  26 #include <boost/variant.hpp>
  27 #include <boost/mpl/bool.hpp>
  28 #include <boost/mpl/vector.hpp>
  29 #include <boost/mpl/is_sequence.hpp>
  30 #include <boost/mpl/begin.hpp>
  31 #include <boost/mpl/insert.hpp>
  32 #include <boost/mpl/vector.hpp>
  33 #include <boost/mpl/if.hpp>
  34 #include <boost/mpl/or.hpp>
  35 #include <boost/range/iterator_range_core.hpp>
  36 #include <boost/type_traits/integral_promotion.hpp>
  37 #include <boost/type_traits/is_same.hpp>
  38 #include <boost/static_assert.hpp>
  39
  40 #if defined(BOOST_SPIRIT_DEBUG)
  41 #include <iosfwd>
  42 #endif
  43
  44 namespace boost { namespace spirit { namespace lex { namespace lexertl
  45 {
  46     ///////////////////////////////////////////////////////////////////////////
  47     //
  48     //  The token is the type of the objects returned by default by the
  49     //  iterator.
  50     //
  51     //    template parameters:
  52     //        Iterator        The type of the iterator used to access the
  53     //                        underlying character stream.
  54     //        AttributeTypes  A mpl sequence containing the types of all
  55     //                        required different token values to be supported
  56     //                        by this token type.
  57     //        HasState        A mpl::bool_ indicating, whether this token type
  58     //                        should support lexer states.
  59     //        Idtype          The type to use for the token id (defaults to
  60     //                        std::size_t).
  61     //
  62     //  It is possible to use other token types with the spirit::lex
  63     //  framework as well. If you plan to use a different type as your token
  64     //  type, you'll need to expose the following things from your token type
  65     //  to make it compatible with spirit::lex:
  66     //
  67     //    typedefs
  68     //        iterator_type   The type of the iterator used to access the
  69     //                        underlying character stream.
  70     //
  71     //        id_type         The type of the token id used.
  72     //
  73     //    methods
  74     //        default constructor
  75     //                        This should initialize the token as an end of
  76     //                        input token.
  77     //        constructors    The prototype of the other required
  78     //                        constructors should be:
  79     //
  80     //              token(int)
  81     //                        This constructor should initialize the token as
  82     //                        an invalid token (not carrying any specific
  83     //                        values)
  84     //
  85     //              where:  the int is used as a tag only and its value is
  86     //                      ignored
  87     //
  88     //                        and:
  89     //
  90     //              token(Idtype id, std::size_t state,
  91     //                    iterator_type first, iterator_type last);
  92     //
  93     //              where:  id:           token id
  94     //                      state:        lexer state this token was matched in
  95     //                      first, last:  pair of iterators marking the matched
  96     //                                    range in the underlying input stream
  97     //
  98     //        accessors
  99     //              id()      return the token id of the matched input sequence
 100     //              id(newid) set the token id of the token instance
 101     //
 102     //              state()   return the lexer state this token was matched in
 103     //
 104     //              value()   return the token value
 105     //
 106     //  Additionally, you will have to implement a couple of helper functions
 107     //  in the same namespace as the token type: a comparison operator==() to
 108     //  compare your token instances, a token_is_valid() function and different
 109     //  specializations of the Spirit customization point
 110     //  assign_to_attribute_from_value as shown below.
 111     //
 112     ///////////////////////////////////////////////////////////////////////////
 113     template <typename Iterator = char const*
 114       , typename AttributeTypes = mpl::vector0<>
 115       , typename HasState = mpl::true_
 116       , typename Idtype = std::size_t>
 117     struct token;
 118
 119     ///////////////////////////////////////////////////////////////////////////
 120     //  This specialization of the token type doesn't contain any item data and
 121     //  doesn't support working with lexer states.
 122     ///////////////////////////////////////////////////////////////////////////
 123     template <typename Iterator, typename Idtype>
 124     struct token<Iterator, lex::omit, mpl::false_, Idtype>
 125     {
 126         typedef Iterator iterator_type;
 127         typedef mpl::false_ has_state;
 128         typedef Idtype id_type;
 129         typedef unused_type token_value_type;
 130
 131         //  default constructed tokens correspond to EOI tokens
 132         token() : id_(id_type(boost::lexer::npos)) {}
 133
 134         //  construct an invalid token
 135         explicit token(int) : id_(id_type(0)) {}
 136
 137         token(id_type id, std::size_t) : id_(id) {}
 138
 139         token(id_type id, std::size_t, token_value_type)
 140           : id_(id) {}
 141
 142         token_value_type& value() { static token_value_type u; return u; }
 143         token_value_type const& value() const { return unused; }
 144
 145 #if defined(BOOST_SPIRIT_DEBUG)
 146         token(id_type id, std::size_t, Iterator const& first
 147               , Iterator const& last)
 148           : matched_(first, last)
 149           , id_(id) {}
 150 #else
 151         token(id_type id, std::size_t, Iterator const&, Iterator const&)
 152           : id_(id) {}
 153 #endif
 154
 155         //  this default conversion operator is needed to allow the direct
 156         //  usage of tokens in conjunction with the primitive parsers defined
 157         //  in Qi
 158         operator id_type() const { return static_cast<id_type>(id_); }
 159
 160         //  Retrieve or set the token id of this token instance.
 161         id_type id() const { return static_cast<id_type>(id_); }
 162         void id(id_type newid) { id_ = newid; }
 163
 164         std::size_t state() const { return 0; }   // always '0' (INITIAL state)
 165
 166         bool is_valid() const
 167         {
 168             return 0 != id_ && id_type(boost::lexer::npos) != id_;
 169         }
 170
 171 #if defined(BOOST_SPIRIT_DEBUG)
 172 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 173         // workaround for MSVC10 which has problems copying a default
 174         // constructed iterator_range
 175         token& operator= (token const& rhs)
 176         {
 177             if (this != &rhs)
 178             {
 179                 id_ = rhs.id_;
 180                 if (is_valid())
 181                     matched_ = rhs.matched_;
 182             }
 183             return *this;
 184         }
 185 #endif
 186         std::pair<Iterator, Iterator> matched_;
 187 #endif
 188
 189     protected:
 190         typename boost::integral_promotion<id_type>::type id_;            // token id, 0 if nothing has been matched
 191     };
 192
 193 #if defined(BOOST_SPIRIT_DEBUG)
 194     template <typename Char, typename Traits, typename Iterator
 195       , typename AttributeTypes, typename HasState, typename Idtype>
 196     inline std::basic_ostream<Char, Traits>&
 197     operator<< (std::basic_ostream<Char, Traits>& os
 198       , token<Iterator, AttributeTypes, HasState, Idtype> const& t)
 199     {
 200         if (t.is_valid()) {
 201             Iterator end = t.matched_.second;
 202             for (Iterator it = t.matched_.first; it != end; ++it)
 203                 os << *it;
 204         }
 205         else {
 206             os << "<invalid token>";
 207         }
 208         return os;
 209     }
 210 #endif
 211
 212     ///////////////////////////////////////////////////////////////////////////
 213     //  This specialization of the token type doesn't contain any item data but
 214     //  supports working with lexer states.
 215     ///////////////////////////////////////////////////////////////////////////
 216     template <typename Iterator, typename Idtype>
 217     struct token<Iterator, lex::omit, mpl::true_, Idtype>
 218       : token<Iterator, lex::omit, mpl::false_, Idtype>
 219     {
 220     private:
 221         typedef token<Iterator, lex::omit, mpl::false_, Idtype> base_type;
 222
 223     public:
 224         typedef typename base_type::id_type id_type;
 225         typedef Iterator iterator_type;
 226         typedef mpl::true_ has_state;
 227         typedef unused_type token_value_type;
 228
 229         //  default constructed tokens correspond to EOI tokens
 230         token() : state_(boost::lexer::npos) {}
 231
 232         //  construct an invalid token
 233         explicit token(int) : base_type(0), state_(boost::lexer::npos) {}
 234
 235         token(id_type id, std::size_t state)
 236           : base_type(id, boost::lexer::npos), state_(state) {}
 237
 238         token(id_type id, std::size_t state, token_value_type)
 239           : base_type(id, boost::lexer::npos, unused)
 240           , state_(state) {}
 241
 242         token(id_type id, std::size_t state
 243               , Iterator const& first, Iterator const& last)
 244           : base_type(id, boost::lexer::npos, first, last)
 245           , state_(state) {}
 246
 247         std::size_t state() const { return state_; }
 248
 249 #if defined(BOOST_SPIRIT_DEBUG) && BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 250         // workaround for MSVC10 which has problems copying a default
 251         // constructed iterator_range
 252         token& operator= (token const& rhs)
 253         {
 254             if (this != &rhs)
 255             {
 256                 this->base_type::operator=(static_cast<base_type const&>(rhs));
 257                 state_ = rhs.state_;
 258             }
 259             return *this;
 260         }
 261 #endif
 262
 263     protected:
 264         std::size_t state_;      // lexer state this token was matched in
 265     };
 266
 267     ///////////////////////////////////////////////////////////////////////////
 268     //  The generic version of the token type derives from the
 269     //  specialization above and adds a single data member holding the item
 270     //  data carried by the token instance.
 271     ///////////////////////////////////////////////////////////////////////////
 272     namespace detail
 273     {
 274         ///////////////////////////////////////////////////////////////////////
 275         //  Meta-function to calculate the type of the variant data item to be
 276         //  stored with each token instance.
 277         //
 278         //  Note: The iterator pair needs to be the first type in the list of
 279         //        types supported by the generated variant type (this is being
 280         //        used to identify whether the stored data item in a particular
 281         //        token instance needs to be converted from the pair of
 282         //        iterators (see the first of the assign_to_attribute_from_value
 283         //        specializations below).
 284         ///////////////////////////////////////////////////////////////////////
 285         template <typename IteratorPair, typename AttributeTypes>
 286         struct token_value_typesequence
 287         {
 288             typedef typename mpl::insert<
 289                 AttributeTypes
 290               , typename mpl::begin<AttributeTypes>::type
 291               , IteratorPair
 292             >::type sequence_type;
 293             typedef typename make_variant_over<sequence_type>::type type;
 294         };
 295
 296         ///////////////////////////////////////////////////////////////////////
 297         //  The type of the data item stored with a token instance is defined
 298         //  by the template parameter 'AttributeTypes' and may be:
 299         //
 300         //     lex::omit:         no data item is stored with the token
 301         //                        instance (this is handled by the
 302         //                        specializations of the token class
 303         //                        below)
 304         //     mpl::vector0<>:    each token instance stores a pair of
 305         //                        iterators pointing to the matched input
 306         //                        sequence
 307         //     mpl::vector<...>:  each token instance stores a variant being
 308         //                        able to store the pair of iterators pointing
 309         //                        to the matched input sequence, or any of the
 310         //                        types a specified in the mpl::vector<>
 311         //
 312         //  All this is done to ensure the token type is as small (in terms
 313         //  of its byte-size) as possible.
 314         ///////////////////////////////////////////////////////////////////////
 315         template <typename IteratorPair, typename AttributeTypes>
 316         struct token_value_type
 317           : mpl::eval_if<
 318                 mpl::or_<
 319                     is_same<AttributeTypes, mpl::vector0<> >
 320                   , is_same<AttributeTypes, mpl::vector<> > >
 321               , mpl::identity<IteratorPair>
 322               , token_value_typesequence<IteratorPair, AttributeTypes> >
 323         {};
 324     }
 325
 326     template <typename Iterator, typename AttributeTypes, typename HasState
 327       , typename Idtype>
 328     struct token : token<Iterator, lex::omit, HasState, Idtype>
 329     {
 330     private: // precondition assertions
 331         BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value ||
 332                             is_same<AttributeTypes, lex::omit>::value));
 333         typedef token<Iterator, lex::omit, HasState, Idtype> base_type;
 334
 335     protected:
 336         //  If no additional token value types are given, the token will
 337         //  hold the plain pair of iterators pointing to the matched range
 338         //  in the underlying input sequence. Otherwise the token value is
 339         //  stored as a variant and will again hold the pair of iterators but
 340         //  is able to hold any of the given data types as well. The conversion
 341         //  from the iterator pair to the required data type is done when it is
 342         //  accessed for the first time.
 343         typedef iterator_range<Iterator> iterpair_type;
 344
 345     public:
 346         typedef typename base_type::id_type id_type;
 347         typedef typename detail::token_value_type<
 348             iterpair_type, AttributeTypes
 349         >::type token_value_type;
 350
 351         typedef Iterator iterator_type;
 352
 353         //  default constructed tokens correspond to EOI tokens
 354         token() : value_(iterpair_type(iterator_type(), iterator_type())) {}
 355
 356         //  construct an invalid token
 357         explicit token(int)
 358           : base_type(0)
 359           , value_(iterpair_type(iterator_type(), iterator_type())) {}
 360
 361         token(id_type id, std::size_t state, token_value_type const& value)
 362           : base_type(id, state, value)
 363           , value_(value) {}
 364
 365         token(id_type id, std::size_t state, Iterator const& first
 366               , Iterator const& last)
 367           : base_type(id, state, first, last)
 368           , value_(iterpair_type(first, last)) {}
 369
 370         token_value_type& value() { return value_; }
 371         token_value_type const& value() const { return value_; }
 372
 373 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
 374         // workaround for MSVC10 which has problems copying a default
 375         // constructed iterator_range
 376         token& operator= (token const& rhs)
 377         {
 378             if (this != &rhs)
 379             {
 380                 this->base_type::operator=(static_cast<base_type const&>(rhs));
 381                 if (this->is_valid())
 382                     value_ = rhs.value_;
 383             }
 384             return *this;
 385         }
 386 #endif
 387
 388     protected:
 389         token_value_type value_; // token value, by default a pair of iterators
 390     };
 391
 392     ///////////////////////////////////////////////////////////////////////////
 393     //  tokens are considered equal, if their id's match (these are unique)
 394     template <typename Iterator, typename AttributeTypes, typename HasState
 395       , typename Idtype>
 396     inline bool
 397     operator== (token<Iterator, AttributeTypes, HasState, Idtype> const& lhs,
 398                 token<Iterator, AttributeTypes, HasState, Idtype> const& rhs)
 399     {
 400         return lhs.id() == rhs.id();
 401     }
 402
 403     ///////////////////////////////////////////////////////////////////////////
 404     //  This overload is needed by the multi_pass/functor_input_policy to
 405     //  validate a token instance. It has to be defined in the same namespace
 406     //  as the token class itself to allow ADL to find it.
 407     ///////////////////////////////////////////////////////////////////////////
 408     template <typename Iterator, typename AttributeTypes, typename HasState
 409       , typename Idtype>
 410     inline bool
 411     token_is_valid(token<Iterator, AttributeTypes, HasState, Idtype> const& t)
 412     {
 413         return t.is_valid();
 414     }
 415 }}}}
 416
 417 namespace boost { namespace spirit { namespace traits
 418 {
 419     ///////////////////////////////////////////////////////////////////////////
 420     //  We have to provide specializations for the customization point
 421     //  assign_to_attribute_from_value allowing to extract the needed value
 422     //  from the token.
 423     ///////////////////////////////////////////////////////////////////////////
 424
 425     //  This is called from the parse function of token_def if the token_def
 426     //  has been defined to carry a special attribute type
 427     template <typename Attribute, typename Iterator, typename AttributeTypes
 428       , typename HasState, typename Idtype>
 429     struct assign_to_attribute_from_value<Attribute
 430       , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
 431     {
 432         static void
 433         call(lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> const& t
 434           , Attribute& attr)
 435         {
 436         //  The goal of this function is to avoid the conversion of the pair of
 437         //  iterators (to the matched character sequence) into the token value
 438         //  of the required type being done more than once. For this purpose it
 439         //  checks whether the stored value type is still the default one (pair
 440         //  of iterators) and if yes, replaces the pair of iterators with the
 441         //  converted value to be returned from subsequent calls.
 442
 443             if (0 == t.value().which()) {
 444             //  first access to the token value
 445                 typedef iterator_range<Iterator> iterpair_type;
 446                 iterpair_type const& ip = boost::get<iterpair_type>(t.value());
 447
 448             // Interestingly enough we use the assign_to() framework defined in
 449             // Spirit.Qi allowing to convert the pair of iterators to almost any
 450             // required type (assign_to(), if available, uses the standard Spirit
 451             // parsers to do the conversion).
 452                 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
 453
 454             //  If you get an error during the compilation of the following
 455             //  assignment expression, you probably forgot to list one or more
 456             //  types used as token value types (in your token_def<...>
 457             //  definitions) in your definition of the token class. I.e. any token
 458             //  value type used for a token_def<...> definition has to be listed
 459             //  during the declaration of the token type to use. For instance let's
 460             //  assume we have two token_def's:
 461             //
 462             //      token_def<int> number; number = "...";
 463             //      token_def<std::string> identifier; identifier = "...";
 464             //
 465             //  Then you'll have to use the following token type definition
 466             //  (assuming you are using the token class):
 467             //
 468             //      typedef mpl::vector<int, std::string> token_values;
 469             //      typedef token<base_iter_type, token_values> token_type;
 470             //
 471             //  where: base_iter_type is the iterator type used to expose the
 472             //         underlying input stream.
 473             //
 474             //  This token_type has to be used as the second template parameter
 475             //  to the lexer class:
 476             //
 477             //      typedef lexer<base_iter_type, token_type> lexer_type;
 478             //
 479             //  again, assuming you're using the lexer<> template for your
 480             //  tokenization.
 481
 482                 typedef lex::lexertl::token<
 483                     Iterator, AttributeTypes, HasState, Idtype> token_type;
 484                 spirit::traits::assign_to(
 485                     attr, const_cast<token_type&>(t).value());   // re-assign value
 486             }
 487             else {
 488             // reuse the already assigned value
 489                 spirit::traits::assign_to(boost::get<Attribute>(t.value()), attr);
 490             }
 491         }
 492     };
 493
 494     template <typename Attribute, typename Iterator, typename AttributeTypes
 495       , typename HasState, typename Idtype>
 496     struct assign_to_container_from_value<Attribute
 497           , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
 498       : assign_to_attribute_from_value<Attribute
 499           , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
 500     {};
 501
 502     template <typename Iterator, typename AttributeTypes
 503       , typename HasState, typename Idtype>
 504     struct assign_to_container_from_value<utree
 505           , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
 506       : assign_to_attribute_from_value<utree
 507           , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
 508     {};
 509
 510     template <typename Iterator>
 511     struct assign_to_container_from_value<
 512         iterator_range<Iterator>, iterator_range<Iterator> >
 513     {
 514         static void
 515         call(iterator_range<Iterator> const& val, iterator_range<Iterator>& attr)
 516         {
 517             attr = val;
 518         }
 519     };
 520
 521     //  These are called from the parse function of token_def if the token type
 522     //  has no special attribute type assigned
 523     template <typename Attribute, typename Iterator, typename HasState
 524       , typename Idtype>
 525     struct assign_to_attribute_from_value<Attribute
 526       , lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> >
 527     {
 528         static void
 529         call(lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> const& t
 530           , Attribute& attr)
 531         {
 532             //  The default type returned by the token_def parser component (if
 533             //  it has no token value type assigned) is the pair of iterators
 534             //  to the matched character sequence.
 535             spirit::traits::assign_to(t.value().begin(), t.value().end(), attr);
 536         }
 537     };
 538
 539 //     template <typename Attribute, typename Iterator, typename HasState
 540 //       , typename Idtype>
 541 //     struct assign_to_container_from_value<Attribute
 542 //           , lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> >
 543 //       : assign_to_attribute_from_value<Attribute
 544 //           , lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> >
 545 //     {};
 546
 547     // same as above but using mpl::vector<> instead of mpl::vector0<>
 548     template <typename Attribute, typename Iterator, typename HasState
 549       , typename Idtype>
 550     struct assign_to_attribute_from_value<Attribute
 551       , lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> >
 552     {
 553         static void
 554         call(lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> const& t
 555           , Attribute& attr)
 556         {
 557             //  The default type returned by the token_def parser component (if
 558             //  it has no token value type assigned) is the pair of iterators
 559             //  to the matched character sequence.
 560             spirit::traits::assign_to(t.value().begin(), t.value().end(), attr);
 561         }
 562     };
 563
 564 //     template <typename Attribute, typename Iterator, typename HasState
 565 //       , typename Idtype>
 566 //     struct assign_to_container_from_value<Attribute
 567 //           , lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> >
 568 //       : assign_to_attribute_from_value<Attribute
 569 //           , lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> >
 570 //     {};
 571
 572     //  This is called from the parse function of token_def if the token type
 573     //  has been explicitly omitted (i.e. no attribute value is used), which
 574     //  essentially means that every attribute gets initialized using default
 575     //  constructed values.
 576     template <typename Attribute, typename Iterator, typename HasState
 577       , typename Idtype>
 578     struct assign_to_attribute_from_value<Attribute
 579       , lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> >
 580     {
 581         static void
 582         call(lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> const&
 583           , Attribute&)
 584         {
 585             // do nothing
 586         }
 587     };
 588
 589     template <typename Attribute, typename Iterator, typename HasState
 590       , typename Idtype>
 591     struct assign_to_container_from_value<Attribute
 592           , lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> >
 593       : assign_to_attribute_from_value<Attribute
 594           , lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> >
 595     {};
 596
 597     //  This is called from the parse function of lexer_def_
 598     template <typename Iterator, typename AttributeTypes, typename HasState
 599       , typename Idtype_, typename Idtype>
 600     struct assign_to_attribute_from_value<
 601         fusion::vector2<Idtype_, iterator_range<Iterator> >
 602       , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
 603     {
 604         static void
 605         call(lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> const& t
 606           , fusion::vector2<Idtype_, iterator_range<Iterator> >& attr)
 607         {
 608             //  The type returned by the lexer_def_ parser components is a
 609             //  fusion::vector containing the token id of the matched token
 610             //  and the pair of iterators to the matched character sequence.
 611             typedef iterator_range<Iterator> iterpair_type;
 612             typedef fusion::vector2<Idtype_, iterator_range<Iterator> >
 613                 attribute_type;
 614
 615             iterpair_type const& ip = boost::get<iterpair_type>(t.value());
 616             attr = attribute_type(t.id(), ip);
 617         }
 618     };
 619
 620     template <typename Iterator, typename AttributeTypes, typename HasState
 621       , typename Idtype_, typename Idtype>
 622     struct assign_to_container_from_value<
 623             fusion::vector2<Idtype_, iterator_range<Iterator> >
 624           , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
 625       : assign_to_attribute_from_value<
 626             fusion::vector2<Idtype_, iterator_range<Iterator> >
 627           , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
 628     {};
 629
 630     ///////////////////////////////////////////////////////////////////////////
 631     // Overload debug output for a single token, this integrates lexer tokens
 632     // with Qi's simple_trace debug facilities
 633     template <typename Iterator, typename Attribute, typename HasState
 634       , typename Idtype>
 635     struct token_printer_debug<
 636         lex::lexertl::token<Iterator, Attribute, HasState, Idtype> >
 637     {
 638         typedef lex::lexertl::token<Iterator, Attribute, HasState, Idtype> token_type;
 639
 640         template <typename Out>
 641         static void print(Out& out, token_type const& val)
 642         {
 643             out << '[';
 644             spirit::traits::print_token(out, val.value());
 645             out << ']';
 646         }
 647     };
 648 }}}
 649
 650 #endif