]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // Copyright (c) 2001-2011 Hartmut Kaiser |
2 | // | |
3 | // Distributed under the Boost Software License, Version 1.0. (See accompanying | |
4 | // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) | |
5 | ||
6 | #if !defined(BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM) | |
7 | #define BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM | |
8 | ||
9 | #if defined(_MSC_VER) | |
10 | #pragma once | |
11 | #endif | |
12 | ||
13 | #include <boost/config.hpp> | |
14 | #include <boost/detail/workaround.hpp> | |
15 | #include <boost/spirit/home/qi/detail/assign_to.hpp> | |
16 | #include <boost/spirit/home/support/attributes.hpp> | |
17 | #include <boost/spirit/home/support/argument.hpp> | |
18 | #include <boost/spirit/home/support/detail/lexer/generator.hpp> | |
19 | #include <boost/spirit/home/support/detail/lexer/rules.hpp> | |
20 | #include <boost/spirit/home/support/detail/lexer/consts.hpp> | |
21 | #include <boost/spirit/home/support/utree/utree_traits_fwd.hpp> | |
22 | #include <boost/spirit/home/lex/lexer/terminals.hpp> | |
23 | #include <boost/fusion/include/vector.hpp> | |
24 | #include <boost/fusion/include/at.hpp> | |
25 | #include <boost/fusion/include/value_at.hpp> | |
26 | #include <boost/detail/iterator.hpp> | |
27 | #include <boost/variant.hpp> | |
28 | #include <boost/mpl/vector.hpp> | |
29 | #include <boost/mpl/bool.hpp> | |
30 | #include <boost/mpl/is_sequence.hpp> | |
31 | #include <boost/mpl/begin.hpp> | |
32 | #include <boost/mpl/insert.hpp> | |
33 | #include <boost/mpl/vector.hpp> | |
34 | #include <boost/mpl/if.hpp> | |
35 | #include <boost/mpl/or.hpp> | |
36 | #include <boost/type_traits/is_same.hpp> | |
37 | #include <boost/range/iterator_range.hpp> | |
38 | #include <boost/static_assert.hpp> | |
39 | ||
40 | #if defined(BOOST_SPIRIT_DEBUG) | |
41 | #include <iosfwd> | |
42 | #endif | |
43 | ||
44 | namespace boost { namespace spirit { namespace lex { namespace lexertl | |
45 | { | |
46 | /////////////////////////////////////////////////////////////////////////// | |
47 | // | |
48 | // The position_token is the type of the objects returned by the | |
49 | // iterator if it has been specified while instantiating the lexer object. | |
50 | // | |
51 | // template parameters: | |
52 | // Iterator The type of the iterator used to access the | |
53 | // underlying character stream. | |
54 | // AttributeTypes A mpl sequence containing the types of all | |
55 | // required different token values to be supported | |
56 | // by this token type. | |
57 | // HasState A mpl::bool_ indicating, whether this token type | |
58 | // should support lexer states. | |
59 | // Idtype The type to use for the token id (defaults to | |
60 | // std::size_t). | |
61 | // | |
62 | // It is possible to use other token types with the spirit::lex | |
63 | // framework as well. If you plan to use a different type as your token | |
64 | // type, you'll need to expose the following things from your token type | |
65 | // to make it compatible with spirit::lex: | |
66 | // | |
67 | // typedefs | |
68 | // iterator_type The type of the iterator used to access the | |
69 | // underlying character stream. | |
70 | // | |
71 | // id_type The type of the token id used. | |
72 | // | |
73 | // methods | |
74 | // default constructor | |
75 | // This should initialize the token as an end of | |
76 | // input token. | |
77 | // constructors The prototype of the other required | |
78 | // constructors should be: | |
79 | // | |
80 | // token(int) | |
81 | // This constructor should initialize the token as | |
82 | // an invalid token (not carrying any specific | |
83 | // values) | |
84 | // | |
85 | // where: the int is used as a tag only and its value is | |
86 | // ignored | |
87 | // | |
88 | // and: | |
89 | // | |
90 | // token(Idtype id, std::size_t state, | |
91 | // iterator_type first, iterator_type last); | |
92 | // | |
93 | // where: id: token id | |
94 | // state: lexer state this token was matched in | |
95 | // first, last: pair of iterators marking the matched | |
96 | // range in the underlying input stream | |
97 | // | |
98 | // accessors | |
99 | // id() return the token id of the matched input sequence | |
100 | // id(newid) set the token id of the token instance | |
101 | // | |
102 | // state() return the lexer state this token was matched in | |
103 | // | |
104 | // value() return the token value | |
105 | // | |
106 | // Additionally, you will have to implement a couple of helper functions | |
107 | // in the same namespace as the token type: a comparison operator==() to | |
108 | // compare your token instances, a token_is_valid() function and different | |
109 | // specializations of the Spirit customization point | |
110 | // assign_to_attribute_from_value as shown below. | |
111 | // | |
112 | /////////////////////////////////////////////////////////////////////////// | |
113 | template <typename Iterator = char const* | |
114 | , typename AttributeTypes = mpl::vector0<> | |
115 | , typename HasState = mpl::true_ | |
116 | , typename Idtype = std::size_t> | |
117 | struct position_token; | |
118 | ||
119 | /////////////////////////////////////////////////////////////////////////// | |
120 | // This specialization of the token type doesn't contain any item data and | |
121 | // doesn't support working with lexer states. Although, like all other | |
122 | // variants of position_token, it carries a pair of iterators marking the | |
123 | // begin and the end of the matched character sequence. | |
124 | /////////////////////////////////////////////////////////////////////////// | |
125 | template <typename Iterator, typename Idtype> | |
126 | struct position_token<Iterator, lex::omit, mpl::false_, Idtype> | |
127 | { | |
128 | typedef Iterator iterator_type; | |
129 | typedef iterator_range<iterator_type> iterpair_type; | |
130 | typedef mpl::false_ has_state; | |
131 | typedef Idtype id_type; | |
132 | typedef unused_type token_value_type; | |
133 | ||
134 | // default constructed tokens correspond to EOI tokens | |
135 | position_token() | |
136 | : id_(id_type(boost::lexer::npos)) {} | |
137 | ||
138 | // construct an invalid token | |
139 | explicit position_token(int) | |
140 | : id_(id_type(0)) {} | |
141 | ||
142 | position_token(id_type id, std::size_t) | |
143 | : id_(id) {} | |
144 | ||
145 | position_token(id_type id, std::size_t, token_value_type) | |
146 | : id_(id) {} | |
147 | ||
148 | position_token(id_type id, std::size_t, Iterator const& first | |
149 | , Iterator const& last) | |
150 | : id_(id), matched_(first, last) {} | |
151 | ||
152 | // this default conversion operator is needed to allow the direct | |
153 | // usage of tokens in conjunction with the primitive parsers defined | |
154 | // in Qi | |
155 | operator id_type() const { return id_; } | |
156 | ||
157 | // Retrieve or set the token id of this token instance. | |
158 | id_type id() const { return id_; } | |
159 | void id(id_type newid) { id_ = newid; } | |
160 | ||
161 | std::size_t state() const { return 0; } // always '0' (INITIAL state) | |
162 | ||
163 | bool is_valid() const | |
164 | { | |
165 | return 0 != id_ && id_type(boost::lexer::npos) != id_; | |
166 | } | |
167 | ||
168 | // access the stored iterator range of the matched input sequence | |
169 | iterator_type begin() const { return matched_.begin(); } | |
170 | iterator_type end() const { return matched_.end(); } | |
171 | ||
172 | iterpair_type& matched() { return matched_; } | |
173 | iterpair_type const& matched() const { return matched_; } | |
174 | ||
175 | token_value_type& value() { static token_value_type u; return u; } | |
176 | token_value_type const& value() const { return unused; } | |
177 | ||
178 | #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) | |
179 | // workaround for MSVC10 which has problems copying a default | |
180 | // constructed iterator_range | |
181 | position_token& operator= (position_token const& rhs) | |
182 | { | |
183 | if (this != &rhs) | |
184 | { | |
185 | id_ = rhs.id_; | |
186 | if (is_valid()) | |
187 | matched_ = rhs.matched_; | |
188 | } | |
189 | return *this; | |
190 | } | |
191 | #endif | |
192 | ||
193 | protected: | |
194 | id_type id_; // token id, 0 if nothing has been matched | |
195 | iterpair_type matched_; // matched input sequence | |
196 | }; | |
197 | ||
198 | #if defined(BOOST_SPIRIT_DEBUG) | |
199 | template <typename Char, typename Traits, typename Iterator | |
200 | , typename AttributeTypes, typename HasState, typename Idtype> | |
201 | inline std::basic_ostream<Char, Traits>& | |
202 | operator<< (std::basic_ostream<Char, Traits>& os | |
203 | , position_token<Iterator, AttributeTypes, HasState, Idtype> const& t) | |
204 | { | |
205 | if (t.is_valid()) { | |
206 | Iterator end = t.end(); | |
207 | for (Iterator it = t.begin(); it != end; ++it) | |
208 | os << *it; | |
209 | } | |
210 | else { | |
211 | os << "<invalid token>"; | |
212 | } | |
213 | return os; | |
214 | } | |
215 | #endif | |
216 | ||
217 | /////////////////////////////////////////////////////////////////////////// | |
218 | // This specialization of the token type doesn't contain any item data but | |
219 | // supports working with lexer states. | |
220 | /////////////////////////////////////////////////////////////////////////// | |
221 | template <typename Iterator, typename Idtype> | |
222 | struct position_token<Iterator, lex::omit, mpl::true_, Idtype> | |
223 | : position_token<Iterator, lex::omit, mpl::false_, Idtype> | |
224 | { | |
225 | private: | |
226 | typedef position_token<Iterator, lex::omit, mpl::false_, Idtype> | |
227 | base_type; | |
228 | ||
229 | public: | |
230 | typedef typename base_type::id_type id_type; | |
231 | typedef Iterator iterator_type; | |
232 | typedef mpl::true_ has_state; | |
233 | typedef unused_type token_value_type; | |
234 | ||
235 | // default constructed tokens correspond to EOI tokens | |
236 | position_token() : state_(boost::lexer::npos) {} | |
237 | ||
238 | // construct an invalid token | |
239 | explicit position_token(int) | |
240 | : base_type(0), state_(boost::lexer::npos) {} | |
241 | ||
242 | position_token(id_type id, std::size_t state) | |
243 | : base_type(id, boost::lexer::npos), state_(state) {} | |
244 | ||
245 | position_token(id_type id, std::size_t state, token_value_type) | |
246 | : base_type(id, boost::lexer::npos, unused) | |
247 | , state_(state) {} | |
248 | ||
249 | position_token(id_type id, std::size_t state | |
250 | , Iterator const& first, Iterator const& last) | |
251 | : base_type(id, boost::lexer::npos, first, last) | |
252 | , state_(state) {} | |
253 | ||
254 | std::size_t state() const { return state_; } | |
255 | ||
256 | #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) | |
257 | // workaround for MSVC10 which has problems copying a default | |
258 | // constructed iterator_range | |
259 | position_token& operator= (position_token const& rhs) | |
260 | { | |
261 | if (this != &rhs) | |
262 | { | |
263 | this->base_type::operator=(static_cast<base_type const&>(rhs)); | |
264 | state_ = rhs.state_; | |
265 | } | |
266 | return *this; | |
267 | } | |
268 | #endif | |
269 | ||
270 | protected: | |
271 | std::size_t state_; // lexer state this token was matched in | |
272 | }; | |
273 | ||
274 | /////////////////////////////////////////////////////////////////////////// | |
275 | // These specializations for an empty attribute list cause all token | |
276 | // instances to expose as it attribute the iterator_range pointing to the | |
277 | // matched input sequence. | |
278 | /////////////////////////////////////////////////////////////////////////// | |
279 | template <typename Iterator, typename HasState, typename Idtype> | |
280 | struct position_token<Iterator, mpl::vector<>, HasState, Idtype> | |
281 | : position_token<Iterator, lex::omit, HasState, Idtype> | |
282 | { | |
283 | private: | |
284 | typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; | |
285 | ||
286 | public: | |
287 | typedef typename base_type::id_type id_type; | |
288 | typedef typename base_type::iterator_type iterator_type; | |
289 | typedef typename base_type::iterpair_type iterpair_type; | |
290 | typedef HasState has_state; | |
291 | typedef iterpair_type token_value_type; | |
292 | ||
293 | // default constructed tokens correspond to EOI tokens | |
294 | position_token() {} | |
295 | ||
296 | // construct an invalid token | |
297 | explicit position_token(int) | |
298 | : base_type(0) {} | |
299 | ||
300 | position_token(id_type id, std::size_t state) | |
301 | : base_type(id, state) {} | |
302 | ||
303 | position_token(id_type id, std::size_t state, token_value_type) | |
304 | : base_type(id, state, unused) {} | |
305 | ||
306 | position_token(id_type id, std::size_t state | |
307 | , Iterator const& first, Iterator const& last) | |
308 | : base_type(id, state, first, last) {} | |
309 | ||
310 | token_value_type& value() { return this->base_type::matched(); } | |
311 | token_value_type const& value() const { return this->base_type::matched(); } | |
312 | }; | |
313 | ||
314 | template <typename Iterator, typename HasState, typename Idtype> | |
315 | struct position_token<Iterator, mpl::vector0<>, HasState, Idtype> | |
316 | : position_token<Iterator, lex::omit, HasState, Idtype> | |
317 | { | |
318 | private: | |
319 | typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; | |
320 | ||
321 | public: | |
322 | typedef typename base_type::id_type id_type; | |
323 | typedef typename base_type::iterator_type iterator_type; | |
324 | typedef typename base_type::iterpair_type iterpair_type; | |
325 | typedef HasState has_state; | |
326 | typedef iterpair_type token_value_type; | |
327 | ||
328 | // default constructed tokens correspond to EOI tokens | |
329 | position_token() {} | |
330 | ||
331 | // construct an invalid token | |
332 | explicit position_token(int) | |
333 | : base_type(0) {} | |
334 | ||
335 | position_token(id_type id, std::size_t state) | |
336 | : base_type(id, state) {} | |
337 | ||
338 | position_token(id_type id, std::size_t state, token_value_type) | |
339 | : base_type(id, state, unused) {} | |
340 | ||
341 | position_token(id_type id, std::size_t state | |
342 | , Iterator const& first, Iterator const& last) | |
343 | : base_type(id, state, first, last) {} | |
344 | ||
345 | token_value_type& value() { return this->base_type::matched(); } | |
346 | token_value_type const& value() const { return this->base_type::matched(); } | |
347 | }; | |
348 | ||
349 | /////////////////////////////////////////////////////////////////////////// | |
350 | // These specializations for an attribute list of length one cause all token | |
351 | // instances to expose the specified type as its attribute. | |
352 | /////////////////////////////////////////////////////////////////////////// | |
353 | template <typename Iterator, typename Attribute, typename HasState | |
354 | , typename Idtype> | |
355 | struct position_token<Iterator, mpl::vector<Attribute>, HasState, Idtype> | |
356 | : position_token<Iterator, lex::omit, HasState, Idtype> | |
357 | { | |
358 | private: | |
359 | typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; | |
360 | ||
361 | public: | |
362 | typedef typename base_type::id_type id_type; | |
363 | typedef typename base_type::iterator_type iterator_type; | |
364 | typedef typename base_type::iterpair_type iterpair_type; | |
365 | typedef HasState has_state; | |
366 | typedef boost::optional<Attribute> token_value_type; | |
367 | ||
368 | // default constructed tokens correspond to EOI tokens | |
369 | position_token() {} | |
370 | ||
371 | // construct an invalid token | |
372 | explicit position_token(int) | |
373 | : base_type(0) {} | |
374 | ||
375 | position_token(id_type id, std::size_t state) | |
376 | : base_type(id, state) {} | |
377 | ||
378 | position_token(id_type id, std::size_t state, token_value_type const& v) | |
379 | : base_type(id, state, unused), value_(v) {} | |
380 | ||
381 | position_token(id_type id, std::size_t state | |
382 | , Iterator const& first, Iterator const& last) | |
383 | : base_type(id, state, first, last) {} | |
384 | ||
385 | token_value_type& value() { return value_; } | |
386 | token_value_type const& value() const { return value_; } | |
387 | ||
388 | bool has_value() const { return !!value_; } | |
389 | ||
390 | #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) | |
391 | // workaround for MSVC10 which has problems copying a default | |
392 | // constructed iterator_range | |
393 | position_token& operator= (position_token const& rhs) | |
394 | { | |
395 | if (this != &rhs) | |
396 | { | |
397 | this->base_type::operator=(static_cast<base_type const&>(rhs)); | |
398 | if (this->is_valid()) | |
399 | value_ = rhs.value_; | |
400 | } | |
401 | return *this; | |
402 | } | |
403 | #endif | |
404 | ||
405 | protected: | |
406 | token_value_type value_; // token value | |
407 | }; | |
408 | ||
409 | template <typename Iterator, typename Attribute, typename HasState | |
410 | , typename Idtype> | |
411 | struct position_token<Iterator, mpl::vector1<Attribute>, HasState, Idtype> | |
412 | : position_token<Iterator, lex::omit, HasState, Idtype> | |
413 | { | |
414 | private: | |
415 | typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; | |
416 | ||
417 | public: | |
418 | typedef typename base_type::id_type id_type; | |
419 | typedef typename base_type::iterator_type iterator_type; | |
420 | typedef typename base_type::iterpair_type iterpair_type; | |
421 | typedef HasState has_state; | |
422 | typedef boost::optional<Attribute> token_value_type; | |
423 | ||
424 | // default constructed tokens correspond to EOI tokens | |
425 | position_token() {} | |
426 | ||
427 | // construct an invalid token | |
428 | explicit position_token(int) | |
429 | : base_type(0) {} | |
430 | ||
431 | position_token(id_type id, std::size_t state) | |
432 | : base_type(id, state) {} | |
433 | ||
434 | position_token(id_type id, std::size_t state, token_value_type const& v) | |
435 | : base_type(id, state, unused), value_(v) {} | |
436 | ||
437 | position_token(id_type id, std::size_t state | |
438 | , Iterator const& first, Iterator const& last) | |
439 | : base_type(id, state, first, last) {} | |
440 | ||
441 | token_value_type& value() { return value_; } | |
442 | token_value_type const& value() const { return value_; } | |
443 | ||
444 | bool has_value() const { return value_; } | |
445 | ||
446 | #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) | |
447 | // workaround for MSVC10 which has problems copying a default | |
448 | // constructed iterator_range | |
449 | position_token& operator= (position_token const& rhs) | |
450 | { | |
451 | if (this != &rhs) | |
452 | { | |
453 | this->base_type::operator=(static_cast<base_type const&>(rhs)); | |
454 | if (this->is_valid()) | |
455 | value_ = rhs.value_; | |
456 | } | |
457 | return *this; | |
458 | } | |
459 | #endif | |
460 | ||
461 | protected: | |
462 | token_value_type value_; // token value | |
463 | }; | |
464 | ||
465 | /////////////////////////////////////////////////////////////////////////// | |
466 | // The generic version of the position_token type derives from the | |
467 | // specialization above and adds a single data member holding the item | |
468 | // data carried by the token instance. | |
469 | /////////////////////////////////////////////////////////////////////////// | |
470 | namespace detail | |
471 | { | |
472 | /////////////////////////////////////////////////////////////////////// | |
473 | // Meta-function to calculate the type of the variant data item to be | |
474 | // stored with each token instance. | |
475 | // | |
476 | // Note: The iterator pair needs to be the first type in the list of | |
477 | // types supported by the generated variant type (this is being | |
478 | // used to identify whether the stored data item in a particular | |
479 | // token instance needs to be converted from the pair of | |
480 | // iterators (see the first of the assign_to_attribute_from_value | |
481 | // specializations below). | |
482 | /////////////////////////////////////////////////////////////////////// | |
483 | template <typename IteratorPair, typename AttributeTypes> | |
484 | struct position_token_value_typesequence | |
485 | { | |
486 | typedef typename mpl::insert< | |
487 | AttributeTypes | |
488 | , typename mpl::begin<AttributeTypes>::type | |
489 | , IteratorPair | |
490 | >::type sequence_type; | |
491 | typedef typename make_variant_over<sequence_type>::type type; | |
492 | }; | |
493 | ||
494 | /////////////////////////////////////////////////////////////////////// | |
495 | // The type of the data item stored with a token instance is defined | |
496 | // by the template parameter 'AttributeTypes' and may be: | |
497 | // | |
498 | // lex::omit: no data item is stored with the token | |
499 | // instance (this is handled by the | |
500 | // specializations of the token class | |
501 | // below) | |
502 | // mpl::vector0<>: each token instance stores a pair of | |
503 | // iterators pointing to the matched input | |
504 | // sequence | |
505 | // mpl::vector<...>: each token instance stores a variant being | |
506 | // able to store the pair of iterators pointing | |
507 | // to the matched input sequence, or any of the | |
508 | // types a specified in the mpl::vector<> | |
509 | // | |
510 | // All this is done to ensure the token type is as small (in terms | |
511 | // of its byte-size) as possible. | |
512 | /////////////////////////////////////////////////////////////////////// | |
513 | template <typename IteratorPair, typename AttributeTypes> | |
514 | struct position_token_value | |
515 | : mpl::eval_if< | |
516 | mpl::or_< | |
517 | is_same<AttributeTypes, mpl::vector0<> > | |
518 | , is_same<AttributeTypes, mpl::vector<> > > | |
519 | , mpl::identity<IteratorPair> | |
520 | , position_token_value_typesequence<IteratorPair, AttributeTypes> > | |
521 | {}; | |
522 | } | |
523 | ||
524 | template <typename Iterator, typename AttributeTypes, typename HasState | |
525 | , typename Idtype> | |
526 | struct position_token | |
527 | : position_token<Iterator, lex::omit, HasState, Idtype> | |
528 | { | |
529 | private: // precondition assertions | |
530 | BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value || | |
531 | is_same<AttributeTypes, lex::omit>::value)); | |
532 | typedef position_token<Iterator, lex::omit, HasState, Idtype> | |
533 | base_type; | |
534 | ||
535 | protected: | |
536 | // If no additional token value types are given, the token will | |
537 | // hold no token value at all as the base class already has the | |
538 | // iterator pair of the matched range in the underlying input sequence. | |
539 | // Otherwise the token value is stored as a variant and will | |
540 | // initially hold an unused_type but is able to hold any of | |
541 | // the given data types as well. The conversion from the iterator pair | |
542 | // to the required data type is done when it is accessed for the first | |
543 | // time. | |
544 | typedef iterator_range<Iterator> iterpair_type; | |
545 | ||
546 | public: | |
547 | typedef typename base_type::id_type id_type; | |
548 | typedef typename detail::position_token_value< | |
549 | iterpair_type, AttributeTypes>::type token_value_type; | |
550 | ||
551 | typedef Iterator iterator_type; | |
552 | ||
553 | // default constructed tokens correspond to EOI tokens | |
554 | position_token() {} | |
555 | ||
556 | // construct an invalid token | |
557 | explicit position_token(int) | |
558 | : base_type(0) {} | |
559 | ||
560 | position_token(id_type id, std::size_t state, token_value_type const& value) | |
561 | : base_type(id, state, value), value_(value) {} | |
562 | ||
563 | position_token(id_type id, std::size_t state, Iterator const& first | |
564 | , Iterator const& last) | |
565 | : base_type(id, state, first, last) | |
566 | , value_(iterpair_type(first, last)) | |
567 | {} | |
568 | ||
569 | token_value_type& value() { return value_; } | |
570 | token_value_type const& value() const { return value_; } | |
571 | ||
572 | #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) | |
573 | // workaround for MSVC10 which has problems copying a default | |
574 | // constructed iterator_range | |
575 | position_token& operator= (position_token const& rhs) | |
576 | { | |
577 | if (this != &rhs) | |
578 | { | |
579 | this->base_type::operator=(static_cast<base_type const&>(rhs)); | |
580 | if (this->is_valid()) | |
581 | value_ = rhs.value_; | |
582 | } | |
583 | return *this; | |
584 | } | |
585 | #endif | |
586 | ||
587 | protected: | |
588 | token_value_type value_; // token value, by default a pair of iterators | |
589 | }; | |
590 | ||
591 | /////////////////////////////////////////////////////////////////////////// | |
592 | // tokens are considered equal, if their id's match (these are unique) | |
593 | template <typename Iterator, typename AttributeTypes, typename HasState | |
594 | , typename Idtype> | |
595 | inline bool | |
596 | operator== (position_token<Iterator, AttributeTypes, HasState, Idtype> const& lhs, | |
597 | position_token<Iterator, AttributeTypes, HasState, Idtype> const& rhs) | |
598 | { | |
599 | return lhs.id() == rhs.id(); | |
600 | } | |
601 | ||
602 | /////////////////////////////////////////////////////////////////////////// | |
603 | // This overload is needed by the multi_pass/functor_input_policy to | |
604 | // validate a token instance. It has to be defined in the same namespace | |
605 | // as the token class itself to allow ADL to find it. | |
606 | /////////////////////////////////////////////////////////////////////////// | |
607 | template <typename Iterator, typename AttributeTypes, typename HasState | |
608 | , typename Idtype> | |
609 | inline bool | |
610 | token_is_valid(position_token<Iterator, AttributeTypes, HasState, Idtype> const& t) | |
611 | { | |
612 | return t.is_valid(); | |
613 | } | |
614 | }}}} | |
615 | ||
616 | namespace boost { namespace spirit { namespace traits | |
617 | { | |
618 | /////////////////////////////////////////////////////////////////////////// | |
619 | // We have to provide specializations for the customization point | |
620 | // assign_to_attribute_from_value allowing to extract the needed value | |
621 | // from the token. | |
622 | /////////////////////////////////////////////////////////////////////////// | |
623 | ||
624 | // This is called from the parse function of token_def if the token_def | |
625 | // has been defined to carry a special attribute type | |
626 | template <typename Attribute, typename Iterator, typename AttributeTypes | |
627 | , typename HasState, typename Idtype> | |
628 | struct assign_to_attribute_from_value<Attribute | |
629 | , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > | |
630 | { | |
631 | static void | |
632 | call(lex::lexertl::position_token< | |
633 | Iterator, AttributeTypes, HasState, Idtype> const& t | |
634 | , Attribute& attr) | |
635 | { | |
636 | // The goal of this function is to avoid the conversion of the pair of | |
637 | // iterators (to the matched character sequence) into the token value | |
638 | // of the required type being done more than once. For this purpose it | |
639 | // checks whether the stored value type is still the default one (pair | |
640 | // of iterators) and if yes, replaces the pair of iterators with the | |
641 | // converted value to be returned from subsequent calls. | |
642 | ||
643 | if (0 == t.value().which()) { | |
644 | // first access to the token value | |
645 | typedef iterator_range<Iterator> iterpair_type; | |
646 | iterpair_type const& ip = t.matched(); | |
647 | ||
648 | // Interestingly enough we use the assign_to() framework defined in | |
649 | // Spirit.Qi allowing to convert the pair of iterators to almost any | |
650 | // required type (assign_to(), if available, uses the standard Spirit | |
651 | // parsers to do the conversion). | |
652 | spirit::traits::assign_to(ip.begin(), ip.end(), attr); | |
653 | ||
654 | // If you get an error during the compilation of the following | |
655 | // assignment expression, you probably forgot to list one or more | |
656 | // types used as token value types (in your token_def<...> | |
657 | // definitions) in your definition of the token class. I.e. any token | |
658 | // value type used for a token_def<...> definition has to be listed | |
659 | // during the declaration of the token type to use. For instance let's | |
660 | // assume we have two token_def's: | |
661 | // | |
662 | // token_def<int> number; number = "..."; | |
663 | // token_def<std::string> identifier; identifier = "..."; | |
664 | // | |
665 | // Then you'll have to use the following token type definition | |
666 | // (assuming you are using the token class): | |
667 | // | |
668 | // typedef mpl::vector<int, std::string> token_values; | |
669 | // typedef token<base_iter_type, token_values> token_type; | |
670 | // | |
671 | // where: base_iter_type is the iterator type used to expose the | |
672 | // underlying input stream. | |
673 | // | |
674 | // This token_type has to be used as the second template parameter | |
675 | // to the lexer class: | |
676 | // | |
677 | // typedef lexer<base_iter_type, token_type> lexer_type; | |
678 | // | |
679 | // again, assuming you're using the lexer<> template for your | |
680 | // tokenization. | |
681 | ||
682 | typedef lex::lexertl::position_token< | |
683 | Iterator, AttributeTypes, HasState, Idtype> token_type; | |
684 | spirit::traits::assign_to( | |
685 | attr, const_cast<token_type&>(t).value()); // re-assign value | |
686 | } | |
687 | else { | |
688 | // reuse the already assigned value | |
689 | spirit::traits::assign_to(get<Attribute>(t.value()), attr); | |
690 | } | |
691 | } | |
692 | }; | |
693 | ||
694 | template <typename Attribute, typename Iterator, typename AttributeTypes | |
695 | , typename HasState, typename Idtype> | |
696 | struct assign_to_container_from_value<Attribute | |
697 | , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > | |
698 | : assign_to_attribute_from_value<Attribute | |
699 | , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > | |
700 | {}; | |
701 | ||
702 | /////////////////////////////////////////////////////////////////////////// | |
703 | // These are called from the parse function of token_def if the token type | |
704 | // has no special attribute type assigned | |
705 | template <typename Attribute, typename Iterator, typename HasState | |
706 | , typename Idtype> | |
707 | struct assign_to_attribute_from_value<Attribute | |
708 | , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> > | |
709 | { | |
710 | static void | |
711 | call(lex::lexertl::position_token< | |
712 | Iterator, mpl::vector0<>, HasState, Idtype> const& t | |
713 | , Attribute& attr) | |
714 | { | |
715 | // The default type returned by the token_def parser component (if | |
716 | // it has no token value type assigned) is the pair of iterators | |
717 | // to the matched character sequence. | |
718 | spirit::traits::assign_to(t.begin(), t.end(), attr); | |
719 | } | |
720 | }; | |
721 | ||
722 | // template <typename Attribute, typename Iterator, typename HasState | |
723 | // , typename Idtype> | |
724 | // struct assign_to_container_from_value<Attribute | |
725 | // , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> > | |
726 | // : assign_to_attribute_from_value<Attribute | |
727 | // , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> > | |
728 | // {}; | |
729 | ||
730 | // same as above but using mpl::vector<> instead of mpl::vector0<> | |
731 | template <typename Attribute, typename Iterator, typename HasState | |
732 | , typename Idtype> | |
733 | struct assign_to_attribute_from_value<Attribute | |
734 | , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> > | |
735 | { | |
736 | static void | |
737 | call(lex::lexertl::position_token< | |
738 | Iterator, mpl::vector<>, HasState, Idtype> const& t | |
739 | , Attribute& attr) | |
740 | { | |
741 | // The default type returned by the token_def parser component (if | |
742 | // it has no token value type assigned) is the pair of iterators | |
743 | // to the matched character sequence. | |
744 | spirit::traits::assign_to(t.begin(), t.end(), attr); | |
745 | } | |
746 | }; | |
747 | ||
748 | // template <typename Attribute, typename Iterator, typename HasState | |
749 | // , typename Idtype> | |
750 | // struct assign_to_container_from_value<Attribute | |
751 | // , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> > | |
752 | // : assign_to_attribute_from_value<Attribute | |
753 | // , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> > | |
754 | // {}; | |
755 | ||
756 | /////////////////////////////////////////////////////////////////////////// | |
757 | // These are called from the parse function of token_def if the token type | |
758 | // has no special attribute type assigned | |
759 | template <typename Attribute, typename Iterator, typename Attr | |
760 | , typename HasState, typename Idtype> | |
761 | struct assign_to_attribute_from_value<Attribute | |
762 | , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> > | |
763 | { | |
764 | static void | |
765 | call(lex::lexertl::position_token< | |
766 | Iterator, mpl::vector1<Attr>, HasState, Idtype> const& t | |
767 | , Attribute& attr) | |
768 | { | |
769 | // The goal of this function is to avoid the conversion of the pair of | |
770 | // iterators (to the matched character sequence) into the token value | |
771 | // of the required type being done more than once. | |
772 | ||
773 | if (!t.has_value()) { | |
774 | // first access to the token value | |
775 | typedef iterator_range<Iterator> iterpair_type; | |
776 | iterpair_type const& ip = t.matched(); | |
777 | ||
778 | // Interestingly enough we use the assign_to() framework defined in | |
779 | // Spirit.Qi allowing to convert the pair of iterators to almost any | |
780 | // required type (assign_to(), if available, uses the standard Spirit | |
781 | // parsers to do the conversion). | |
782 | spirit::traits::assign_to(ip.begin(), ip.end(), attr); | |
783 | ||
784 | // Re-assign the attribute to the stored value | |
785 | typedef lex::lexertl::position_token< | |
786 | Iterator, mpl::vector1<Attr>, HasState, Idtype> token_type; | |
787 | spirit::traits::assign_to( | |
788 | attr, const_cast<token_type&>(t).value()); | |
789 | } | |
790 | else { | |
791 | // reuse the already assigned value | |
792 | spirit::traits::assign_to(t.value(), attr); | |
793 | } | |
794 | } | |
795 | }; | |
796 | ||
797 | // template <typename Attribute, typename Iterator, typename Attr | |
798 | // , typename HasState, typename Idtype> | |
799 | // struct assign_to_container_from_value<Attribute | |
800 | // , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> > | |
801 | // : assign_to_attribute_from_value<Attribute | |
802 | // , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> > | |
803 | // {}; | |
804 | ||
805 | // same as above but using mpl::vector<Attr> instead of mpl::vector1<Attr> | |
806 | template <typename Attribute, typename Iterator, typename Attr | |
807 | , typename HasState, typename Idtype> | |
808 | struct assign_to_attribute_from_value<Attribute | |
809 | , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> > | |
810 | { | |
811 | static void | |
812 | call(lex::lexertl::position_token< | |
813 | Iterator, mpl::vector<Attr>, HasState, Idtype> const& t | |
814 | , Attribute& attr) | |
815 | { | |
816 | // The goal of this function is to avoid the conversion of the pair of | |
817 | // iterators (to the matched character sequence) into the token value | |
818 | // of the required type being done more than once. | |
819 | ||
820 | if (!t.has_value()) { | |
821 | // first access to the token value | |
822 | typedef iterator_range<Iterator> iterpair_type; | |
823 | iterpair_type const& ip = t.matched(); | |
824 | ||
825 | // Interestingly enough we use the assign_to() framework defined in | |
826 | // Spirit.Qi allowing to convert the pair of iterators to almost any | |
827 | // required type (assign_to(), if available, uses the standard Spirit | |
828 | // parsers to do the conversion). | |
829 | spirit::traits::assign_to(ip.begin(), ip.end(), attr); | |
830 | ||
831 | // Re-assign the attribute to the stored value | |
832 | typedef lex::lexertl::position_token< | |
833 | Iterator, mpl::vector<Attr>, HasState, Idtype> token_type; | |
834 | spirit::traits::assign_to( | |
835 | attr, const_cast<token_type&>(t).value()); | |
836 | } | |
837 | else { | |
838 | // reuse the already assigned value | |
839 | spirit::traits::assign_to(t.value(), attr); | |
840 | } | |
841 | } | |
842 | }; | |
843 | ||
844 | // template <typename Attribute, typename Iterator, typename Attr | |
845 | // , typename HasState, typename Idtype> | |
846 | // struct assign_to_container_from_value<Attribute | |
847 | // , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> > | |
848 | // : assign_to_attribute_from_value<Attribute | |
849 | // , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> > | |
850 | // {}; | |
851 | ||
852 | // This is called from the parse function of token_def if the token type | |
853 | // has been explicitly omitted (i.e. no attribute value is used), which | |
854 | // essentially means that every attribute gets initialized using default | |
855 | // constructed values. | |
856 | template <typename Attribute, typename Iterator, typename HasState | |
857 | , typename Idtype> | |
858 | struct assign_to_attribute_from_value<Attribute | |
859 | , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> > | |
860 | { | |
861 | static void | |
862 | call(lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> const& t | |
863 | , Attribute& attr) | |
864 | { | |
865 | // do nothing | |
866 | } | |
867 | }; | |
868 | ||
869 | template <typename Attribute, typename Iterator, typename HasState | |
870 | , typename Idtype> | |
871 | struct assign_to_container_from_value<Attribute | |
872 | , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> > | |
873 | : assign_to_attribute_from_value<Attribute | |
874 | , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> > | |
875 | {}; | |
876 | ||
877 | // This is called from the parse function of lexer_def_ | |
878 | template <typename Iterator, typename AttributeTypes, typename HasState | |
879 | , typename Idtype_, typename Idtype> | |
880 | struct assign_to_attribute_from_value< | |
881 | fusion::vector2<Idtype_, iterator_range<Iterator> > | |
882 | , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > | |
883 | { | |
884 | static void | |
885 | call(lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> const& t | |
886 | , fusion::vector2<Idtype_, iterator_range<Iterator> >& attr) | |
887 | { | |
888 | // The type returned by the lexer_def_ parser components is a | |
889 | // fusion::vector containing the token id of the matched token | |
890 | // and the pair of iterators to the matched character sequence. | |
891 | typedef iterator_range<Iterator> iterpair_type; | |
892 | typedef fusion::vector2<Idtype_, iterator_range<Iterator> > | |
893 | attribute_type; | |
894 | ||
895 | iterpair_type const& ip = t.matched(); | |
896 | attr = attribute_type(t.id(), ip); | |
897 | } | |
898 | }; | |
899 | ||
900 | template <typename Iterator, typename AttributeTypes, typename HasState | |
901 | , typename Idtype_, typename Idtype> | |
902 | struct assign_to_container_from_value< | |
903 | fusion::vector2<Idtype_, iterator_range<Iterator> > | |
904 | , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > | |
905 | : assign_to_attribute_from_value< | |
906 | fusion::vector2<Idtype_, iterator_range<Iterator> > | |
907 | , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > | |
908 | {}; | |
909 | ||
910 | /////////////////////////////////////////////////////////////////////////// | |
911 | // Overload debug output for a single token, this integrates lexer tokens | |
912 | // with Qi's simple_trace debug facilities | |
913 | template <typename Iterator, typename Attribute, typename HasState | |
914 | , typename Idtype> | |
915 | struct token_printer_debug< | |
916 | lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> > | |
917 | { | |
918 | typedef lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> token_type; | |
919 | ||
920 | template <typename Out> | |
921 | static void print(Out& out, token_type const& val) | |
922 | { | |
923 | out << '['; | |
924 | spirit::traits::print_token(out, val.value()); | |
925 | out << ']'; | |
926 | } | |
927 | }; | |
928 | }}} | |
929 | ||
930 | #endif |