1 // Copyright (c) 2001-2011 Hartmut Kaiser
3 // Distributed under the Boost Software License, Version 1.0. (See accompanying
4 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 #if !defined(BOOST_SPIRIT_LEX_LEXER_MAR_13_2007_0145PM)
7 #define BOOST_SPIRIT_LEX_LEXER_MAR_13_2007_0145PM
13 #include <boost/spirit/home/support/info.hpp>
14 #include <boost/spirit/home/qi/skip_over.hpp>
15 #include <boost/spirit/home/qi/parser.hpp>
16 #include <boost/spirit/home/qi/detail/assign_to.hpp>
17 #include <boost/spirit/home/lex/reference.hpp>
18 #include <boost/spirit/home/lex/meta_compiler.hpp>
19 #include <boost/spirit/home/lex/lexer_type.hpp>
20 #include <boost/spirit/home/lex/lexer/token_def.hpp>
21 #include <boost/assert.hpp>
22 #include <boost/noncopyable.hpp>
23 #include <boost/detail/iterator.hpp>
24 #include <boost/fusion/include/vector.hpp>
25 #include <boost/mpl/assert.hpp>
26 #include <boost/range/iterator_range.hpp>
29 namespace boost { namespace spirit { namespace lex
31 ///////////////////////////////////////////////////////////////////////////
34 ///////////////////////////////////////////////////////////////////////
35 template <typename LexerDef>
38 typename proto::terminal<
39 lex::reference<lexer_def_<LexerDef> const>
41 , lexer_def_<LexerDef> >
42 , qi::parser<lexer_def_<LexerDef> >
43 , lex::lexer_type<lexer_def_<LexerDef> >
46 // avoid warnings about using 'this' in constructor
47 lexer_def_& this_() { return *this; }
49 typedef typename LexerDef::char_type char_type;
50 typedef typename LexerDef::string_type string_type;
51 typedef typename LexerDef::id_type id_type;
53 typedef lex::reference<lexer_def_ const> reference_;
54 typedef typename proto::terminal<reference_>::type terminal_type;
55 typedef proto::extends<terminal_type, lexer_def_> proto_base_type;
57 reference_ alias() const
59 return reference_(*this);
63 // Qi interface: metafunction calculating parser attribute type
64 template <typename Context, typename Iterator>
67 // the return value of a token set contains the matched token
68 // id, and the corresponding pair of iterators
69 typedef typename Iterator::base_iterator_type iterator_type;
71 fusion::vector2<id_type, iterator_range<iterator_type> >
75 // Qi interface: parse functionality
76 template <typename Iterator, typename Context
77 , typename Skipper, typename Attribute>
78 bool parse(Iterator& first, Iterator const& last
79 , Context& /*context*/, Skipper const& skipper
80 , Attribute& attr) const
82 qi::skip_over(first, last, skipper); // always do a pre-skip
86 boost::detail::iterator_traits<Iterator>::value_type
89 token_type const& t = *first;
90 if (token_is_valid(t) && t.state() == first.get_state()) {
91 // any of the token definitions matched
92 spirit::traits::assign_to(t, attr);
100 // Qi interface: 'what' functionality
101 template <typename Context>
102 info what(Context& /*context*/) const
104 return info("lexer");
108 // allow to use the lexer.self.add("regex1", id1)("regex2", id2);
112 adder(lexer_def_& def_)
115 // Add a token definition based on a single character as given
116 // by the first parameter, the second parameter allows to
117 // specify the token id to use for the new token. If no token
118 // id is given the character code is used.
119 adder const& operator()(char_type c
120 , id_type token_id = id_type()) const
122 if (id_type() == token_id)
123 token_id = static_cast<id_type>(c);
124 def.def.add_token (def.state.c_str(), c, token_id
125 , def.targetstate.empty() ? 0 : def.targetstate.c_str());
129 // Add a token definition based on a character sequence as
130 // given by the first parameter, the second parameter allows to
131 // specify the token id to use for the new token. If no token
132 // id is given this function will generate a unique id to be
133 // used as the token's id.
134 adder const& operator()(string_type const& s
135 , id_type token_id = id_type()) const
137 if (id_type() == token_id)
138 token_id = def.def.get_next_id();
139 def.def.add_token (def.state.c_str(), s, token_id
140 , def.targetstate.empty() ? 0 : def.targetstate.c_str());
144 template <typename Attribute>
145 adder const& operator()(
146 token_def<Attribute, char_type, id_type>& tokdef
147 , id_type token_id = id_type()) const
149 // make sure we have a token id
150 if (id_type() == token_id) {
151 if (id_type() == tokdef.id()) {
152 token_id = def.def.get_next_id();
156 token_id = tokdef.id();
160 // the following assertion makes sure that the token_def
161 // instance has not been assigned a different id earlier
162 BOOST_ASSERT(id_type() == tokdef.id()
163 || token_id == tokdef.id());
171 // template <typename F>
172 // adder const& operator()(char_type c, id_type token_id, F act) const
174 // if (id_type() == token_id)
175 // token_id = def.def.get_next_id();
176 // std::size_t unique_id =
177 // def.def.add_token (def.state.c_str(), s, token_id);
178 // def.def.add_action(unique_id, def.state.c_str(), act);
185 // silence MSVC warning C4512: assignment operator could not be generated
186 adder& operator= (adder const&);
190 // allow to use lexer.self.add_pattern("pattern1", "regex1")(...);
194 pattern_adder(lexer_def_& def_)
197 pattern_adder const& operator()(string_type const& p
198 , string_type const& s) const
200 def.def.add_pattern (def.state.c_str(), p, s);
207 // silence MSVC warning C4512: assignment operator could not be generated
208 pattern_adder& operator= (pattern_adder const&);
210 friend struct pattern_adder;
213 // Helper function to invoke the necessary 2 step compilation
214 // process on token definition expressions
215 template <typename TokenExpr>
216 void compile2pass(TokenExpr const& expr)
218 expr.collect(def, state, targetstate);
219 expr.add_actions(def);
223 ///////////////////////////////////////////////////////////////////
224 template <typename Expr>
225 void define(Expr const& expr)
227 compile2pass(compile<lex::domain>(expr));
230 lexer_def_(LexerDef& def_, string_type const& state_
231 , string_type const& targetstate_ = string_type())
232 : proto_base_type(terminal_type::make(alias()))
233 , add(this_()), add_pattern(this_()), def(def_)
234 , state(state_), targetstate(targetstate_)
237 // allow to switch states
238 lexer_def_ operator()(char_type const* state) const
240 return lexer_def_(def, state);
242 lexer_def_ operator()(char_type const* state
243 , char_type const* targetstate) const
245 return lexer_def_(def, state, targetstate);
247 lexer_def_ operator()(string_type const& state
248 , string_type const& targetstate = string_type()) const
250 return lexer_def_(def, state, targetstate);
253 // allow to assign a token definition expression
254 template <typename Expr>
255 lexer_def_& operator= (Expr const& xpr)
257 // Report invalid expression error as early as possible.
258 // If you got an error_invalid_expression error message here,
259 // then the expression (expr) is not a valid spirit lex
261 BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
263 def.clear(state.c_str());
268 // explicitly tell the lexer that the given state will be defined
269 // (useful in conjunction with "*")
270 std::size_t add_state(char_type const* state = 0)
272 return def.add_state(state ? state : def.initial_state().c_str());
276 pattern_adder add_pattern;
281 string_type targetstate;
284 // silence MSVC warning C4512: assignment operator could not be generated
285 lexer_def_& operator= (lexer_def_ const&);
288 #if defined(BOOST_NO_CXX11_RVALUE_REFERENCES)
289 // allow to assign a token definition expression
290 template <typename LexerDef, typename Expr>
291 inline lexer_def_<LexerDef>&
292 operator+= (lexer_def_<LexerDef>& lexdef, Expr& xpr)
294 // Report invalid expression error as early as possible.
295 // If you got an error_invalid_expression error message here,
296 // then the expression (expr) is not a valid spirit lex
298 BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
304 // allow to assign a token definition expression
305 template <typename LexerDef, typename Expr>
306 inline lexer_def_<LexerDef>&
307 operator+= (lexer_def_<LexerDef>& lexdef, Expr&& xpr)
309 // Report invalid expression error as early as possible.
310 // If you got an error_invalid_expression error message here,
311 // then the expression (expr) is not a valid spirit lex
313 BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
320 template <typename LexerDef, typename Expr>
321 inline lexer_def_<LexerDef>&
322 operator+= (lexer_def_<LexerDef>& lexdef, Expr const& xpr)
324 // Report invalid expression error as early as possible.
325 // If you got an error_invalid_expression error message here,
326 // then the expression (expr) is not a valid spirit lex
328 BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
335 ///////////////////////////////////////////////////////////////////////////
336 // The match_flags flags are used to influence different matching
337 // modes of the lexer
342 match_default = 0, // no flags
343 match_not_dot_newline = 1, // the regex '.' doesn't match newlines
344 match_icase = 2 // all matching operations are case insensitive
348 ///////////////////////////////////////////////////////////////////////////
349 // This represents a lexer object
350 ///////////////////////////////////////////////////////////////////////////
352 ///////////////////////////////////////////////////////////////////////////
353 // This is the first token id automatically assigned by the library
357 min_token_id = 0x10000
360 template <typename Lexer>
361 class lexer : public Lexer
364 // avoid warnings about using 'this' in constructor
365 lexer& this_() { return *this; }
367 std::size_t next_token_id; // has to be an integral type
370 typedef Lexer lexer_type;
371 typedef typename Lexer::id_type id_type;
372 typedef typename Lexer::char_type char_type;
373 typedef typename Lexer::iterator_type iterator_type;
374 typedef lexer base_type;
376 typedef detail::lexer_def_<lexer> lexer_def;
377 typedef std::basic_string<char_type> string_type;
379 lexer(unsigned int flags = match_flags::match_default
380 , id_type first_id = id_type(min_token_id))
382 , next_token_id(first_id)
383 , self(this_(), lexer_type::initial_state())
386 // access iterator interface
387 template <typename Iterator>
388 iterator_type begin(Iterator& first, Iterator const& last
389 , char_type const* initial_state = 0) const
390 { return this->lexer_type::begin(first, last, initial_state); }
391 iterator_type end() const
392 { return this->lexer_type::end(); }
394 std::size_t map_state(char_type const* state)
395 { return this->lexer_type::add_state(state); }
397 // create a unique token id
398 id_type get_next_id() { return id_type(next_token_id++); }
400 lexer_def self; // allow for easy token definition