1 /*=============================================================================
2 Boost.Wave: A Standard compliant C++ preprocessor library
8 Copyright (c) 2001-2010 Hartmut Kaiser. Distributed under the Boost
9 Software License, Version 1.0. (See accompanying file
10 LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
11 =============================================================================*/
13 #if !defined(IDL_RE2C_LEXER_HPP_B81A2629_D5B1_4944_A97D_60254182B9A8_INCLUDED)
14 #define IDL_RE2C_LEXER_HPP_B81A2629_D5B1_4944_A97D_60254182B9A8_INCLUDED
19 #if defined(BOOST_SPIRIT_DEBUG)
21 #endif // defined(BOOST_SPIRIT_DEBUG)
23 #include <boost/concept_check.hpp>
24 #include <boost/assert.hpp>
25 #include <boost/spirit/include/classic_core.hpp>
27 #include <boost/wave/token_ids.hpp>
28 #include <boost/wave/language_support.hpp>
29 #include <boost/wave/util/file_position.hpp>
30 #include <boost/wave/cpplexer/validate_universal_char.hpp>
31 #include <boost/wave/cpplexer/cpplexer_exceptions.hpp>
33 // reuse the default token type and re2c lexer helpers
34 #include <boost/wave/cpplexer/cpp_lex_token.hpp>
35 #include <boost/wave/cpplexer/cpp_lex_interface.hpp>
36 #include <boost/wave/cpplexer/re2clex/scanner.hpp>
40 ///////////////////////////////////////////////////////////////////////////////
46 ///////////////////////////////////////////////////////////////////////////////
48 // encapsulation of the re2c based idl lexer
50 ///////////////////////////////////////////////////////////////////////////////
54 typename PositionT = boost::wave::util::file_position_type
58 typedef boost::wave::cpplexer::re2clex::Scanner scanner_t;
63 typedef boost::wave::cpplexer::re2clex::Scanner base_t;
64 typedef boost::wave::cpplexer::lex_token<PositionT> token_type;
65 typedef typename token_type::string_type string_type;
67 lexer(IteratorT const &first, IteratorT const &last,
68 PositionT const &pos, boost::wave::language_support language);
71 token_type& get(token_type& t);
72 void set_position(PositionT const &pos)
74 // set position has to change the file name and line number only
75 filename = pos.get_file();
76 scanner.line = pos.get_line();
77 scanner.file_name = filename.c_str();
80 // error reporting from the re2c generated lexer
81 static int report_error(scanner_t const *s, int code, char const *, ...);
84 static char const *tok_names[];
89 boost::wave::language_support language;
92 ///////////////////////////////////////////////////////////////////////////////
93 // initialize cpp lexer
94 template <typename IteratorT, typename PositionT>
96 lexer<IteratorT, PositionT>::lexer(IteratorT const &first,
97 IteratorT const &last, PositionT const &pos,
98 boost::wave::language_support language)
99 : filename(pos.get_file()), at_eof(false), language(language)
101 using namespace std; // some systems have memset in std
102 using namespace boost::wave::cpplexer::re2clex;
104 memset(&scanner, '\0', sizeof(scanner_t));
105 scanner.eol_offsets = aq_create();
106 scanner.first = scanner.act = (uchar *)&(*first);
107 scanner.last = scanner.first + std::distance(first, last);
108 scanner.line = pos.get_line();
109 scanner.error_proc = report_error;
110 scanner.file_name = filename.c_str();
112 // not used by the lexer
113 scanner.enable_ms_extensions = 0;
114 scanner.act_in_c99_mode = 0;
116 boost::ignore_unused_variable_warning(language);
119 template <typename IteratorT, typename PositionT>
121 lexer<IteratorT, PositionT>::~lexer()
123 boost::wave::cpplexer::re2clex::aq_terminate(scanner.eol_offsets);
127 ///////////////////////////////////////////////////////////////////////////////
128 // get the next token from the input stream
129 template <typename IteratorT, typename PositionT>
130 inline boost::wave::cpplexer::lex_token<PositionT>&
131 lexer<IteratorT, PositionT>::get(boost::wave::cpplexer::lex_token<PositionT>& t)
133 using namespace boost::wave; // to import token ids to this scope
136 return t = boost::wave::cpplexer::lex_token<PositionT>(); // return T_EOI
138 token_id id = token_id(scan(&scanner));
139 string_type value((char const *)scanner.tok, scanner.cur-scanner.tok);
141 if (T_IDENTIFIER == id) {
142 // test identifier characters for validity (throws if invalid chars found)
143 if (!boost::wave::need_no_character_validation(language)) {
144 boost::wave::cpplexer::impl::validate_identifier_name(value,
145 scanner.line, -1, filename);
148 else if (T_STRINGLIT == id || T_CHARLIT == id) {
149 // test literal characters for validity (throws if invalid chars found)
150 if (!boost::wave::need_no_character_validation(language)) {
151 boost::wave::cpplexer::impl::validate_literal(value, scanner.line,
155 else if (T_EOF == id) {
156 // T_EOF is returned as a valid token, the next call will return T_EOI,
157 // i.e. the actual end of input
161 return t = boost::wave::cpplexer::lex_token<PositionT>(id, value,
162 PositionT(filename, scanner.line, -1));
165 template <typename IteratorT, typename PositionT>
167 lexer<IteratorT, PositionT>::report_error(scanner_t const *s, int errcode,
168 char const* msg, ...)
170 BOOST_ASSERT(0 != s);
171 BOOST_ASSERT(0 != msg);
173 using namespace std; // some system have vsprintf in namespace std
175 char buffer[200]; // should be large enough
177 va_start(params, msg);
178 vsprintf(buffer, msg, params);
181 BOOST_WAVE_LEXER_THROW_VAR(boost::wave::cpplexer::lexing_exception,
182 errcode, buffer, s->line, -1, s->file_name);
186 ///////////////////////////////////////////////////////////////////////////////
190 ///////////////////////////////////////////////////////////////////////////////
194 typename PositionT = boost::wave::util::file_position_type
197 : public lex_input_interface_generator<
198 typename lexer<IteratorT, PositionT>::token_type
203 typedef typename lexer<IteratorT, PositionT>::token_type token_type;
205 lex_functor(IteratorT const &first, IteratorT const &last,
206 PositionT const &pos, boost::wave::language_support language)
207 : lexer(first, last, pos, language)
209 virtual ~lex_functor() {}
211 // get the next token from the input stream
212 token_type& get(token_type& t) { return lexer.get(t); }
213 void set_position(PositionT const &pos)
214 { lexer.set_position(pos); }
215 #if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
216 bool has_include_guards(std::string&) const { return false; }
220 lexer<IteratorT, PositionT> lexer;
223 } // namespace re2clex
225 ///////////////////////////////////////////////////////////////////////////////
227 // The new_lexer_gen<>::new_lexer function (declared in cpp_slex_token.hpp)
228 // should be defined inline, if the lex_functor shouldn't be instantiated
229 // separately from the lex_iterator.
231 // Separate (explicit) instantiation helps to reduce compilation time.
233 ///////////////////////////////////////////////////////////////////////////////
235 #if BOOST_WAVE_SEPARATE_LEXER_INSTANTIATION != 0
236 #define BOOST_WAVE_RE2C_NEW_LEXER_INLINE
238 #define BOOST_WAVE_RE2C_NEW_LEXER_INLINE inline
241 ///////////////////////////////////////////////////////////////////////////////
243 // The 'new_lexer' function allows the opaque generation of a new lexer object.
244 // It is coupled to the iterator type to allow to decouple the lexer/iterator
245 // configurations at compile time.
247 // This function is declared inside the cpp_slex_token.hpp file, which is
248 // referenced by the source file calling the lexer and the source file, which
249 // instantiates the lex_functor. But it is defined here, so it will be
250 // instantiated only while compiling the source file, which instantiates the
251 // lex_functor. While the cpp_re2c_token.hpp file may be included everywhere,
252 // this file (cpp_re2c_lexer.hpp) should be included only once. This allows
253 // to decouple the lexer interface from the lexer implementation and reduces
256 ///////////////////////////////////////////////////////////////////////////////
258 template <typename IteratorT, typename PositionT>
259 BOOST_WAVE_RE2C_NEW_LEXER_INLINE
260 cpplexer::lex_input_interface<cpplexer::lex_token<PositionT> > *
261 new_lexer_gen<IteratorT, PositionT>::new_lexer(IteratorT const &first,
262 IteratorT const &last, PositionT const &pos,
263 wave::language_support language)
265 return new re2clex::lex_functor<IteratorT, PositionT>(first, last, pos,
269 #undef BOOST_WAVE_RE2C_NEW_LEXER_INLINE
271 ///////////////////////////////////////////////////////////////////////////////
272 } // namespace idllexer
276 #endif // !defined(IDL_RE2C_LEXER_HPP_B81A2629_D5B1_4944_A97D_60254182B9A8_INCLUDED)