1 // Copyright (c) 2008-2009 Ben Hanson
2 // Copyright (c) 2008-2011 Hartmut Kaiser
4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7 #if !defined(BOOST_SPIRIT_LEX_LEXERTL_GENERATE_CPP_FEB_10_2008_0855PM)
8 #define BOOST_SPIRIT_LEX_LEXERTL_GENERATE_CPP_FEB_10_2008_0855PM
14 #include <boost/spirit/home/support/detail/lexer/char_traits.hpp>
15 #include <boost/spirit/home/support/detail/lexer/consts.hpp>
16 #include <boost/spirit/home/support/detail/lexer/rules.hpp>
17 #include <boost/spirit/home/support/detail/lexer/size_t.hpp>
18 #include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
19 #include <boost/spirit/home/support/detail/lexer/debug.hpp>
20 #include <boost/spirit/home/lex/lexer/lexertl/static_version.hpp>
21 #include <boost/scoped_array.hpp>
25 ///////////////////////////////////////////////////////////////////////////////
26 namespace boost { namespace spirit { namespace lex { namespace lexertl
31 ///////////////////////////////////////////////////////////////////////////
32 template <typename CharT>
36 struct string_lit<char>
38 static char get(char c) { return c; }
39 static std::string get(char const* str = "") { return str; }
43 struct string_lit<wchar_t>
45 static wchar_t get(char c)
47 typedef std::ctype<wchar_t> ctype_t;
48 return std::use_facet<ctype_t>(std::locale()).widen(c);
50 static std::basic_string<wchar_t> get(char const* source = "")
52 using namespace std; // some systems have size_t in ns std
53 size_t len = strlen(source);
54 boost::scoped_array<wchar_t> result (new wchar_t[len+1]);
55 result.get()[len] = '\0';
57 // working with wide character streams is supported only if the
58 // platform provides the std::ctype<wchar_t> facet
59 BOOST_ASSERT(std::has_facet<std::ctype<wchar_t> >(std::locale()));
61 std::use_facet<std::ctype<wchar_t> >(std::locale())
62 .widen(source, source + len, result.get());
67 template <typename Char>
70 return string_lit<Char>::get(c);
73 template <typename Char>
74 inline std::basic_string<Char> L(char const* c = "")
76 return string_lit<Char>::get(c);
79 ///////////////////////////////////////////////////////////////////////////
80 template <typename Char>
82 generate_delimiter(std::basic_ostream<Char> &os_)
84 os_ << std::basic_string<Char>(80, '/') << "\n";
88 ///////////////////////////////////////////////////////////////////////////
89 // Generate a table of the names of the used lexer states, which is a bit
90 // tricky, because the table stored with the rules is sorted based on the
91 // names, but we need it sorted using the state ids.
92 template <typename Char>
94 generate_cpp_state_info (boost::lexer::basic_rules<Char> const& rules_
95 , std::basic_ostream<Char> &os_, Char const* name_suffix)
97 // we need to re-sort the state names in ascending order of the state
98 // ids, filling possible gaps in between later
100 boost::lexer::basic_rules<Char>::string_size_t_map::const_iterator
102 typedef std::map<std::size_t, Char const*> reverse_state_map_type;
104 reverse_state_map_type reverse_state_map;
105 state_iterator send = rules_.statemap().end();
106 for (state_iterator sit = rules_.statemap().begin(); sit != send; ++sit)
108 typedef typename reverse_state_map_type::value_type value_type;
109 reverse_state_map.insert(value_type((*sit).second, (*sit).first.c_str()));
112 generate_delimiter(os_);
113 os_ << "// this table defines the names of the lexer states\n";
114 os_ << boost::lexer::detail::strings<Char>::char_name()
115 << " const* const lexer_state_names"
116 << (name_suffix[0] ? "_" : "") << name_suffix
117 << "[" << rules_.statemap().size() << "] = \n{\n";
119 typedef typename reverse_state_map_type::iterator iterator;
120 iterator rend = reverse_state_map.end();
121 std::size_t last_id = 0;
122 for (iterator rit = reverse_state_map.begin(); rit != rend; ++last_id)
124 for (/**/; last_id < (*rit).first; ++last_id)
126 os_ << " 0, // \"<undefined state>\"\n";
129 << boost::lexer::detail::strings<Char>::char_prefix()
130 << "\"" << (*rit).second << "\"";
134 os_ << "\n"; // don't generate the final comma
138 generate_delimiter(os_);
139 os_ << "// this variable defines the number of lexer states\n";
140 os_ << "std::size_t const lexer_state_count"
141 << (name_suffix[0] ? "_" : "") << name_suffix
142 << " = " << rules_.statemap().size() << ";\n\n";
146 template <typename Char>
148 generate_cpp_state_table (std::basic_ostream<Char> &os_
149 , Char const* name_suffix, bool bol, bool eol)
151 std::basic_string<Char> suffix(L<Char>(name_suffix[0] ? "_" : ""));
152 suffix += name_suffix;
154 generate_delimiter(os_);
155 os_ << "// this defines a generic accessors for the information above\n";
156 os_ << "struct lexer" << suffix << "\n{\n";
157 os_ << " // version number and feature-set of compatible static lexer engine\n";
159 os_ << " {\n static_version = " << SPIRIT_STATIC_LEXER_VERSION << ",\n";
160 os_ << " supports_bol = " << std::boolalpha << bol << ",\n";
161 os_ << " supports_eol = " << std::boolalpha << eol << "\n";
163 os_ << " // return the number of lexer states\n";
164 os_ << " static std::size_t state_count()\n";
165 os_ << " {\n return lexer_state_count" << suffix << "; \n }\n\n";
166 os_ << " // return the name of the lexer state as given by 'idx'\n";
167 os_ << " static " << boost::lexer::detail::strings<Char>::char_name()
168 << " const* state_name(std::size_t idx)\n";
169 os_ << " {\n return lexer_state_names" << suffix << "[idx]; \n }\n\n";
170 os_ << " // return the next matched token\n";
171 os_ << " template<typename Iterator>\n";
172 os_ << " static std::size_t next(std::size_t &start_state_, bool& bol_\n";
173 os_ << " , Iterator &start_token_, Iterator const& end_, std::size_t& unique_id_)\n";
174 os_ << " {\n return next_token" << suffix
175 << "(start_state_, bol_, start_token_, end_, unique_id_);\n }\n";
180 ///////////////////////////////////////////////////////////////////////////
181 // generate function body based on traversing the DFA tables
182 template <typename Char>
183 bool generate_function_body_dfa(std::basic_ostream<Char>& os_
184 , boost::lexer::basic_state_machine<Char> const &sm_)
186 std::size_t const dfas_ = sm_.data()._dfa->size();
187 std::size_t const lookups_ = sm_.data()._lookup->front()->size();
189 os_ << " enum {end_state_index, id_index, unique_id_index, "
190 "state_index, bol_index,\n";
191 os_ << " eol_index, dead_state_index, dfa_offset};\n\n";
192 os_ << " static std::size_t const npos = "
193 "static_cast<std::size_t>(~0);\n";
197 for (std::size_t state_ = 0; state_ < dfas_; ++state_)
201 std::size_t count_ = lookups_ / 8;
202 std::size_t const* lookup_ = &sm_.data()._lookup[state_]->front();
203 std::size_t const* dfa_ = &sm_.data()._dfa[state_]->front();
205 os_ << " static std::size_t const lookup" << state_
206 << "_[" << lookups_ << "] = {\n ";
207 for (/**/; i_ < count_; ++i_)
209 std::size_t const index_ = i_ * 8;
210 os_ << lookup_[index_];
211 for (/**/; j_ < 8; ++j_)
213 os_ << ", " << lookup_[index_ + j_];
223 count_ = sm_.data()._dfa[state_]->size ();
224 os_ << " static const std::size_t dfa" << state_ << "_["
225 << count_ << "] = {\n ";
227 for (i_ = 0; i_ < count_; ++i_)
229 std::size_t const index_ = i_ * 8;
231 for (j_ = 1; j_ < 8; ++j_)
233 os_ << ", " << dfa_[index_ + j_];
241 std::size_t const mod_ = sm_.data()._dfa[state_]->size () % 8;
244 std::size_t const index_ = count_ * 8;
250 for (j_ = 1; j_ < mod_; ++j_)
252 os_ << ", " << dfa_[index_ + j_];
258 std::size_t count_ = sm_.data()._dfa_alphabet.size();
261 os_ << " static std::size_t const* lookup_arr_[" << count_
263 for (i_ = 1; i_ < count_; ++i_)
265 os_ << ", " << "lookup" << i_ << "_";
269 os_ << " static std::size_t const dfa_alphabet_arr_["
270 << count_ << "] = { ";
271 os_ << sm_.data()._dfa_alphabet.front ();
272 for (i_ = 1; i_ < count_; ++i_)
274 os_ << ", " << sm_.data()._dfa_alphabet[i_];
278 os_ << " static std::size_t const* dfa_arr_[" << count_
281 for (i_ = 1; i_ < count_; ++i_)
283 os_ << ", " << "dfa" << i_ << "_";
289 std::size_t const* lookup_ = &sm_.data()._lookup[0]->front();
290 std::size_t const* dfa_ = &sm_.data()._dfa[0]->front();
293 std::size_t count_ = lookups_ / 8;
295 os_ << " static std::size_t const lookup_[";
296 os_ << sm_.data()._lookup[0]->size() << "] = {\n ";
297 for (/**/; i_ < count_; ++i_)
299 const std::size_t index_ = i_ * 8;
300 os_ << lookup_[index_];
301 for (/**/; j_ < 8; ++j_)
303 os_ << ", " << lookup_[index_ + j_];
313 os_ << " static std::size_t const dfa_alphabet_ = "
314 << sm_.data()._dfa_alphabet.front () << ";\n";
315 os_ << " static std::size_t const dfa_["
316 << sm_.data()._dfa[0]->size () << "] = {\n ";
317 count_ = sm_.data()._dfa[0]->size () / 8;
318 for (i_ = 0; i_ < count_; ++i_)
320 const std::size_t index_ = i_ * 8;
322 for (j_ = 1; j_ < 8; ++j_)
324 os_ << ", " << dfa_[index_ + j_];
332 const std::size_t mod_ = sm_.data()._dfa[0]->size () % 8;
335 const std::size_t index_ = count_ * 8;
341 for (j_ = 1; j_ < mod_; ++j_)
343 os_ << ", " << dfa_[index_ + j_];
349 os_ << "\n if (start_token_ == end_)\n";
351 os_ << " unique_id_ = npos;\n";
352 os_ << " return 0;\n";
354 if (sm_.data()._seen_BOL_assertion)
356 os_ << " bool bol = bol_;\n\n";
362 os_ << " std::size_t const* lookup_ = lookup_arr_[start_state_];\n";
363 os_ << " std::size_t dfa_alphabet_ = dfa_alphabet_arr_[start_state_];\n";
364 os_ << " std::size_t const*dfa_ = dfa_arr_[start_state_];\n";
367 os_ << " std::size_t const* ptr_ = dfa_ + dfa_alphabet_;\n";
368 os_ << " Iterator curr_ = start_token_;\n";
369 os_ << " bool end_state_ = *ptr_ != 0;\n";
370 os_ << " std::size_t id_ = *(ptr_ + id_index);\n";
371 os_ << " std::size_t uid_ = *(ptr_ + unique_id_index);\n";
374 os_ << " std::size_t end_start_state_ = start_state_;\n";
376 if (sm_.data()._seen_BOL_assertion)
378 os_ << " bool end_bol_ = bol_;\n";
380 os_ << " Iterator end_token_ = start_token_;\n\n";
382 os_ << " while (curr_ != end_)\n";
385 if (sm_.data()._seen_BOL_assertion)
387 os_ << " std::size_t const BOL_state_ = ptr_[bol_index];\n\n";
390 if (sm_.data()._seen_EOL_assertion)
392 os_ << " std::size_t const EOL_state_ = ptr_[eol_index];\n\n";
395 if (sm_.data()._seen_BOL_assertion && sm_.data()._seen_EOL_assertion)
397 os_ << " if (BOL_state_ && bol)\n";
399 os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
401 os_ << " else if (EOL_state_ && *curr_ == '\\n')\n";
403 os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
409 os_ << " unsigned char index = \n";
410 os_ << " static_cast<unsigned char>(*curr_++);\n";
414 os_ << " std::size_t index = *curr_++\n";
416 os_ << " bol = (index == '\\n') ? true : false;\n";
417 os_ << " std::size_t const state_ = ptr_[\n";
418 os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
421 os_ << " if (state_ == 0) break;\n";
423 os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
426 else if (sm_.data()._seen_BOL_assertion)
428 os_ << " if (BOL_state_ && bol)\n";
430 os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
436 os_ << " unsigned char index = \n";
437 os_ << " static_cast<unsigned char>(*curr_++);\n";
441 os_ << " std::size_t index = *curr_++\n";
443 os_ << " bol = (index == '\\n') ? true : false;\n";
444 os_ << " std::size_t const state_ = ptr_[\n";
445 os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
448 os_ << " if (state_ == 0) break;\n";
450 os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
453 else if (sm_.data()._seen_EOL_assertion)
455 os_ << " if (EOL_state_ && *curr_ == '\\n')\n";
457 os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
463 os_ << " unsigned char index = \n";
464 os_ << " static_cast<unsigned char>(*curr_++);\n";
468 os_ << " std::size_t index = *curr_++\n";
470 os_ << " bol = (index == '\\n') ? true : false;\n";
471 os_ << " std::size_t const state_ = ptr_[\n";
472 os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
475 os_ << " if (state_ == 0) break;\n";
477 os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
482 os_ << " std::size_t const state_ =\n";
486 os_ << " ptr_[lookup_["
487 "static_cast<unsigned char>(*curr_++)]];\n";
491 os_ << " ptr_[lookup_[*curr_++]];\n";
495 os_ << " if (state_ == 0) break;\n";
497 os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n\n";
500 os_ << " if (*ptr_)\n";
502 os_ << " end_state_ = true;\n";
503 os_ << " id_ = *(ptr_ + id_index);\n";
504 os_ << " uid_ = *(ptr_ + unique_id_index);\n";
507 os_ << " end_start_state_ = *(ptr_ + state_index);\n";
509 if (sm_.data()._seen_BOL_assertion)
511 os_ << " end_bol_ = bol;\n";
513 os_ << " end_token_ = curr_;\n";
517 if (sm_.data()._seen_EOL_assertion)
519 os_ << " std::size_t const EOL_state_ = ptr_[eol_index];\n\n";
521 os_ << " if (EOL_state_ && curr_ == end_)\n";
523 os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n\n";
525 os_ << " if (*ptr_)\n";
527 os_ << " end_state_ = true;\n";
528 os_ << " id_ = *(ptr_ + id_index);\n";
529 os_ << " uid_ = *(ptr_ + unique_id_index);\n";
532 os_ << " end_start_state_ = *(ptr_ + state_index);\n";
534 if (sm_.data()._seen_BOL_assertion)
536 os_ << " end_bol_ = bol;\n";
538 os_ << " end_token_ = curr_;\n";
543 os_ << " if (end_state_)\n";
545 os_ << " // return longest match\n";
546 os_ << " start_token_ = end_token_;\n";
550 os_ << " start_state_ = end_start_state_;\n";
551 os_ << " if (id_ == 0)\n";
553 if (sm_.data()._seen_BOL_assertion)
555 os_ << " bol = end_bol_;\n";
557 os_ << " goto again;\n";
559 if (sm_.data()._seen_BOL_assertion)
563 os_ << " bol_ = end_bol_;\n";
567 else if (sm_.data()._seen_BOL_assertion)
569 os_ << " bol_ = end_bol_;\n";
576 if (sm_.data()._seen_BOL_assertion)
578 os_ << " bol_ = (*start_token_ == '\\n') ? true : false;\n";
581 os_ << " id_ = npos;\n";
582 os_ << " uid_ = npos;\n";
585 os_ << " unique_id_ = uid_;\n";
586 os_ << " return id_;\n";
590 ///////////////////////////////////////////////////////////////////////////
591 template <typename Char>
592 inline std::basic_string<Char> get_charlit(Char ch)
594 std::basic_string<Char> result;
595 boost::lexer::basic_string_token<Char>::escape_char(ch, result);
599 // check whether state0_0 is referenced from any of the other states
600 template <typename Char>
601 bool need_label0_0(boost::lexer::basic_state_machine<Char> const &sm_)
603 typedef typename boost::lexer::basic_state_machine<Char>::iterator
605 iterator_type iter_ = sm_.begin();
606 std::size_t const states_ = iter_->states;
608 for (std::size_t state_ = 0; state_ < states_; ++state_)
610 if (0 == iter_->bol_index || 0 == iter_->eol_index)
615 std::size_t const transitions_ = iter_->transitions;
616 for (std::size_t t_ = 0; t_ < transitions_; ++t_)
618 if (0 == iter_->goto_state)
624 if (transitions_ == 0) ++iter_;
629 ///////////////////////////////////////////////////////////////////////////
630 template <typename Char>
631 bool generate_function_body_switch(std::basic_ostream<Char> & os_
632 , boost::lexer::basic_state_machine<Char> const &sm_)
634 typedef typename boost::lexer::basic_state_machine<Char>::iterator
637 std::size_t const lookups_ = sm_.data()._lookup->front ()->size ();
638 iterator_type iter_ = sm_.begin();
639 iterator_type labeliter_ = iter_;
640 iterator_type end_ = sm_.end();
641 std::size_t const dfas_ = sm_.data()._dfa->size ();
643 os_ << " static std::size_t const npos = "
644 "static_cast<std::size_t>(~0);\n";
646 os_ << "\n if (start_token_ == end_)\n";
648 os_ << " unique_id_ = npos;\n";
649 os_ << " return 0;\n";
652 if (sm_.data()._seen_BOL_assertion)
654 os_ << " bool bol = bol_;\n";
662 os_ << " Iterator curr_ = start_token_;\n";
663 os_ << " bool end_state_ = false;\n";
664 os_ << " std::size_t id_ = npos;\n";
665 os_ << " std::size_t uid_ = npos;\n";
669 os_ << " std::size_t end_start_state_ = start_state_;\n";
672 if (sm_.data()._seen_BOL_assertion)
674 os_ << " bool end_bol_ = bol_;\n";
677 os_ << " Iterator end_token_ = start_token_;\n";
680 os_ << " " << ((lookups_ == 256) ? "char" : "wchar_t")
685 os_ << " switch (start_state_)\n";
688 for (std::size_t i_ = 0; i_ < dfas_; ++i_)
690 os_ << " case " << i_ << ":\n";
691 os_ << " goto state" << i_ << "_0;\n";
695 os_ << " default:\n";
696 os_ << " goto end;\n";
701 bool need_state0_0_label = need_label0_0(sm_);
703 for (std::size_t dfa_ = 0; dfa_ < dfas_; ++dfa_)
705 std::size_t const states_ = iter_->states;
706 for (std::size_t state_ = 0; state_ < states_; ++state_)
708 std::size_t const transitions_ = iter_->transitions;
711 if (dfas_ > 1 || dfa_ != 0 || state_ != 0 || need_state0_0_label)
713 os_ << "\nstate" << dfa_ << '_' << state_ << ":\n";
716 if (iter_->end_state)
718 os_ << " end_state_ = true;\n";
719 os_ << " id_ = " << iter_->id << ";\n";
720 os_ << " uid_ = " << iter_->unique_id << ";\n";
721 os_ << " end_token_ = curr_;\n";
725 os_ << " end_start_state_ = " << iter_->goto_dfa <<
729 if (sm_.data()._seen_BOL_assertion)
731 os_ << " end_bol_ = bol;\n";
734 if (transitions_) os_ << '\n';
737 if (t_ < transitions_ ||
738 iter_->bol_index != boost::lexer::npos ||
739 iter_->eol_index != boost::lexer::npos)
741 os_ << " if (curr_ == end_) goto end;\n";
742 os_ << " ch_ = *curr_;\n";
743 if (iter_->bol_index != boost::lexer::npos)
745 os_ << "\n if (bol) goto state" << dfa_ << '_'
746 << iter_->bol_index << ";\n";
748 if (iter_->eol_index != boost::lexer::npos)
750 os_ << "\n if (ch_ == '\\n') goto state" << dfa_
751 << '_' << iter_->eol_index << ";\n";
753 os_ << " ++curr_;\n";
756 for (/**/; t_ < transitions_; ++t_)
758 Char const *ptr_ = iter_->token._charset.c_str();
759 Char const *end2_ = ptr_ + iter_->token._charset.size();
760 Char start_char_ = 0;
763 bool first_char_ = true;
767 while (ptr_ != end2_)
769 curr_char_ = *ptr_++;
771 if (*ptr_ == curr_char_ + 1)
775 start_char_ = curr_char_;
783 os_ << ((iter_->token._negated) ? " && " : " || ");
791 if (iter_->token._negated)
795 os_ << "(ch_ >= '" << get_charlit(start_char_)
797 << get_charlit(curr_char_) << "')";
803 << ((iter_->token._negated) ? "!=" : "==")
804 << " '" << get_charlit(curr_char_) << "'";
809 os_ << ") goto state" << dfa_ << '_' << iter_->goto_state
814 if (!(dfa_ == dfas_ - 1 && state_ == states_ - 1))
816 os_ << " goto end;\n";
819 if (transitions_ == 0) ++iter_;
824 os_ << " if (end_state_)\n";
826 os_ << " // return longest match\n";
827 os_ << " start_token_ = end_token_;\n";
831 os_ << " start_state_ = end_start_state_;\n";
832 os_ << "\n if (id_ == 0)\n";
835 if (sm_.data()._seen_BOL_assertion)
837 os_ << " bol = end_bol_;\n";
840 os_ << " goto again;\n";
843 if (sm_.data()._seen_BOL_assertion)
847 os_ << " bol_ = end_bol_;\n";
851 else if (sm_.data()._seen_BOL_assertion)
853 os_ << " bol_ = end_bol_;\n";
860 if (sm_.data()._seen_BOL_assertion)
862 os_ << " bol_ = (*start_token_ == '\\n') ? true : false;\n";
864 os_ << " id_ = npos;\n";
865 os_ << " uid_ = npos;\n";
868 os_ << " unique_id_ = uid_;\n";
869 os_ << " return id_;\n";
873 ///////////////////////////////////////////////////////////////////////////
874 // Generate a tokenizer for the given state machine.
875 template <typename Char, typename F>
877 generate_cpp (boost::lexer::basic_state_machine<Char> const& sm_
878 , boost::lexer::basic_rules<Char> const& rules_
879 , std::basic_ostream<Char> &os_, Char const* name_suffix
880 , F generate_function_body)
882 if (sm_.data()._lookup->empty())
885 std::size_t const dfas_ = sm_.data()._dfa->size();
886 // std::size_t const lookups_ = sm_.data()._lookup->front()->size();
888 os_ << "// Copyright (c) 2008-2009 Ben Hanson\n";
889 os_ << "// Copyright (c) 2008-2011 Hartmut Kaiser\n";
891 os_ << "// Distributed under the Boost Software License, "
892 "Version 1.0. (See accompanying\n";
893 os_ << "// file licence_1_0.txt or copy at "
894 "http://www.boost.org/LICENSE_1_0.txt)\n\n";
895 os_ << "// Auto-generated by boost::lexer, do not edit\n\n";
897 std::basic_string<Char> guard(name_suffix);
898 guard += L<Char>(name_suffix[0] ? "_" : "");
899 guard += L<Char>(__DATE__ "_" __TIME__);
900 typename std::basic_string<Char>::size_type p =
901 guard.find_first_of(L<Char>(": "));
902 while (std::string::npos != p)
904 guard.replace(p, 1, L<Char>("_"));
905 p = guard.find_first_of(L<Char>(": "), p);
908 typedef std::ctype<Char> facet_t;
909 facet_t const& facet = std::use_facet<facet_t>(std::locale());
910 typedef typename std::basic_string<Char>::iterator iter_t;
911 for (iter_t iter = guard.begin(),
912 last = guard.end(); iter != last; ++iter)
913 *iter = facet.toupper(*iter);
916 os_ << "#if !defined(BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << ")\n";
917 os_ << "#define BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << "\n\n";
919 os_ << "#include <boost/spirit/home/support/detail/lexer/char_traits.hpp>\n\n";
921 generate_delimiter(os_);
922 os_ << "// the generated table of state names and the tokenizer have to be\n"
923 "// defined in the boost::spirit::lex::lexertl::static_ namespace\n";
924 os_ << "namespace boost { namespace spirit { namespace lex { "
925 "namespace lexertl { namespace static_ {\n\n";
927 // generate the lexer state information variables
928 if (!generate_cpp_state_info(rules_, os_, name_suffix))
931 generate_delimiter(os_);
932 os_ << "// this function returns the next matched token\n";
933 os_ << "template<typename Iterator>\n";
934 os_ << "std::size_t next_token" << (name_suffix[0] ? "_" : "")
935 << name_suffix << " (";
939 os_ << "std::size_t& start_state_, ";
943 os_ << "std::size_t& /*start_state_*/, ";
945 if (sm_.data()._seen_BOL_assertion)
947 os_ << "bool& bol_, ";
951 os_ << "bool& /*bol_*/, ";
955 os_ << "Iterator &start_token_, Iterator const& end_, ";
956 os_ << "std::size_t& unique_id_)\n";
958 if (!generate_function_body(os_, sm_))
962 if (!generate_cpp_state_table<Char>(os_, name_suffix
963 , sm_.data()._seen_BOL_assertion, sm_.data()._seen_EOL_assertion))
968 os_ << "}}}}} // namespace boost::spirit::lex::lexertl::static_\n\n";
975 } // namespace detail
977 ///////////////////////////////////////////////////////////////////////////
978 template <typename Lexer, typename F>
980 generate_static(Lexer const& lexer
981 , std::basic_ostream<typename Lexer::char_type>& os
982 , typename Lexer::char_type const* name_suffix, F f)
984 if (!lexer.init_dfa(true)) // always minimize DFA for static lexers
986 return detail::generate_cpp(lexer.state_machine_, lexer.rules_, os
990 ///////////////////////////////////////////////////////////////////////////
991 // deprecated function, will be removed in the future (this has been
992 // replaced by the function generate_static_dfa - see below).
993 template <typename Lexer>
995 generate_static(Lexer const& lexer
996 , std::basic_ostream<typename Lexer::char_type>& os
997 , typename Lexer::char_type const* name_suffix =
998 detail::L<typename Lexer::char_type>())
1000 return generate_static(lexer, os, name_suffix
1001 , &detail::generate_function_body_dfa<typename Lexer::char_type>);
1004 ///////////////////////////////////////////////////////////////////////////
1005 template <typename Lexer>
1007 generate_static_dfa(Lexer const& lexer
1008 , std::basic_ostream<typename Lexer::char_type>& os
1009 , typename Lexer::char_type const* name_suffix =
1010 detail::L<typename Lexer::char_type>())
1012 return generate_static(lexer, os, name_suffix
1013 , &detail::generate_function_body_dfa<typename Lexer::char_type>);
1016 ///////////////////////////////////////////////////////////////////////////
1017 template <typename Lexer>
1019 generate_static_switch(Lexer const& lexer
1020 , std::basic_ostream<typename Lexer::char_type>& os
1021 , typename Lexer::char_type const* name_suffix =
1022 detail::L<typename Lexer::char_type>())
1024 return generate_static(lexer, os, name_suffix
1025 , &detail::generate_function_body_switch<typename Lexer::char_type>);
1028 ///////////////////////////////////////////////////////////////////////////////