ceph/src/boost/libs/spirit/test/lex/token_onetype.cpp

   1 //  Copyright (c) 2001-2011 Hartmut Kaiser
   2 //
   3 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
   4 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
   5
   6 #include <boost/config/warning_disable.hpp>
   7 #include <boost/detail/lightweight_test.hpp>
   8
   9 #include <boost/spirit/include/lex_lexertl.hpp>
  10 #include <boost/spirit/include/lex_lexertl_position_token.hpp>
  11 #include <boost/spirit/include/phoenix_object.hpp>
  12 #include <boost/spirit/include/phoenix_operator.hpp>
  13 #include <boost/spirit/include/phoenix_container.hpp>
  14 #include <boost/spirit/include/qi_numeric.hpp>
  15
  16 namespace spirit = boost::spirit;
  17 namespace lex = boost::spirit::lex;
  18 namespace phoenix = boost::phoenix;
  19 namespace mpl = boost::mpl;
  20
  21 ///////////////////////////////////////////////////////////////////////////////
  22 enum tokenids
  23 {
  24     ID_INT = 1000,
  25     ID_DOUBLE
  26 };
  27
  28 template <typename Lexer>
  29 struct token_definitions : lex::lexer<Lexer>
  30 {
  31     token_definitions()
  32     {
  33         this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
  34         this->self.add_pattern("OCTALDIGIT", "[0-7]");
  35         this->self.add_pattern("DIGIT", "[0-9]");
  36
  37         this->self.add_pattern("OPTSIGN", "[-+]?");
  38         this->self.add_pattern("EXPSTART", "[eE][-+]");
  39         this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
  40
  41         // define tokens and associate them with the lexer
  42         int_ = "{OPTSIGN}[1-9]{DIGIT}*";
  43         int_.id(ID_INT);
  44
  45         double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
  46         double_.id(ID_DOUBLE);
  47
  48         whitespace = "[ \t\n]+";
  49
  50         this->self =
  51                 double_
  52             |   int_
  53             |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
  54             ;
  55     }
  56
  57     lex::token_def<double> int_;
  58     lex::token_def<double> double_;
  59     lex::token_def<lex::omit> whitespace;
  60 };
  61
  62 template <typename Lexer>
  63 struct token_definitions_with_state : lex::lexer<Lexer>
  64 {
  65     token_definitions_with_state()
  66     {
  67         this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
  68         this->self.add_pattern("OCTALDIGIT", "[0-7]");
  69         this->self.add_pattern("DIGIT", "[0-9]");
  70
  71         this->self.add_pattern("OPTSIGN", "[-+]?");
  72         this->self.add_pattern("EXPSTART", "[eE][-+]");
  73         this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
  74
  75         this->self.add_state();
  76         this->self.add_state("INT");
  77         this->self.add_state("DOUBLE");
  78
  79         // define tokens and associate them with the lexer
  80         int_ = "{OPTSIGN}[1-9]{DIGIT}*";
  81         int_.id(ID_INT);
  82
  83         double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
  84         double_.id(ID_DOUBLE);
  85
  86         whitespace = "[ \t\n]+";
  87
  88         this->self("*") =
  89                 double_ [ lex::_state = "DOUBLE"]
  90             |   int_ [ lex::_state = "INT" ]
  91             |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
  92             ;
  93     }
  94
  95     lex::token_def<double> int_;
  96     lex::token_def<double> double_;
  97     lex::token_def<lex::omit> whitespace;
  98 };
  99
 100 ///////////////////////////////////////////////////////////////////////////////
 101 template <typename Token>
 102 inline bool
 103 test_token_ids(int const* ids, std::vector<Token> const& tokens)
 104 {
 105     BOOST_FOREACH(Token const& t, tokens)
 106     {
 107         if (*ids == -1)
 108             return false;           // reached end of expected data
 109
 110         if (t.id() != static_cast<std::size_t>(*ids))        // token id must match
 111             return false;
 112
 113         ++ids;
 114     }
 115
 116     return (*ids == -1) ? true : false;
 117 }
 118
 119 ///////////////////////////////////////////////////////////////////////////////
 120 template <typename Token>
 121 inline bool
 122 test_token_states(std::size_t const* states, std::vector<Token> const& tokens)
 123 {
 124     BOOST_FOREACH(Token const& t, tokens)
 125     {
 126         if (*states == std::size_t(-1))
 127             return false;           // reached end of expected data
 128
 129         if (t.state() != *states)            // token state must match
 130             return false;
 131
 132         ++states;
 133     }
 134
 135     return (*states == std::size_t(-1)) ? true : false;
 136 }
 137
 138 ///////////////////////////////////////////////////////////////////////////////
 139 struct position_type
 140 {
 141     std::size_t begin, end;
 142 };
 143
 144 template <typename Iterator, typename Token>
 145 inline bool
 146 test_token_positions(Iterator begin, position_type const* positions,
 147     std::vector<Token> const& tokens)
 148 {
 149     BOOST_FOREACH(Token const& t, tokens)
 150     {
 151         if (positions->begin == std::size_t(-1) &&
 152             positions->end == std::size_t(-1))
 153         {
 154             return false;           // reached end of expected data
 155         }
 156
 157         boost::iterator_range<Iterator> matched = t.matched();
 158         std::size_t start = std::distance(begin, matched.begin());
 159         std::size_t end = std::distance(begin, matched.end());
 160
 161         // position must match
 162         if (start != positions->begin || end != positions->end)
 163             return false;
 164
 165         ++positions;
 166     }
 167
 168     return (positions->begin == std::size_t(-1) &&
 169             positions->end == std::size_t(-1)) ? true : false;
 170 }
 171
 172 ///////////////////////////////////////////////////////////////////////////////
 173 template <typename Token>
 174 inline bool
 175 test_token_values(double const* values, std::vector<Token> const& tokens)
 176 {
 177     BOOST_FOREACH(Token const& t, tokens)
 178     {
 179         if (*values == 0.0)
 180             return false;               // reached end of expected data
 181
 182         double val;
 183         spirit::traits::assign_to(t, val);
 184         if (val != *values)             // token value must match
 185             return false;
 186
 187         ++values;
 188     }
 189
 190     return (*values == 0.0) ? true : false;
 191 }
 192
 193 ///////////////////////////////////////////////////////////////////////////////
 194 int main()
 195 {
 196     typedef std::string::iterator base_iterator_type;
 197     std::string input("  1 1.2 -2   3 2.3e6 -3.4");
 198     int ids[] = { ID_INT, ID_DOUBLE, ID_INT, ID_INT, ID_DOUBLE, ID_DOUBLE, -1 };
 199     std::size_t states[] = { 0, 1, 2, 1, 1, 2, std::size_t(-1) };
 200     position_type positions[] =
 201     {
 202         { 2, 3 }, { 4, 7 }, { 8, 10 }, { 13, 14 }, { 15, 20 }, { 21, 25 },
 203         { std::size_t(-1), std::size_t(-1) }
 204     };
 205     double values[] = { 1.0, 1.2, -2.0, 3.0, 2.3e6, -3.4, 0.0 };
 206
 207     // token type: token id, iterator_pair as token value, no state
 208     {
 209         typedef lex::lexertl::token<
 210             base_iterator_type, mpl::vector<double>, mpl::false_> token_type;
 211         typedef lex::lexertl::actor_lexer<token_type> lexer_type;
 212
 213         token_definitions<lexer_type> lexer;
 214         std::vector<token_type> tokens;
 215         base_iterator_type first = input.begin();
 216
 217         using phoenix::arg_names::_1;
 218         BOOST_TEST(lex::tokenize(first, input.end(), lexer
 219           , phoenix::push_back(phoenix::ref(tokens), _1)));
 220
 221         BOOST_TEST(test_token_ids(ids, tokens));
 222         BOOST_TEST(test_token_values(values, tokens));
 223     }
 224
 225     {
 226         typedef lex::lexertl::position_token<
 227             base_iterator_type, mpl::vector<double>, mpl::false_> token_type;
 228         typedef lex::lexertl::actor_lexer<token_type> lexer_type;
 229
 230         token_definitions<lexer_type> lexer;
 231         std::vector<token_type> tokens;
 232         base_iterator_type first = input.begin();
 233
 234         using phoenix::arg_names::_1;
 235         BOOST_TEST(lex::tokenize(first, input.end(), lexer
 236           , phoenix::push_back(phoenix::ref(tokens), _1)));
 237
 238         BOOST_TEST(test_token_ids(ids, tokens));
 239         BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
 240         BOOST_TEST(test_token_values(values, tokens));
 241     }
 242
 243     // token type: holds token id, state, iterator_pair as token value
 244     {
 245         typedef lex::lexertl::token<
 246             base_iterator_type, mpl::vector<double>, mpl::true_> token_type;
 247         typedef lex::lexertl::actor_lexer<token_type> lexer_type;
 248
 249         token_definitions_with_state<lexer_type> lexer;
 250         std::vector<token_type> tokens;
 251         base_iterator_type first = input.begin();
 252
 253         using phoenix::arg_names::_1;
 254         BOOST_TEST(lex::tokenize(first, input.end(), lexer
 255           , phoenix::push_back(phoenix::ref(tokens), _1)));
 256
 257         BOOST_TEST(test_token_ids(ids, tokens));
 258         BOOST_TEST(test_token_states(states, tokens));
 259         BOOST_TEST(test_token_values(values, tokens));
 260     }
 261
 262     {
 263         typedef lex::lexertl::position_token<
 264             base_iterator_type, mpl::vector<double>, mpl::true_> token_type;
 265         typedef lex::lexertl::actor_lexer<token_type> lexer_type;
 266
 267         token_definitions_with_state<lexer_type> lexer;
 268         std::vector<token_type> tokens;
 269         base_iterator_type first = input.begin();
 270
 271         using phoenix::arg_names::_1;
 272         BOOST_TEST(lex::tokenize(first, input.end(), lexer
 273           , phoenix::push_back(phoenix::ref(tokens), _1)));
 274
 275         BOOST_TEST(test_token_ids(ids, tokens));
 276         BOOST_TEST(test_token_states(states, tokens));
 277         BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
 278         BOOST_TEST(test_token_values(values, tokens));
 279     }
 280
 281     return boost::report_errors();
 282 }