ceph/src/boost/libs/spirit/test/lex/token_onetype.cpp

   1 //  Copyright (c) 2001-2011 Hartmut Kaiser
   2 //
   3 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
   4 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
   5
   6 #include <boost/spirit/include/lex_lexertl.hpp>
   7 #include <boost/spirit/include/lex_lexertl_position_token.hpp>
   8
   9 #include <boost/core/lightweight_test.hpp>
  10 #include <boost/phoenix/object.hpp>
  11 #include <boost/phoenix/operator.hpp>
  12 #include <boost/phoenix/stl/container.hpp>
  13 #include <boost/spirit/include/qi_numeric.hpp>
  14
  15 namespace spirit = boost::spirit;
  16 namespace lex = boost::spirit::lex;
  17 namespace phoenix = boost::phoenix;
  18 namespace mpl = boost::mpl;
  19
  20 ///////////////////////////////////////////////////////////////////////////////
  21 enum tokenids
  22 {
  23     ID_INT = 1000,
  24     ID_DOUBLE
  25 };
  26
  27 template <typename Lexer>
  28 struct token_definitions : lex::lexer<Lexer>
  29 {
  30     token_definitions()
  31     {
  32         this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
  33         this->self.add_pattern("OCTALDIGIT", "[0-7]");
  34         this->self.add_pattern("DIGIT", "[0-9]");
  35
  36         this->self.add_pattern("OPTSIGN", "[-+]?");
  37         this->self.add_pattern("EXPSTART", "[eE][-+]");
  38         this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
  39
  40         // define tokens and associate them with the lexer
  41         int_ = "{OPTSIGN}[1-9]{DIGIT}*";
  42         int_.id(ID_INT);
  43
  44         double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
  45         double_.id(ID_DOUBLE);
  46
  47         whitespace = "[ \t\n]+";
  48
  49         this->self =
  50                 double_
  51             |   int_
  52             |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
  53             ;
  54     }
  55
  56     lex::token_def<double> int_;
  57     lex::token_def<double> double_;
  58     lex::token_def<lex::omit> whitespace;
  59 };
  60
  61 template <typename Lexer>
  62 struct token_definitions_with_state : lex::lexer<Lexer>
  63 {
  64     token_definitions_with_state()
  65     {
  66         this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
  67         this->self.add_pattern("OCTALDIGIT", "[0-7]");
  68         this->self.add_pattern("DIGIT", "[0-9]");
  69
  70         this->self.add_pattern("OPTSIGN", "[-+]?");
  71         this->self.add_pattern("EXPSTART", "[eE][-+]");
  72         this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
  73
  74         this->self.add_state();
  75         this->self.add_state("INT");
  76         this->self.add_state("DOUBLE");
  77
  78         // define tokens and associate them with the lexer
  79         int_ = "{OPTSIGN}[1-9]{DIGIT}*";
  80         int_.id(ID_INT);
  81
  82         double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
  83         double_.id(ID_DOUBLE);
  84
  85         whitespace = "[ \t\n]+";
  86
  87         this->self("*") =
  88                 double_ [ lex::_state = "DOUBLE"]
  89             |   int_ [ lex::_state = "INT" ]
  90             |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
  91             ;
  92     }
  93
  94     lex::token_def<double> int_;
  95     lex::token_def<double> double_;
  96     lex::token_def<lex::omit> whitespace;
  97 };
  98
  99 ///////////////////////////////////////////////////////////////////////////////
 100 template <typename Token>
 101 inline bool
 102 test_token_ids(int const* ids, std::vector<Token> const& tokens)
 103 {
 104     for (std::size_t i = 0, len = tokens.size(); i < len; ++i)
 105     {
 106         if (*ids == -1)
 107             return false;           // reached end of expected data
 108
 109         if (tokens[i].id() != static_cast<std::size_t>(*ids))        // token id must match
 110             return false;
 111
 112         ++ids;
 113     }
 114
 115     return (*ids == -1) ? true : false;
 116 }
 117
 118 ///////////////////////////////////////////////////////////////////////////////
 119 template <typename Token>
 120 inline bool
 121 test_token_states(std::size_t const* states, std::vector<Token> const& tokens)
 122 {
 123     for (std::size_t i = 0, len = tokens.size(); i < len; ++i)
 124     {
 125         if (*states == std::size_t(-1))
 126             return false;           // reached end of expected data
 127
 128         if (tokens[i].state() != *states)            // token state must match
 129             return false;
 130
 131         ++states;
 132     }
 133
 134     return (*states == std::size_t(-1)) ? true : false;
 135 }
 136
 137 ///////////////////////////////////////////////////////////////////////////////
 138 struct position_type
 139 {
 140     std::size_t begin, end;
 141 };
 142
 143 template <typename Iterator, typename Token>
 144 inline bool
 145 test_token_positions(Iterator begin, position_type const* positions,
 146     std::vector<Token> const& tokens)
 147 {
 148     for (std::size_t i = 0, len = tokens.size(); i < len; ++i)
 149     {
 150         if (positions->begin == std::size_t(-1) &&
 151             positions->end == std::size_t(-1))
 152         {
 153             return false;           // reached end of expected data
 154         }
 155
 156         boost::iterator_range<Iterator> matched = tokens[i].matched();
 157         std::size_t start = std::distance(begin, matched.begin());
 158         std::size_t end = std::distance(begin, matched.end());
 159
 160         // position must match
 161         if (start != positions->begin || end != positions->end)
 162             return false;
 163
 164         ++positions;
 165     }
 166
 167     return (positions->begin == std::size_t(-1) &&
 168             positions->end == std::size_t(-1)) ? true : false;
 169 }
 170
 171 ///////////////////////////////////////////////////////////////////////////////
 172 template <typename Token>
 173 inline bool
 174 test_token_values(double const* values, std::vector<Token> const& tokens)
 175 {
 176     for (std::size_t i = 0, len = tokens.size(); i < len; ++i)
 177     {
 178         if (*values == 0.0)
 179             return false;               // reached end of expected data
 180
 181         double val;
 182         spirit::traits::assign_to(tokens[i], val);
 183         if (val != *values)             // token value must match
 184             return false;
 185
 186         ++values;
 187     }
 188
 189     return (*values == 0.0) ? true : false;
 190 }
 191
 192 ///////////////////////////////////////////////////////////////////////////////
 193 int main()
 194 {
 195     typedef std::string::iterator base_iterator_type;
 196     std::string input("  1 1.2 -2   3 2.3e6 -3.4");
 197     int ids[] = { ID_INT, ID_DOUBLE, ID_INT, ID_INT, ID_DOUBLE, ID_DOUBLE, -1 };
 198     std::size_t states[] = { 0, 1, 2, 1, 1, 2, std::size_t(-1) };
 199     position_type positions[] =
 200     {
 201         { 2, 3 }, { 4, 7 }, { 8, 10 }, { 13, 14 }, { 15, 20 }, { 21, 25 },
 202         { std::size_t(-1), std::size_t(-1) }
 203     };
 204     double values[] = { 1.0, 1.2, -2.0, 3.0, 2.3e6, -3.4, 0.0 };
 205
 206     // token type: token id, iterator_pair as token value, no state
 207     {
 208         typedef lex::lexertl::token<
 209             base_iterator_type, mpl::vector<double>, mpl::false_> token_type;
 210         typedef lex::lexertl::actor_lexer<token_type> lexer_type;
 211
 212         token_definitions<lexer_type> lexer;
 213         std::vector<token_type> tokens;
 214         base_iterator_type first = input.begin();
 215
 216         using phoenix::arg_names::_1;
 217         BOOST_TEST(lex::tokenize(first, input.end(), lexer
 218           , phoenix::push_back(phoenix::ref(tokens), _1)));
 219
 220         BOOST_TEST(test_token_ids(ids, tokens));
 221         BOOST_TEST(test_token_values(values, tokens));
 222     }
 223
 224     {
 225         typedef lex::lexertl::position_token<
 226             base_iterator_type, mpl::vector<double>, mpl::false_> token_type;
 227         typedef lex::lexertl::actor_lexer<token_type> lexer_type;
 228
 229         token_definitions<lexer_type> lexer;
 230         std::vector<token_type> tokens;
 231         base_iterator_type first = input.begin();
 232
 233         using phoenix::arg_names::_1;
 234         BOOST_TEST(lex::tokenize(first, input.end(), lexer
 235           , phoenix::push_back(phoenix::ref(tokens), _1)));
 236
 237         BOOST_TEST(test_token_ids(ids, tokens));
 238         BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
 239         BOOST_TEST(test_token_values(values, tokens));
 240     }
 241
 242     // token type: holds token id, state, iterator_pair as token value
 243     {
 244         typedef lex::lexertl::token<
 245             base_iterator_type, mpl::vector<double>, mpl::true_> token_type;
 246         typedef lex::lexertl::actor_lexer<token_type> lexer_type;
 247
 248         token_definitions_with_state<lexer_type> lexer;
 249         std::vector<token_type> tokens;
 250         base_iterator_type first = input.begin();
 251
 252         using phoenix::arg_names::_1;
 253         BOOST_TEST(lex::tokenize(first, input.end(), lexer
 254           , phoenix::push_back(phoenix::ref(tokens), _1)));
 255
 256         BOOST_TEST(test_token_ids(ids, tokens));
 257         BOOST_TEST(test_token_states(states, tokens));
 258         BOOST_TEST(test_token_values(values, tokens));
 259     }
 260
 261     {
 262         typedef lex::lexertl::position_token<
 263             base_iterator_type, mpl::vector<double>, mpl::true_> token_type;
 264         typedef lex::lexertl::actor_lexer<token_type> lexer_type;
 265
 266         token_definitions_with_state<lexer_type> lexer;
 267         std::vector<token_type> tokens;
 268         base_iterator_type first = input.begin();
 269
 270         using phoenix::arg_names::_1;
 271         BOOST_TEST(lex::tokenize(first, input.end(), lexer
 272           , phoenix::push_back(phoenix::ref(tokens), _1)));
 273
 274         BOOST_TEST(test_token_ids(ids, tokens));
 275         BOOST_TEST(test_token_states(states, tokens));
 276         BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
 277         BOOST_TEST(test_token_values(values, tokens));
 278     }
 279
 280     return boost::report_errors();
 281 }