]> git.proxmox.com Git - ceph.git/blame - ceph/src/boost/boost/spirit/home/support/detail/lexer/file_input.hpp
update source to Ceph Pacific 16.2.2
[ceph.git] / ceph / src / boost / boost / spirit / home / support / detail / lexer / file_input.hpp
CommitLineData
7c673cae
FG
1// file_input.hpp
2// Copyright (c) 2008-2009 Ben Hanson (http://www.benhanson.net/)
3//
4// Distributed under the Boost Software License, Version 1.0. (See accompanying
5// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
f67539c2
TL
6#ifndef BOOST_SPIRIT_SUPPORT_DETAIL_LEXER_FILE_INPUT_HPP
7#define BOOST_SPIRIT_SUPPORT_DETAIL_LEXER_FILE_INPUT_HPP
7c673cae
FG
8
9#include "char_traits.hpp"
10// memcpy
11#include <cstring>
12#include <fstream>
13#include "size_t.hpp"
14#include "state_machine.hpp"
15
16namespace boost
17{
18namespace lexer
19{
20template<typename CharT, typename Traits = char_traits<CharT> >
21class basic_file_input
22{
23public:
24 class iterator
25 {
26 public:
27 friend class basic_file_input;
28
29 struct data
30 {
31 std::size_t id;
32 std::size_t unique_id;
33 const CharT *start;
34 const CharT *end;
35 std::size_t state;
36
37 // Construct in end() state.
38 data () :
39 id (0),
40 unique_id (npos),
41 state (npos)
42 {
43 }
44
45 bool operator == (const data &rhs_) const
46 {
47 return id == rhs_.id && unique_id == rhs_.unique_id &&
48 start == rhs_.start && end == rhs_.end &&
49 state == rhs_.state;
50 }
51 };
52
53 iterator () :
54 _input (0)
55 {
56 }
57
58 bool operator == (const iterator &rhs_) const
59 {
60 return _data == rhs_._data;
61 }
62
63 bool operator != (const iterator &rhs_) const
64 {
65 return !(*this == rhs_);
66 }
67
68 data &operator * ()
69 {
70 return _data;
71 }
72
73 data *operator -> ()
74 {
75 return &_data;
76 }
77
78 // Let compiler generate operator = ().
79
80 // prefix version
81 iterator &operator ++ ()
82 {
83 next_token ();
84 return *this;
85 }
86
87 // postfix version
88 iterator operator ++ (int)
89 {
90 iterator iter_ = *this;
91
92 next_token ();
93 return iter_;
94 }
95
96 void next_token ()
97 {
98 const detail::internals &internals_ =
99 _input->_state_machine->data ();
100
101 _data.start = _data.end;
102
103 if (internals_._dfa->size () == 1)
104 {
105 _data.id = _input->next (&internals_._lookup->front ()->
106 front (), internals_._dfa_alphabet.front (),
107 &internals_._dfa->front ()->front (), _data.start,
108 _data.end, _data.unique_id);
109 }
110 else
111 {
112 _data.id = _input->next (internals_, _data.state, _data.start,
113 _data.end, _data.unique_id);
114 }
115
116 if (_data.id == 0)
117 {
118 _data.start = 0;
119 _data.end = 0;
120 // Ensure current state matches that returned by end().
121 _data.state = npos;
122 }
123 }
124
125 private:
126 // Not owner (obviously!)
127 basic_file_input *_input;
128 data _data;
129 };
130
131 friend class iterator;
132
f67539c2 133 // Make it explicit that we are NOT taking a copy of state_machine_!
7c673cae
FG
134 basic_file_input (const basic_state_machine<CharT> *state_machine_,
135 std::basic_ifstream<CharT> *is_,
136 const std::streamsize buffer_size_ = 4096,
137 const std::streamsize buffer_increment_ = 1024) :
138 _state_machine (state_machine_),
139 _stream (is_),
140 _buffer_size (buffer_size_),
141 _buffer_increment (buffer_increment_),
142 _buffer (_buffer_size, '!')
143 {
144 _start_buffer = &_buffer.front ();
145 _end_buffer = _start_buffer + _buffer.size ();
146 _start_token = _end_buffer;
147 _end_token = _end_buffer;
148 }
149
150 iterator begin ()
151 {
152 iterator iter_;
153
154 iter_._input = this;
155 // Over-ride default of 0 (EOF)
156 iter_._data.id = npos;
157 iter_._data.start = 0;
158 iter_._data.end = 0;
159 iter_._data.state = 0;
160 ++iter_;
161 return iter_;
162 }
163
164 iterator end ()
165 {
166 iterator iter_;
167
168 iter_._input = this;
169 iter_._data.start = 0;
170 iter_._data.end = 0;
171 return iter_;
172 }
173
174 void flush ()
175 {
176 // This temporary is mandatory, otherwise the
177 // pointer calculations won't work!
178 const CharT *temp_ = _end_buffer;
179
180 _start_token = _end_token = _end_buffer;
181 reload_buffer (temp_, true, _end_token);
182 }
183
184private:
185 typedef std::basic_istream<CharT> istream;
186 typedef std::vector<CharT> buffer;
187
188 const basic_state_machine<CharT> *_state_machine;
189 const std::streamsize _buffer_size;
190 const std::streamsize _buffer_increment;
191
192 buffer _buffer;
193 CharT *_start_buffer;
194 istream *_stream;
195 const CharT *_start_token;
196 const CharT *_end_token;
197 CharT *_end_buffer;
198
199 std::size_t next (const detail::internals &internals_,
200 std::size_t &start_state_, const CharT * &start_, const CharT * &end_,
201 std::size_t &unique_id_)
202 {
203 _start_token = _end_token;
204
205again:
206 const std::size_t * lookup_ = &internals_._lookup[start_state_]->
207 front ();
208 std::size_t dfa_alphabet_ = internals_._dfa_alphabet[start_state_];
209 const std::size_t *dfa_ = &internals_._dfa[start_state_]->front ();
210 const std::size_t *ptr_ = dfa_ + dfa_alphabet_;
211 const CharT *curr_ = _start_token;
212 bool end_state_ = *ptr_ != 0;
213 std::size_t id_ = *(ptr_ + id_index);
214 std::size_t uid_ = *(ptr_ + unique_id_index);
215 const CharT *end_token_ = curr_;
216
217 for (;;)
218 {
219 if (curr_ >= _end_buffer)
220 {
221 if (!reload_buffer (curr_, end_state_, end_token_))
222 {
223 // EOF
224 break;
225 }
226 }
227
228 const std::size_t BOL_state_ = ptr_[bol_index];
229 const std::size_t EOL_state_ = ptr_[eol_index];
230
231 if (BOL_state_ && (_start_token == _start_buffer ||
232 *(_start_token - 1) == '\n'))
233 {
234 ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
235 }
236 else if (EOL_state_ && *curr_ == '\n')
237 {
238 ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
239 }
240 else
241 {
242 const std::size_t state_ =
243 ptr_[lookup_[static_cast<typename Traits::index_type>
244 (*curr_++)]];
245
246 if (state_ == 0)
247 {
248 break;
249 }
250
251 ptr_ = &dfa_[state_ * dfa_alphabet_];
252 }
253
254 if (*ptr_)
255 {
256 end_state_ = true;
257 id_ = *(ptr_ + id_index);
258 uid_ = *(ptr_ + unique_id_index);
259 start_state_ = *(ptr_ + state_index);
260 end_token_ = curr_;
261 }
262 }
263
264 if (_start_token >= _end_buffer)
265 {
266 // No more tokens...
267 unique_id_ = npos;
268 return 0;
269 }
270
271 const std::size_t EOL_state_ = ptr_[eol_index];
272
273 if (EOL_state_ && curr_ == end_)
274 {
275 ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
276
277 if (*ptr_)
278 {
279 end_state_ = true;
280 id_ = *(ptr_ + id_index);
281 uid_ = *(ptr_ + unique_id_index);
282 start_state_ = *(ptr_ + state_index);
283 end_token_ = curr_;
284 }
285 }
286
287 if (end_state_)
288 {
289 // return longest match
290 _end_token = end_token_;
291
292 if (id_ == 0) goto again;
293 }
294 else
295 {
296 // No match causes char to be skipped
297 _end_token = _start_token + 1;
298 id_ = npos;
299 uid_ = npos;
300 }
301
302 start_ = _start_token;
303 end_ = _end_token;
304 unique_id_ = uid_;
305 return id_;
306 }
307
308 std::size_t next (const std::size_t * const lookup_,
309 const std::size_t dfa_alphabet_, const std::size_t * const dfa_,
310 const CharT * &start_, const CharT * &end_, std::size_t &unique_id_)
311 {
312 _start_token = _end_token;
313
314 const std::size_t *ptr_ = dfa_ + dfa_alphabet_;
315 const CharT *curr_ = _start_token;
316 bool end_state_ = *ptr_ != 0;
317 std::size_t id_ = *(ptr_ + id_index);
318 std::size_t uid_ = *(ptr_ + unique_id_index);
319 const CharT *end_token_ = curr_;
320
321 for (;;)
322 {
323 if (curr_ >= _end_buffer)
324 {
325 if (!reload_buffer (curr_, end_state_, end_token_))
326 {
327 // EOF
328 break;
329 }
330 }
331
332 const std::size_t BOL_state_ = ptr_[bol_index];
333 const std::size_t EOL_state_ = ptr_[eol_index];
334
335 if (BOL_state_ && (_start_token == _start_buffer ||
336 *(_start_token - 1) == '\n'))
337 {
338 ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
339 }
340 else if (EOL_state_ && *curr_ == '\n')
341 {
342 ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
343 }
344 else
345 {
346 const std::size_t state_ =
347 ptr_[lookup_[static_cast<typename Traits::index_type>
348 (*curr_++)]];
349
350 if (state_ == 0)
351 {
352 break;
353 }
354
355 ptr_ = &dfa_[state_ * dfa_alphabet_];
356 }
357
358 if (*ptr_)
359 {
360 end_state_ = true;
361 id_ = *(ptr_ + id_index);
362 uid_ = *(ptr_ + unique_id_index);
363 end_token_ = curr_;
364 }
365 }
366
367 if (_start_token >= _end_buffer)
368 {
369 // No more tokens...
370 unique_id_ = npos;
371 return 0;
372 }
373
374 const std::size_t EOL_state_ = ptr_[eol_index];
375
376 if (EOL_state_ && curr_ == end_)
377 {
378 ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
379
380 if (*ptr_)
381 {
382 end_state_ = true;
383 id_ = *(ptr_ + id_index);
384 uid_ = *(ptr_ + unique_id_index);
385 end_token_ = curr_;
386 }
387 }
388
389 if (end_state_)
390 {
391 // return longest match
392 _end_token = end_token_;
393 }
394 else
395 {
396 // No match causes char to be skipped
397 _end_token = _start_token + 1;
398 id_ = npos;
399 uid_ = npos;
400 }
401
402 start_ = _start_token;
403 end_ = _end_token;
404 unique_id_ = uid_;
405 return id_;
406 }
407
408 bool reload_buffer (const CharT * &curr_, const bool end_state_,
409 const CharT * &end_token_)
410 {
411 bool success_ = !_stream->eof ();
412
413 if (success_)
414 {
415 const CharT *old_start_token_ = _start_token;
416 std::size_t old_size_ = _buffer.size ();
417 std::size_t count_ = 0;
418
419 if (_start_token - 1 == _start_buffer)
420 {
421 // Run out of buffer space, so increase.
422 _buffer.resize (old_size_ + _buffer_increment, '!');
423 _start_buffer = &_buffer.front ();
424 _start_token = _start_buffer + 1;
425 _stream->read (_start_buffer + old_size_,
426 _buffer_increment);
427 count_ = _stream->gcount ();
428 _end_buffer = _start_buffer + old_size_ + count_;
429 }
430 else if (_start_token < _end_buffer)
431 {
432 const std::size_t len_ = _end_buffer - _start_token;
433 // Some systems have memcpy in namespace std.
434 using namespace std;
435
436 memcpy (_start_buffer, _start_token - 1, (len_ + 1) *
437 sizeof (CharT));
438 _stream->read (_start_buffer + len_ + 1,
439 static_cast<std::streamsize> (_buffer.size () - len_ - 1));
440 count_ = _stream->gcount ();
441 _start_token = _start_buffer + 1;
442 _end_buffer = _start_buffer + len_ + 1 + count_;
443 }
444 else
445 {
446 _stream->read (_start_buffer, static_cast<std::streamsize>
447 (_buffer.size ()));
448 count_ = _stream->gcount ();
449 _start_token = _start_buffer;
450 _end_buffer = _start_buffer + count_;
451 }
452
453 if (end_state_)
454 {
455 end_token_ = _start_token +
456 (end_token_ - old_start_token_);
457 }
458
459 curr_ = _start_token + (curr_ - old_start_token_);
460 }
461
462 return success_;
463 }
464
465 // Disallow copying of buffer
466 basic_file_input (const basic_file_input &);
467 const basic_file_input &operator = (const basic_file_input &);
468};
469
470typedef basic_file_input<char> file_input;
471typedef basic_file_input<wchar_t> wfile_input;
472}
473}
474
475#endif