]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // file_input.hpp |
2 | // Copyright (c) 2008-2009 Ben Hanson (http://www.benhanson.net/) | |
3 | // | |
4 | // Distributed under the Boost Software License, Version 1.0. (See accompanying | |
5 | // file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) | |
f67539c2 TL |
6 | #ifndef BOOST_SPIRIT_SUPPORT_DETAIL_LEXER_FILE_INPUT_HPP |
7 | #define BOOST_SPIRIT_SUPPORT_DETAIL_LEXER_FILE_INPUT_HPP | |
7c673cae FG |
8 | |
9 | #include "char_traits.hpp" | |
10 | // memcpy | |
11 | #include <cstring> | |
12 | #include <fstream> | |
13 | #include "size_t.hpp" | |
14 | #include "state_machine.hpp" | |
15 | ||
16 | namespace boost | |
17 | { | |
18 | namespace lexer | |
19 | { | |
20 | template<typename CharT, typename Traits = char_traits<CharT> > | |
21 | class basic_file_input | |
22 | { | |
23 | public: | |
24 | class iterator | |
25 | { | |
26 | public: | |
27 | friend class basic_file_input; | |
28 | ||
29 | struct data | |
30 | { | |
31 | std::size_t id; | |
32 | std::size_t unique_id; | |
33 | const CharT *start; | |
34 | const CharT *end; | |
35 | std::size_t state; | |
36 | ||
37 | // Construct in end() state. | |
38 | data () : | |
39 | id (0), | |
40 | unique_id (npos), | |
41 | state (npos) | |
42 | { | |
43 | } | |
44 | ||
45 | bool operator == (const data &rhs_) const | |
46 | { | |
47 | return id == rhs_.id && unique_id == rhs_.unique_id && | |
48 | start == rhs_.start && end == rhs_.end && | |
49 | state == rhs_.state; | |
50 | } | |
51 | }; | |
52 | ||
53 | iterator () : | |
54 | _input (0) | |
55 | { | |
56 | } | |
57 | ||
58 | bool operator == (const iterator &rhs_) const | |
59 | { | |
60 | return _data == rhs_._data; | |
61 | } | |
62 | ||
63 | bool operator != (const iterator &rhs_) const | |
64 | { | |
65 | return !(*this == rhs_); | |
66 | } | |
67 | ||
68 | data &operator * () | |
69 | { | |
70 | return _data; | |
71 | } | |
72 | ||
73 | data *operator -> () | |
74 | { | |
75 | return &_data; | |
76 | } | |
77 | ||
78 | // Let compiler generate operator = (). | |
79 | ||
80 | // prefix version | |
81 | iterator &operator ++ () | |
82 | { | |
83 | next_token (); | |
84 | return *this; | |
85 | } | |
86 | ||
87 | // postfix version | |
88 | iterator operator ++ (int) | |
89 | { | |
90 | iterator iter_ = *this; | |
91 | ||
92 | next_token (); | |
93 | return iter_; | |
94 | } | |
95 | ||
96 | void next_token () | |
97 | { | |
98 | const detail::internals &internals_ = | |
99 | _input->_state_machine->data (); | |
100 | ||
101 | _data.start = _data.end; | |
102 | ||
103 | if (internals_._dfa->size () == 1) | |
104 | { | |
105 | _data.id = _input->next (&internals_._lookup->front ()-> | |
106 | front (), internals_._dfa_alphabet.front (), | |
107 | &internals_._dfa->front ()->front (), _data.start, | |
108 | _data.end, _data.unique_id); | |
109 | } | |
110 | else | |
111 | { | |
112 | _data.id = _input->next (internals_, _data.state, _data.start, | |
113 | _data.end, _data.unique_id); | |
114 | } | |
115 | ||
116 | if (_data.id == 0) | |
117 | { | |
118 | _data.start = 0; | |
119 | _data.end = 0; | |
120 | // Ensure current state matches that returned by end(). | |
121 | _data.state = npos; | |
122 | } | |
123 | } | |
124 | ||
125 | private: | |
126 | // Not owner (obviously!) | |
127 | basic_file_input *_input; | |
128 | data _data; | |
129 | }; | |
130 | ||
131 | friend class iterator; | |
132 | ||
f67539c2 | 133 | // Make it explicit that we are NOT taking a copy of state_machine_! |
7c673cae FG |
134 | basic_file_input (const basic_state_machine<CharT> *state_machine_, |
135 | std::basic_ifstream<CharT> *is_, | |
136 | const std::streamsize buffer_size_ = 4096, | |
137 | const std::streamsize buffer_increment_ = 1024) : | |
138 | _state_machine (state_machine_), | |
139 | _stream (is_), | |
140 | _buffer_size (buffer_size_), | |
141 | _buffer_increment (buffer_increment_), | |
142 | _buffer (_buffer_size, '!') | |
143 | { | |
144 | _start_buffer = &_buffer.front (); | |
145 | _end_buffer = _start_buffer + _buffer.size (); | |
146 | _start_token = _end_buffer; | |
147 | _end_token = _end_buffer; | |
148 | } | |
149 | ||
150 | iterator begin () | |
151 | { | |
152 | iterator iter_; | |
153 | ||
154 | iter_._input = this; | |
155 | // Over-ride default of 0 (EOF) | |
156 | iter_._data.id = npos; | |
157 | iter_._data.start = 0; | |
158 | iter_._data.end = 0; | |
159 | iter_._data.state = 0; | |
160 | ++iter_; | |
161 | return iter_; | |
162 | } | |
163 | ||
164 | iterator end () | |
165 | { | |
166 | iterator iter_; | |
167 | ||
168 | iter_._input = this; | |
169 | iter_._data.start = 0; | |
170 | iter_._data.end = 0; | |
171 | return iter_; | |
172 | } | |
173 | ||
174 | void flush () | |
175 | { | |
176 | // This temporary is mandatory, otherwise the | |
177 | // pointer calculations won't work! | |
178 | const CharT *temp_ = _end_buffer; | |
179 | ||
180 | _start_token = _end_token = _end_buffer; | |
181 | reload_buffer (temp_, true, _end_token); | |
182 | } | |
183 | ||
184 | private: | |
185 | typedef std::basic_istream<CharT> istream; | |
186 | typedef std::vector<CharT> buffer; | |
187 | ||
188 | const basic_state_machine<CharT> *_state_machine; | |
189 | const std::streamsize _buffer_size; | |
190 | const std::streamsize _buffer_increment; | |
191 | ||
192 | buffer _buffer; | |
193 | CharT *_start_buffer; | |
194 | istream *_stream; | |
195 | const CharT *_start_token; | |
196 | const CharT *_end_token; | |
197 | CharT *_end_buffer; | |
198 | ||
199 | std::size_t next (const detail::internals &internals_, | |
200 | std::size_t &start_state_, const CharT * &start_, const CharT * &end_, | |
201 | std::size_t &unique_id_) | |
202 | { | |
203 | _start_token = _end_token; | |
204 | ||
205 | again: | |
206 | const std::size_t * lookup_ = &internals_._lookup[start_state_]-> | |
207 | front (); | |
208 | std::size_t dfa_alphabet_ = internals_._dfa_alphabet[start_state_]; | |
209 | const std::size_t *dfa_ = &internals_._dfa[start_state_]->front (); | |
210 | const std::size_t *ptr_ = dfa_ + dfa_alphabet_; | |
211 | const CharT *curr_ = _start_token; | |
212 | bool end_state_ = *ptr_ != 0; | |
213 | std::size_t id_ = *(ptr_ + id_index); | |
214 | std::size_t uid_ = *(ptr_ + unique_id_index); | |
215 | const CharT *end_token_ = curr_; | |
216 | ||
217 | for (;;) | |
218 | { | |
219 | if (curr_ >= _end_buffer) | |
220 | { | |
221 | if (!reload_buffer (curr_, end_state_, end_token_)) | |
222 | { | |
223 | // EOF | |
224 | break; | |
225 | } | |
226 | } | |
227 | ||
228 | const std::size_t BOL_state_ = ptr_[bol_index]; | |
229 | const std::size_t EOL_state_ = ptr_[eol_index]; | |
230 | ||
231 | if (BOL_state_ && (_start_token == _start_buffer || | |
232 | *(_start_token - 1) == '\n')) | |
233 | { | |
234 | ptr_ = &dfa_[BOL_state_ * dfa_alphabet_]; | |
235 | } | |
236 | else if (EOL_state_ && *curr_ == '\n') | |
237 | { | |
238 | ptr_ = &dfa_[EOL_state_ * dfa_alphabet_]; | |
239 | } | |
240 | else | |
241 | { | |
242 | const std::size_t state_ = | |
243 | ptr_[lookup_[static_cast<typename Traits::index_type> | |
244 | (*curr_++)]]; | |
245 | ||
246 | if (state_ == 0) | |
247 | { | |
248 | break; | |
249 | } | |
250 | ||
251 | ptr_ = &dfa_[state_ * dfa_alphabet_]; | |
252 | } | |
253 | ||
254 | if (*ptr_) | |
255 | { | |
256 | end_state_ = true; | |
257 | id_ = *(ptr_ + id_index); | |
258 | uid_ = *(ptr_ + unique_id_index); | |
259 | start_state_ = *(ptr_ + state_index); | |
260 | end_token_ = curr_; | |
261 | } | |
262 | } | |
263 | ||
264 | if (_start_token >= _end_buffer) | |
265 | { | |
266 | // No more tokens... | |
267 | unique_id_ = npos; | |
268 | return 0; | |
269 | } | |
270 | ||
271 | const std::size_t EOL_state_ = ptr_[eol_index]; | |
272 | ||
273 | if (EOL_state_ && curr_ == end_) | |
274 | { | |
275 | ptr_ = &dfa_[EOL_state_ * dfa_alphabet_]; | |
276 | ||
277 | if (*ptr_) | |
278 | { | |
279 | end_state_ = true; | |
280 | id_ = *(ptr_ + id_index); | |
281 | uid_ = *(ptr_ + unique_id_index); | |
282 | start_state_ = *(ptr_ + state_index); | |
283 | end_token_ = curr_; | |
284 | } | |
285 | } | |
286 | ||
287 | if (end_state_) | |
288 | { | |
289 | // return longest match | |
290 | _end_token = end_token_; | |
291 | ||
292 | if (id_ == 0) goto again; | |
293 | } | |
294 | else | |
295 | { | |
296 | // No match causes char to be skipped | |
297 | _end_token = _start_token + 1; | |
298 | id_ = npos; | |
299 | uid_ = npos; | |
300 | } | |
301 | ||
302 | start_ = _start_token; | |
303 | end_ = _end_token; | |
304 | unique_id_ = uid_; | |
305 | return id_; | |
306 | } | |
307 | ||
308 | std::size_t next (const std::size_t * const lookup_, | |
309 | const std::size_t dfa_alphabet_, const std::size_t * const dfa_, | |
310 | const CharT * &start_, const CharT * &end_, std::size_t &unique_id_) | |
311 | { | |
312 | _start_token = _end_token; | |
313 | ||
314 | const std::size_t *ptr_ = dfa_ + dfa_alphabet_; | |
315 | const CharT *curr_ = _start_token; | |
316 | bool end_state_ = *ptr_ != 0; | |
317 | std::size_t id_ = *(ptr_ + id_index); | |
318 | std::size_t uid_ = *(ptr_ + unique_id_index); | |
319 | const CharT *end_token_ = curr_; | |
320 | ||
321 | for (;;) | |
322 | { | |
323 | if (curr_ >= _end_buffer) | |
324 | { | |
325 | if (!reload_buffer (curr_, end_state_, end_token_)) | |
326 | { | |
327 | // EOF | |
328 | break; | |
329 | } | |
330 | } | |
331 | ||
332 | const std::size_t BOL_state_ = ptr_[bol_index]; | |
333 | const std::size_t EOL_state_ = ptr_[eol_index]; | |
334 | ||
335 | if (BOL_state_ && (_start_token == _start_buffer || | |
336 | *(_start_token - 1) == '\n')) | |
337 | { | |
338 | ptr_ = &dfa_[BOL_state_ * dfa_alphabet_]; | |
339 | } | |
340 | else if (EOL_state_ && *curr_ == '\n') | |
341 | { | |
342 | ptr_ = &dfa_[EOL_state_ * dfa_alphabet_]; | |
343 | } | |
344 | else | |
345 | { | |
346 | const std::size_t state_ = | |
347 | ptr_[lookup_[static_cast<typename Traits::index_type> | |
348 | (*curr_++)]]; | |
349 | ||
350 | if (state_ == 0) | |
351 | { | |
352 | break; | |
353 | } | |
354 | ||
355 | ptr_ = &dfa_[state_ * dfa_alphabet_]; | |
356 | } | |
357 | ||
358 | if (*ptr_) | |
359 | { | |
360 | end_state_ = true; | |
361 | id_ = *(ptr_ + id_index); | |
362 | uid_ = *(ptr_ + unique_id_index); | |
363 | end_token_ = curr_; | |
364 | } | |
365 | } | |
366 | ||
367 | if (_start_token >= _end_buffer) | |
368 | { | |
369 | // No more tokens... | |
370 | unique_id_ = npos; | |
371 | return 0; | |
372 | } | |
373 | ||
374 | const std::size_t EOL_state_ = ptr_[eol_index]; | |
375 | ||
376 | if (EOL_state_ && curr_ == end_) | |
377 | { | |
378 | ptr_ = &dfa_[EOL_state_ * dfa_alphabet_]; | |
379 | ||
380 | if (*ptr_) | |
381 | { | |
382 | end_state_ = true; | |
383 | id_ = *(ptr_ + id_index); | |
384 | uid_ = *(ptr_ + unique_id_index); | |
385 | end_token_ = curr_; | |
386 | } | |
387 | } | |
388 | ||
389 | if (end_state_) | |
390 | { | |
391 | // return longest match | |
392 | _end_token = end_token_; | |
393 | } | |
394 | else | |
395 | { | |
396 | // No match causes char to be skipped | |
397 | _end_token = _start_token + 1; | |
398 | id_ = npos; | |
399 | uid_ = npos; | |
400 | } | |
401 | ||
402 | start_ = _start_token; | |
403 | end_ = _end_token; | |
404 | unique_id_ = uid_; | |
405 | return id_; | |
406 | } | |
407 | ||
408 | bool reload_buffer (const CharT * &curr_, const bool end_state_, | |
409 | const CharT * &end_token_) | |
410 | { | |
411 | bool success_ = !_stream->eof (); | |
412 | ||
413 | if (success_) | |
414 | { | |
415 | const CharT *old_start_token_ = _start_token; | |
416 | std::size_t old_size_ = _buffer.size (); | |
417 | std::size_t count_ = 0; | |
418 | ||
419 | if (_start_token - 1 == _start_buffer) | |
420 | { | |
421 | // Run out of buffer space, so increase. | |
422 | _buffer.resize (old_size_ + _buffer_increment, '!'); | |
423 | _start_buffer = &_buffer.front (); | |
424 | _start_token = _start_buffer + 1; | |
425 | _stream->read (_start_buffer + old_size_, | |
426 | _buffer_increment); | |
427 | count_ = _stream->gcount (); | |
428 | _end_buffer = _start_buffer + old_size_ + count_; | |
429 | } | |
430 | else if (_start_token < _end_buffer) | |
431 | { | |
432 | const std::size_t len_ = _end_buffer - _start_token; | |
433 | // Some systems have memcpy in namespace std. | |
434 | using namespace std; | |
435 | ||
436 | memcpy (_start_buffer, _start_token - 1, (len_ + 1) * | |
437 | sizeof (CharT)); | |
438 | _stream->read (_start_buffer + len_ + 1, | |
439 | static_cast<std::streamsize> (_buffer.size () - len_ - 1)); | |
440 | count_ = _stream->gcount (); | |
441 | _start_token = _start_buffer + 1; | |
442 | _end_buffer = _start_buffer + len_ + 1 + count_; | |
443 | } | |
444 | else | |
445 | { | |
446 | _stream->read (_start_buffer, static_cast<std::streamsize> | |
447 | (_buffer.size ())); | |
448 | count_ = _stream->gcount (); | |
449 | _start_token = _start_buffer; | |
450 | _end_buffer = _start_buffer + count_; | |
451 | } | |
452 | ||
453 | if (end_state_) | |
454 | { | |
455 | end_token_ = _start_token + | |
456 | (end_token_ - old_start_token_); | |
457 | } | |
458 | ||
459 | curr_ = _start_token + (curr_ - old_start_token_); | |
460 | } | |
461 | ||
462 | return success_; | |
463 | } | |
464 | ||
465 | // Disallow copying of buffer | |
466 | basic_file_input (const basic_file_input &); | |
467 | const basic_file_input &operator = (const basic_file_input &); | |
468 | }; | |
469 | ||
470 | typedef basic_file_input<char> file_input; | |
471 | typedef basic_file_input<wchar_t> wfile_input; | |
472 | } | |
473 | } | |
474 | ||
475 | #endif |