]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | #ifndef CEPH_BUFFER_H | |
15 | #define CEPH_BUFFER_H | |
16 | ||
17 | #if defined(__linux__) || defined(__FreeBSD__) | |
18 | #include <stdlib.h> | |
19 | #endif | |
20 | #include <limits.h> | |
21 | ||
22 | #ifndef _XOPEN_SOURCE | |
23 | # define _XOPEN_SOURCE 600 | |
24 | #endif | |
25 | ||
26 | #include <stdio.h> | |
27 | #include <sys/uio.h> | |
28 | ||
29 | #if defined(__linux__) // For malloc(2). | |
30 | #include <malloc.h> | |
31 | #endif | |
32 | ||
33 | #include <inttypes.h> | |
34 | #include <stdint.h> | |
35 | #include <string.h> | |
36 | ||
37 | #ifndef __CYGWIN__ | |
38 | # include <sys/mman.h> | |
39 | #endif | |
40 | ||
41 | #include <iosfwd> | |
42 | #include <iomanip> | |
43 | #include <list> | |
44 | #include <vector> | |
45 | #include <string> | |
46 | #include <exception> | |
47 | #include <type_traits> | |
48 | ||
49 | #include "page.h" | |
50 | #include "crc32c.h" | |
51 | #include "buffer_fwd.h" | |
52 | ||
53 | #ifdef __CEPH__ | |
54 | # include "include/assert.h" | |
55 | #else | |
56 | # include <assert.h> | |
57 | #endif | |
58 | ||
59 | #include "inline_memory.h" | |
60 | ||
61 | #if __GNUC__ >= 4 | |
62 | #define CEPH_BUFFER_API __attribute__ ((visibility ("default"))) | |
63 | #else | |
64 | #define CEPH_BUFFER_API | |
65 | #endif | |
66 | ||
67 | #if defined(HAVE_XIO) | |
68 | struct xio_reg_mem; | |
69 | class XioDispatchHook; | |
70 | #endif | |
71 | class deleter; | |
72 | ||
73 | namespace ceph { | |
74 | ||
75 | namespace buffer CEPH_BUFFER_API { | |
76 | /* | |
77 | * exceptions | |
78 | */ | |
79 | ||
80 | struct error : public std::exception{ | |
81 | const char *what() const throw () override; | |
82 | }; | |
83 | struct bad_alloc : public error { | |
84 | const char *what() const throw () override; | |
85 | }; | |
86 | struct end_of_buffer : public error { | |
87 | const char *what() const throw () override; | |
88 | }; | |
89 | struct malformed_input : public error { | |
90 | explicit malformed_input(const std::string& w) { | |
91 | snprintf(buf, sizeof(buf), "buffer::malformed_input: %s", w.c_str()); | |
92 | } | |
93 | const char *what() const throw () override; | |
94 | private: | |
95 | char buf[256]; | |
96 | }; | |
97 | struct error_code : public malformed_input { | |
98 | explicit error_code(int error); | |
99 | int code; | |
100 | }; | |
101 | ||
102 | ||
103 | /// total bytes allocated | |
104 | int get_total_alloc(); | |
105 | ||
106 | /// history total bytes allocated | |
107 | uint64_t get_history_alloc_bytes(); | |
108 | ||
109 | /// total num allocated | |
110 | uint64_t get_history_alloc_num(); | |
111 | ||
112 | /// enable/disable alloc tracking | |
113 | void track_alloc(bool b); | |
114 | ||
115 | /// count of cached crc hits (matching input) | |
116 | int get_cached_crc(); | |
117 | /// count of cached crc hits (mismatching input, required adjustment) | |
118 | int get_cached_crc_adjusted(); | |
119 | /// count of crc cache misses | |
120 | int get_missed_crc(); | |
121 | /// enable/disable tracking of cached crcs | |
122 | void track_cached_crc(bool b); | |
123 | ||
124 | /// count of calls to buffer::ptr::c_str() | |
125 | int get_c_str_accesses(); | |
126 | /// enable/disable tracking of buffer::ptr::c_str() calls | |
127 | void track_c_str(bool b); | |
128 | ||
129 | /* | |
130 | * an abstract raw buffer. with a reference count. | |
131 | */ | |
132 | class raw; | |
133 | class raw_malloc; | |
134 | class raw_static; | |
135 | class raw_mmap_pages; | |
136 | class raw_posix_aligned; | |
137 | class raw_hack_aligned; | |
138 | class raw_char; | |
31f18b77 | 139 | class raw_claimed_char; |
7c673cae FG |
140 | class raw_pipe; |
141 | class raw_unshareable; // diagnostic, unshareable char buffer | |
142 | class raw_combined; | |
143 | class raw_claim_buffer; | |
144 | ||
145 | ||
146 | class xio_mempool; | |
147 | class xio_msg_buffer; | |
148 | ||
149 | /* | |
150 | * named constructors | |
151 | */ | |
152 | raw* copy(const char *c, unsigned len); | |
153 | raw* create(unsigned len); | |
3efd9988 | 154 | raw* create_in_mempool(unsigned len, int mempool); |
7c673cae FG |
155 | raw* claim_char(unsigned len, char *buf); |
156 | raw* create_malloc(unsigned len); | |
157 | raw* claim_malloc(unsigned len, char *buf); | |
158 | raw* create_static(unsigned len, char *buf); | |
159 | raw* create_aligned(unsigned len, unsigned align); | |
3efd9988 | 160 | raw* create_aligned_in_mempool(unsigned len, unsigned align, int mempool); |
7c673cae FG |
161 | raw* create_page_aligned(unsigned len); |
162 | raw* create_zero_copy(unsigned len, int fd, int64_t *offset); | |
163 | raw* create_unshareable(unsigned len); | |
164 | raw* create_static(unsigned len, char *buf); | |
165 | raw* claim_buffer(unsigned len, char *buf, deleter del); | |
166 | ||
167 | #if defined(HAVE_XIO) | |
168 | raw* create_msg(unsigned len, char *buf, XioDispatchHook *m_hook); | |
169 | #endif | |
170 | ||
171 | /* | |
172 | * a buffer pointer. references (a subsequence of) a raw buffer. | |
173 | */ | |
174 | class CEPH_BUFFER_API ptr { | |
175 | raw *_raw; | |
176 | unsigned _off, _len; | |
177 | ||
178 | void release(); | |
179 | ||
180 | public: | |
181 | class iterator { | |
182 | const ptr *bp; ///< parent ptr | |
183 | const char *start; ///< starting pointer into bp->c_str() | |
184 | const char *pos; ///< pointer into bp->c_str() | |
185 | const char *end_ptr; ///< pointer to bp->end_c_str() | |
186 | bool deep; ///< if true, no not allow shallow ptr copies | |
187 | ||
188 | iterator(const ptr *p, size_t offset, bool d) | |
189 | : bp(p), | |
190 | start(p->c_str() + offset), | |
191 | pos(start), | |
192 | end_ptr(p->end_c_str()), | |
193 | deep(d) {} | |
194 | ||
195 | friend class ptr; | |
196 | ||
197 | public: | |
198 | const char *get_pos_add(size_t n) { | |
199 | const char *r = pos; | |
200 | pos += n; | |
201 | if (pos > end_ptr) | |
202 | throw end_of_buffer(); | |
203 | return r; | |
204 | } | |
205 | ||
206 | ptr get_ptr(size_t len) { | |
207 | if (deep) { | |
208 | return buffer::copy(get_pos_add(len), len); | |
209 | } else { | |
210 | size_t off = pos - bp->c_str(); | |
211 | pos += len; | |
212 | if (pos > end_ptr) | |
213 | throw end_of_buffer(); | |
214 | return ptr(*bp, off, len); | |
215 | } | |
216 | } | |
217 | ptr get_preceding_ptr(size_t len) { | |
218 | if (deep) { | |
219 | return buffer::copy(get_pos() - len, len); | |
220 | } else { | |
221 | size_t off = pos - bp->c_str(); | |
222 | return ptr(*bp, off - len, len); | |
223 | } | |
224 | } | |
225 | ||
226 | void advance(size_t len) { | |
227 | pos += len; | |
228 | if (pos > end_ptr) | |
229 | throw end_of_buffer(); | |
230 | } | |
231 | ||
232 | const char *get_pos() { | |
233 | return pos; | |
234 | } | |
235 | const char *get_end() { | |
236 | return end_ptr; | |
237 | } | |
238 | ||
239 | size_t get_offset() { | |
240 | return pos - start; | |
241 | } | |
242 | ||
243 | bool end() const { | |
244 | return pos == end_ptr; | |
245 | } | |
246 | }; | |
247 | ||
248 | ptr() : _raw(0), _off(0), _len(0) {} | |
249 | // cppcheck-suppress noExplicitConstructor | |
250 | ptr(raw *r); | |
251 | // cppcheck-suppress noExplicitConstructor | |
252 | ptr(unsigned l); | |
253 | ptr(const char *d, unsigned l); | |
254 | ptr(const ptr& p); | |
255 | ptr(ptr&& p) noexcept; | |
256 | ptr(const ptr& p, unsigned o, unsigned l); | |
257 | ptr& operator= (const ptr& p); | |
258 | ptr& operator= (ptr&& p) noexcept; | |
259 | ~ptr() { | |
260 | release(); | |
261 | } | |
262 | ||
263 | bool have_raw() const { return _raw ? true:false; } | |
264 | ||
265 | raw *clone(); | |
266 | void swap(ptr& other); | |
267 | ptr& make_shareable(); | |
268 | ||
269 | iterator begin(size_t offset=0) const { | |
270 | return iterator(this, offset, false); | |
271 | } | |
272 | iterator begin_deep(size_t offset=0) const { | |
273 | return iterator(this, offset, true); | |
274 | } | |
275 | ||
276 | // misc | |
277 | bool at_buffer_head() const { return _off == 0; } | |
278 | bool at_buffer_tail() const; | |
279 | ||
280 | bool is_aligned(unsigned align) const { | |
281 | return ((long)c_str() & (align-1)) == 0; | |
282 | } | |
283 | bool is_page_aligned() const { return is_aligned(CEPH_PAGE_SIZE); } | |
284 | bool is_n_align_sized(unsigned align) const | |
285 | { | |
286 | return (length() % align) == 0; | |
287 | } | |
288 | bool is_n_page_sized() const { return is_n_align_sized(CEPH_PAGE_SIZE); } | |
289 | bool is_partial() const { | |
290 | return have_raw() && (start() > 0 || end() < raw_length()); | |
291 | } | |
292 | ||
3efd9988 FG |
293 | int get_mempool() const; |
294 | void reassign_to_mempool(int pool); | |
295 | void try_assign_to_mempool(int pool); | |
296 | ||
7c673cae FG |
297 | // accessors |
298 | raw *get_raw() const { return _raw; } | |
299 | const char *c_str() const; | |
300 | char *c_str(); | |
301 | const char *end_c_str() const; | |
302 | char *end_c_str(); | |
303 | unsigned length() const { return _len; } | |
304 | unsigned offset() const { return _off; } | |
305 | unsigned start() const { return _off; } | |
306 | unsigned end() const { return _off + _len; } | |
307 | unsigned unused_tail_length() const; | |
308 | const char& operator[](unsigned n) const; | |
309 | char& operator[](unsigned n); | |
310 | ||
311 | const char *raw_c_str() const; | |
312 | unsigned raw_length() const; | |
313 | int raw_nref() const; | |
314 | ||
315 | void copy_out(unsigned o, unsigned l, char *dest) const; | |
316 | ||
317 | bool can_zero_copy() const; | |
318 | int zero_copy_to_fd(int fd, int64_t *offset) const; | |
319 | ||
31f18b77 | 320 | unsigned wasted() const; |
7c673cae FG |
321 | |
322 | int cmp(const ptr& o) const; | |
323 | bool is_zero() const; | |
324 | ||
325 | // modifiers | |
326 | void set_offset(unsigned o) { | |
327 | assert(raw_length() >= o); | |
328 | _off = o; | |
329 | } | |
330 | void set_length(unsigned l) { | |
331 | assert(raw_length() >= l); | |
332 | _len = l; | |
333 | } | |
334 | ||
335 | unsigned append(char c); | |
336 | unsigned append(const char *p, unsigned l); | |
337 | void copy_in(unsigned o, unsigned l, const char *src); | |
338 | void copy_in(unsigned o, unsigned l, const char *src, bool crc_reset); | |
339 | void zero(); | |
340 | void zero(bool crc_reset); | |
341 | void zero(unsigned o, unsigned l); | |
342 | void zero(unsigned o, unsigned l, bool crc_reset); | |
343 | ||
344 | }; | |
345 | ||
346 | ||
347 | /* | |
348 | * list - the useful bit! | |
349 | */ | |
350 | ||
351 | class CEPH_BUFFER_API list { | |
352 | // my private bits | |
353 | std::list<ptr> _buffers; | |
354 | unsigned _len; | |
355 | unsigned _memcopy_count; //the total of memcopy using rebuild(). | |
356 | ptr append_buffer; // where i put small appends. | |
357 | ||
358 | public: | |
359 | class iterator; | |
360 | ||
361 | private: | |
362 | template <bool is_const> | |
363 | class CEPH_BUFFER_API iterator_impl | |
364 | : public std::iterator<std::forward_iterator_tag, char> { | |
365 | protected: | |
366 | typedef typename std::conditional<is_const, | |
367 | const list, | |
368 | list>::type bl_t; | |
369 | typedef typename std::conditional<is_const, | |
370 | const std::list<ptr>, | |
371 | std::list<ptr> >::type list_t; | |
372 | typedef typename std::conditional<is_const, | |
373 | typename std::list<ptr>::const_iterator, | |
374 | typename std::list<ptr>::iterator>::type list_iter_t; | |
375 | bl_t* bl; | |
376 | list_t* ls; // meh.. just here to avoid an extra pointer dereference.. | |
377 | unsigned off; // in bl | |
378 | list_iter_t p; | |
379 | unsigned p_off; // in *p | |
380 | friend class iterator_impl<true>; | |
381 | ||
382 | public: | |
383 | // constructor. position. | |
384 | iterator_impl() | |
385 | : bl(0), ls(0), off(0), p_off(0) {} | |
386 | iterator_impl(bl_t *l, unsigned o=0); | |
387 | iterator_impl(bl_t *l, unsigned o, list_iter_t ip, unsigned po) | |
388 | : bl(l), ls(&bl->_buffers), off(o), p(ip), p_off(po) {} | |
389 | iterator_impl(const list::iterator& i); | |
390 | ||
391 | /// get current iterator offset in buffer::list | |
392 | unsigned get_off() const { return off; } | |
393 | ||
394 | /// get number of bytes remaining from iterator position to the end of the buffer::list | |
395 | unsigned get_remaining() const { return bl->length() - off; } | |
396 | ||
397 | /// true if iterator is at the end of the buffer::list | |
398 | bool end() const { | |
399 | return p == ls->end(); | |
400 | //return off == bl->length(); | |
401 | } | |
402 | ||
403 | void advance(int o); | |
404 | void seek(unsigned o); | |
405 | char operator*() const; | |
406 | iterator_impl& operator++(); | |
407 | ptr get_current_ptr() const; | |
408 | ||
409 | bl_t& get_bl() const { return *bl; } | |
410 | ||
411 | // copy data out. | |
412 | // note that these all _append_ to dest! | |
413 | void copy(unsigned len, char *dest); | |
414 | // deprecated, use copy_deep() | |
415 | void copy(unsigned len, ptr &dest) __attribute__((deprecated)); | |
416 | void copy_deep(unsigned len, ptr &dest); | |
417 | void copy_shallow(unsigned len, ptr &dest); | |
418 | void copy(unsigned len, list &dest); | |
419 | void copy(unsigned len, std::string &dest); | |
420 | void copy_all(list &dest); | |
421 | ||
422 | // get a pointer to the currenet iterator position, return the | |
423 | // number of bytes we can read from that position (up to want), | |
424 | // and advance the iterator by that amount. | |
425 | size_t get_ptr_and_advance(size_t want, const char **p); | |
426 | ||
427 | /// calculate crc from iterator position | |
428 | uint32_t crc32c(size_t length, uint32_t crc); | |
429 | ||
430 | friend bool operator==(const iterator_impl& lhs, | |
431 | const iterator_impl& rhs) { | |
432 | return &lhs.get_bl() == &rhs.get_bl() && lhs.get_off() == rhs.get_off(); | |
433 | } | |
434 | friend bool operator!=(const iterator_impl& lhs, | |
435 | const iterator_impl& rhs) { | |
436 | return &lhs.get_bl() != &rhs.get_bl() || lhs.get_off() != rhs.get_off(); | |
437 | } | |
438 | }; | |
439 | ||
440 | public: | |
441 | typedef iterator_impl<true> const_iterator; | |
442 | ||
443 | class CEPH_BUFFER_API iterator : public iterator_impl<false> { | |
444 | public: | |
445 | iterator() = default; | |
446 | iterator(bl_t *l, unsigned o=0); | |
447 | iterator(bl_t *l, unsigned o, list_iter_t ip, unsigned po); | |
448 | ||
449 | void advance(int o); | |
450 | void seek(unsigned o); | |
3efd9988 | 451 | using iterator_impl<false>::operator*; |
7c673cae FG |
452 | char operator*(); |
453 | iterator& operator++(); | |
454 | ptr get_current_ptr(); | |
455 | ||
456 | // copy data out | |
457 | void copy(unsigned len, char *dest); | |
458 | // deprecated, use copy_deep() | |
459 | void copy(unsigned len, ptr &dest) __attribute__((deprecated)); | |
460 | void copy_deep(unsigned len, ptr &dest); | |
461 | void copy_shallow(unsigned len, ptr &dest); | |
462 | void copy(unsigned len, list &dest); | |
463 | void copy(unsigned len, std::string &dest); | |
464 | void copy_all(list &dest); | |
465 | ||
466 | // copy data in | |
467 | void copy_in(unsigned len, const char *src); | |
468 | void copy_in(unsigned len, const char *src, bool crc_reset); | |
469 | void copy_in(unsigned len, const list& otherl); | |
470 | ||
471 | bool operator==(const iterator& rhs) const { | |
472 | return bl == rhs.bl && off == rhs.off; | |
473 | } | |
474 | bool operator!=(const iterator& rhs) const { | |
475 | return bl != rhs.bl || off != rhs.off; | |
476 | } | |
477 | }; | |
478 | ||
479 | class contiguous_appender { | |
480 | bufferlist *pbl; | |
481 | char *pos; | |
482 | ptr bp; | |
483 | bool deep; | |
484 | ||
485 | /// running count of bytes appended that are not reflected by @pos | |
486 | size_t out_of_band_offset = 0; | |
487 | ||
488 | contiguous_appender(bufferlist *l, size_t len, bool d) | |
489 | : pbl(l), | |
490 | deep(d) { | |
491 | size_t unused = pbl->append_buffer.unused_tail_length(); | |
492 | if (len > unused) { | |
493 | // note: if len < the normal append_buffer size it *might* | |
494 | // be better to allocate a normal-sized append_buffer and | |
495 | // use part of it. however, that optimizes for the case of | |
496 | // old-style types including new-style types. and in most | |
497 | // such cases, this won't be the very first thing encoded to | |
498 | // the list, so append_buffer will already be allocated. | |
499 | // OTOH if everything is new-style, we *should* allocate | |
500 | // only what we need and conserve memory. | |
501 | bp = buffer::create(len); | |
502 | pos = bp.c_str(); | |
503 | } else { | |
504 | pos = pbl->append_buffer.end_c_str(); | |
505 | } | |
506 | } | |
507 | ||
508 | void flush_and_continue() { | |
509 | if (bp.have_raw()) { | |
510 | // we allocated a new buffer | |
511 | size_t l = pos - bp.c_str(); | |
512 | pbl->append(bufferptr(bp, 0, l)); | |
513 | bp.set_length(bp.length() - l); | |
514 | bp.set_offset(bp.offset() + l); | |
515 | } else { | |
516 | // we are using pbl's append_buffer | |
517 | size_t l = pos - pbl->append_buffer.end_c_str(); | |
518 | if (l) { | |
519 | pbl->append_buffer.set_length(pbl->append_buffer.length() + l); | |
520 | pbl->append(pbl->append_buffer, pbl->append_buffer.end() - l, l); | |
521 | pos = pbl->append_buffer.end_c_str(); | |
522 | } | |
523 | } | |
524 | } | |
525 | ||
526 | friend class list; | |
527 | ||
528 | public: | |
529 | ~contiguous_appender() { | |
530 | if (bp.have_raw()) { | |
531 | // we allocated a new buffer | |
532 | bp.set_length(pos - bp.c_str()); | |
533 | pbl->append(std::move(bp)); | |
534 | } else { | |
535 | // we are using pbl's append_buffer | |
536 | size_t l = pos - pbl->append_buffer.end_c_str(); | |
537 | if (l) { | |
538 | pbl->append_buffer.set_length(pbl->append_buffer.length() + l); | |
539 | pbl->append(pbl->append_buffer, pbl->append_buffer.end() - l, l); | |
540 | } | |
541 | } | |
542 | } | |
543 | ||
544 | size_t get_out_of_band_offset() const { | |
545 | return out_of_band_offset; | |
546 | } | |
547 | void append(const char *p, size_t l) { | |
548 | maybe_inline_memcpy(pos, p, l, 16); | |
549 | pos += l; | |
550 | } | |
551 | char *get_pos_add(size_t len) { | |
552 | char *r = pos; | |
553 | pos += len; | |
554 | return r; | |
555 | } | |
556 | char *get_pos() { | |
557 | return pos; | |
558 | } | |
559 | ||
560 | void append(const bufferptr& p) { | |
561 | if (!p.length()) { | |
562 | return; | |
563 | } | |
564 | if (deep) { | |
565 | append(p.c_str(), p.length()); | |
566 | } else { | |
567 | flush_and_continue(); | |
568 | pbl->append(p); | |
569 | out_of_band_offset += p.length(); | |
570 | } | |
571 | } | |
572 | void append(const bufferlist& l) { | |
573 | if (!l.length()) { | |
574 | return; | |
575 | } | |
576 | if (deep) { | |
577 | for (const auto &p : l._buffers) { | |
578 | append(p.c_str(), p.length()); | |
579 | } | |
580 | } else { | |
581 | flush_and_continue(); | |
582 | pbl->append(l); | |
583 | out_of_band_offset += l.length(); | |
584 | } | |
585 | } | |
586 | ||
587 | size_t get_logical_offset() { | |
588 | if (bp.have_raw()) { | |
589 | return out_of_band_offset + (pos - bp.c_str()); | |
590 | } else { | |
591 | return out_of_band_offset + (pos - pbl->append_buffer.end_c_str()); | |
592 | } | |
593 | } | |
594 | }; | |
595 | ||
596 | contiguous_appender get_contiguous_appender(size_t len, bool deep=false) { | |
597 | return contiguous_appender(this, len, deep); | |
598 | } | |
599 | ||
600 | class page_aligned_appender { | |
601 | bufferlist *pbl; | |
602 | size_t offset; | |
603 | unsigned min_alloc; | |
604 | ptr buffer; | |
605 | char *pos, *end; | |
606 | ||
607 | page_aligned_appender(list *l, unsigned min_pages) | |
608 | : pbl(l), | |
609 | min_alloc(min_pages * CEPH_PAGE_SIZE), | |
610 | pos(nullptr), end(nullptr) {} | |
611 | ||
612 | friend class list; | |
613 | ||
614 | public: | |
615 | ~page_aligned_appender() { | |
616 | flush(); | |
617 | } | |
618 | ||
619 | void flush() { | |
620 | if (pos && pos != buffer.c_str()) { | |
621 | size_t len = pos - buffer.c_str(); | |
622 | pbl->append(buffer, 0, len); | |
623 | buffer.set_length(buffer.length() - len); | |
624 | buffer.set_offset(buffer.offset() + len); | |
625 | } | |
626 | } | |
627 | ||
628 | void append(const char *buf, size_t len) { | |
629 | while (len > 0) { | |
630 | if (!pos) { | |
631 | size_t alloc = (len + CEPH_PAGE_SIZE - 1) & CEPH_PAGE_MASK; | |
632 | if (alloc < min_alloc) { | |
633 | alloc = min_alloc; | |
634 | } | |
635 | buffer = create_page_aligned(alloc); | |
636 | pos = buffer.c_str(); | |
637 | end = buffer.end_c_str(); | |
638 | } | |
639 | size_t l = len; | |
640 | if (l > (size_t)(end - pos)) { | |
641 | l = end - pos; | |
642 | } | |
643 | memcpy(pos, buf, l); | |
644 | pos += l; | |
645 | buf += l; | |
646 | len -= l; | |
647 | if (pos == end) { | |
648 | pbl->append(buffer, 0, buffer.length()); | |
649 | pos = end = nullptr; | |
650 | } | |
651 | } | |
652 | } | |
653 | }; | |
654 | ||
655 | page_aligned_appender get_page_aligned_appender(unsigned min_pages=1) { | |
656 | return page_aligned_appender(this, min_pages); | |
657 | } | |
658 | ||
659 | private: | |
660 | mutable iterator last_p; | |
661 | int zero_copy_to_fd(int fd) const; | |
662 | ||
663 | public: | |
664 | // cons/des | |
665 | list() : _len(0), _memcopy_count(0), last_p(this) {} | |
666 | // cppcheck-suppress noExplicitConstructor | |
667 | list(unsigned prealloc) : _len(0), _memcopy_count(0), last_p(this) { | |
668 | reserve(prealloc); | |
669 | } | |
670 | ||
671 | list(const list& other) : _buffers(other._buffers), _len(other._len), | |
672 | _memcopy_count(other._memcopy_count), last_p(this) { | |
673 | make_shareable(); | |
674 | } | |
675 | list(list&& other); | |
676 | list& operator= (const list& other) { | |
677 | if (this != &other) { | |
678 | _buffers = other._buffers; | |
679 | _len = other._len; | |
680 | make_shareable(); | |
681 | } | |
682 | return *this; | |
683 | } | |
684 | ||
685 | list& operator= (list&& other) { | |
686 | _buffers = std::move(other._buffers); | |
687 | _len = other._len; | |
688 | _memcopy_count = other._memcopy_count; | |
689 | last_p = begin(); | |
690 | append_buffer.swap(other.append_buffer); | |
691 | other.clear(); | |
692 | return *this; | |
693 | } | |
694 | ||
695 | unsigned get_num_buffers() const { return _buffers.size(); } | |
696 | const ptr& front() const { return _buffers.front(); } | |
697 | const ptr& back() const { return _buffers.back(); } | |
698 | ||
3efd9988 | 699 | int get_mempool() const; |
31f18b77 FG |
700 | void reassign_to_mempool(int pool); |
701 | void try_assign_to_mempool(int pool); | |
702 | ||
703 | size_t get_append_buffer_unused_tail_length() const { | |
704 | return append_buffer.unused_tail_length(); | |
705 | } | |
706 | ||
7c673cae FG |
707 | unsigned get_memcopy_count() const {return _memcopy_count; } |
708 | const std::list<ptr>& buffers() const { return _buffers; } | |
709 | void swap(list& other); | |
710 | unsigned length() const { | |
711 | #if 0 | |
712 | // DEBUG: verify _len | |
713 | unsigned len = 0; | |
714 | for (std::list<ptr>::const_iterator it = _buffers.begin(); | |
715 | it != _buffers.end(); | |
716 | it++) { | |
717 | len += (*it).length(); | |
718 | } | |
719 | assert(len == _len); | |
720 | #endif | |
721 | return _len; | |
722 | } | |
723 | ||
724 | bool contents_equal(buffer::list& other); | |
725 | bool contents_equal(const buffer::list& other) const; | |
726 | ||
727 | bool can_zero_copy() const; | |
728 | bool is_provided_buffer(const char *dst) const; | |
729 | bool is_aligned(unsigned align) const; | |
730 | bool is_page_aligned() const; | |
731 | bool is_n_align_sized(unsigned align) const; | |
732 | bool is_n_page_sized() const; | |
733 | bool is_aligned_size_and_memory(unsigned align_size, | |
734 | unsigned align_memory) const; | |
735 | ||
736 | bool is_zero() const; | |
737 | ||
738 | // modifiers | |
739 | void clear() { | |
740 | _buffers.clear(); | |
741 | _len = 0; | |
742 | _memcopy_count = 0; | |
743 | last_p = begin(); | |
744 | append_buffer = ptr(); | |
745 | } | |
746 | void push_front(ptr& bp) { | |
747 | if (bp.length() == 0) | |
748 | return; | |
749 | _buffers.push_front(bp); | |
750 | _len += bp.length(); | |
751 | } | |
752 | void push_front(ptr&& bp) { | |
753 | if (bp.length() == 0) | |
754 | return; | |
755 | _len += bp.length(); | |
756 | _buffers.push_front(std::move(bp)); | |
757 | } | |
758 | void push_front(raw *r) { | |
759 | push_front(ptr(r)); | |
760 | } | |
761 | void push_back(const ptr& bp) { | |
762 | if (bp.length() == 0) | |
763 | return; | |
764 | _buffers.push_back(bp); | |
765 | _len += bp.length(); | |
766 | } | |
767 | void push_back(ptr&& bp) { | |
768 | if (bp.length() == 0) | |
769 | return; | |
770 | _len += bp.length(); | |
771 | _buffers.push_back(std::move(bp)); | |
772 | } | |
773 | void push_back(raw *r) { | |
774 | push_back(ptr(r)); | |
775 | } | |
776 | ||
777 | void zero(); | |
778 | void zero(unsigned o, unsigned l); | |
779 | ||
780 | bool is_contiguous() const; | |
781 | void rebuild(); | |
782 | void rebuild(ptr& nb); | |
783 | bool rebuild_aligned(unsigned align); | |
b32b8144 FG |
784 | // max_buffers = 0 mean don't care _buffers.size(), other |
785 | // must make _buffers.size() <= max_buffers after rebuilding. | |
7c673cae | 786 | bool rebuild_aligned_size_and_memory(unsigned align_size, |
b32b8144 FG |
787 | unsigned align_memory, |
788 | unsigned max_buffers = 0); | |
7c673cae FG |
789 | bool rebuild_page_aligned(); |
790 | ||
31f18b77 | 791 | void reserve(size_t prealloc); |
7c673cae FG |
792 | |
793 | // assignment-op with move semantics | |
794 | const static unsigned int CLAIM_DEFAULT = 0; | |
795 | const static unsigned int CLAIM_ALLOW_NONSHAREABLE = 1; | |
796 | ||
797 | void claim(list& bl, unsigned int flags = CLAIM_DEFAULT); | |
798 | void claim_append(list& bl, unsigned int flags = CLAIM_DEFAULT); | |
799 | void claim_prepend(list& bl, unsigned int flags = CLAIM_DEFAULT); | |
31f18b77 FG |
800 | // only for bl is bufferlist::page_aligned_appender |
801 | void claim_append_piecewise(list& bl); | |
7c673cae FG |
802 | |
803 | // clone non-shareable buffers (make shareable) | |
804 | void make_shareable() { | |
805 | std::list<buffer::ptr>::iterator pb; | |
806 | for (pb = _buffers.begin(); pb != _buffers.end(); ++pb) { | |
807 | (void) pb->make_shareable(); | |
808 | } | |
809 | } | |
810 | ||
811 | // copy with explicit volatile-sharing semantics | |
812 | void share(const list& bl) | |
813 | { | |
814 | if (this != &bl) { | |
815 | clear(); | |
816 | std::list<buffer::ptr>::const_iterator pb; | |
817 | for (pb = bl._buffers.begin(); pb != bl._buffers.end(); ++pb) { | |
818 | push_back(*pb); | |
819 | } | |
820 | } | |
821 | } | |
822 | ||
823 | iterator begin() { | |
824 | return iterator(this, 0); | |
825 | } | |
826 | iterator end() { | |
827 | return iterator(this, _len, _buffers.end(), 0); | |
828 | } | |
829 | ||
830 | const_iterator begin() const { | |
831 | return const_iterator(this, 0); | |
832 | } | |
833 | const_iterator end() const { | |
834 | return const_iterator(this, _len, _buffers.end(), 0); | |
835 | } | |
836 | ||
837 | // crope lookalikes. | |
838 | // **** WARNING: this are horribly inefficient for large bufferlists. **** | |
839 | void copy(unsigned off, unsigned len, char *dest) const; | |
840 | void copy(unsigned off, unsigned len, list &dest) const; | |
841 | void copy(unsigned off, unsigned len, std::string& dest) const; | |
842 | void copy_in(unsigned off, unsigned len, const char *src); | |
843 | void copy_in(unsigned off, unsigned len, const char *src, bool crc_reset); | |
844 | void copy_in(unsigned off, unsigned len, const list& src); | |
845 | ||
846 | void append(char c); | |
847 | void append(const char *data, unsigned len); | |
848 | void append(const std::string& s) { | |
849 | append(s.data(), s.length()); | |
850 | } | |
851 | void append(const ptr& bp); | |
852 | void append(ptr&& bp); | |
853 | void append(const ptr& bp, unsigned off, unsigned len); | |
854 | void append(const list& bl); | |
855 | void append(std::istream& in); | |
856 | void append_zero(unsigned len); | |
857 | void prepend_zero(unsigned len); | |
858 | ||
859 | /* | |
860 | * get a char | |
861 | */ | |
862 | const char& operator[](unsigned n) const; | |
863 | char *c_str(); | |
864 | std::string to_str() const; | |
865 | ||
866 | void substr_of(const list& other, unsigned off, unsigned len); | |
867 | ||
868 | /// return a pointer to a contiguous extent of the buffer, | |
869 | /// reallocating as needed | |
870 | char *get_contiguous(unsigned off, ///< offset | |
871 | unsigned len); ///< length | |
872 | ||
873 | // funky modifer | |
874 | void splice(unsigned off, unsigned len, list *claim_by=0 /*, bufferlist& replace_with */); | |
875 | void write(int off, int len, std::ostream& out) const; | |
876 | ||
877 | void encode_base64(list& o); | |
878 | void decode_base64(list& o); | |
879 | ||
880 | void write_stream(std::ostream &out) const; | |
881 | void hexdump(std::ostream &out, bool trailing_newline = true) const; | |
882 | int read_file(const char *fn, std::string *error); | |
883 | ssize_t read_fd(int fd, size_t len); | |
884 | int read_fd_zero_copy(int fd, size_t len); | |
885 | int write_file(const char *fn, int mode=0644); | |
886 | int write_fd(int fd) const; | |
887 | int write_fd(int fd, uint64_t offset) const; | |
888 | int write_fd_zero_copy(int fd) const; | |
889 | template<typename VectorT> | |
890 | void prepare_iov(VectorT *piov) const { | |
891 | assert(_buffers.size() <= IOV_MAX); | |
892 | piov->resize(_buffers.size()); | |
893 | unsigned n = 0; | |
894 | for (auto& p : _buffers) { | |
895 | (*piov)[n].iov_base = (void *)p.c_str(); | |
896 | (*piov)[n].iov_len = p.length(); | |
897 | ++n; | |
898 | } | |
899 | } | |
900 | uint32_t crc32c(uint32_t crc) const; | |
901 | void invalidate_crc(); | |
31f18b77 FG |
902 | |
903 | // These functions return a bufferlist with a pointer to a single | |
904 | // static buffer. They /must/ not outlive the memory they | |
905 | // reference. | |
906 | static list static_from_mem(char* c, size_t l); | |
907 | static list static_from_cstring(char* c); | |
908 | static list static_from_string(std::string& s); | |
7c673cae FG |
909 | }; |
910 | ||
911 | /* | |
912 | * efficient hash of one or more bufferlists | |
913 | */ | |
914 | ||
915 | class hash { | |
916 | uint32_t crc; | |
917 | ||
918 | public: | |
919 | hash() : crc(0) { } | |
920 | // cppcheck-suppress noExplicitConstructor | |
921 | hash(uint32_t init) : crc(init) { } | |
922 | ||
31f18b77 | 923 | void update(const buffer::list& bl) { |
7c673cae FG |
924 | crc = bl.crc32c(crc); |
925 | } | |
926 | ||
927 | uint32_t digest() { | |
928 | return crc; | |
929 | } | |
930 | }; | |
931 | ||
932 | inline bool operator>(bufferlist& l, bufferlist& r) { | |
933 | for (unsigned p = 0; ; p++) { | |
934 | if (l.length() > p && r.length() == p) return true; | |
935 | if (l.length() == p) return false; | |
936 | if (l[p] > r[p]) return true; | |
937 | if (l[p] < r[p]) return false; | |
938 | } | |
939 | } | |
940 | inline bool operator>=(bufferlist& l, bufferlist& r) { | |
941 | for (unsigned p = 0; ; p++) { | |
942 | if (l.length() > p && r.length() == p) return true; | |
943 | if (r.length() == p && l.length() == p) return true; | |
944 | if (l.length() == p && r.length() > p) return false; | |
945 | if (l[p] > r[p]) return true; | |
946 | if (l[p] < r[p]) return false; | |
947 | } | |
948 | } | |
949 | ||
950 | inline bool operator==(const bufferlist &l, const bufferlist &r) { | |
951 | if (l.length() != r.length()) | |
952 | return false; | |
953 | for (unsigned p = 0; p < l.length(); p++) { | |
954 | if (l[p] != r[p]) | |
955 | return false; | |
956 | } | |
957 | return true; | |
958 | } | |
959 | inline bool operator<(bufferlist& l, bufferlist& r) { | |
960 | return r > l; | |
961 | } | |
962 | inline bool operator<=(bufferlist& l, bufferlist& r) { | |
963 | return r >= l; | |
964 | } | |
965 | ||
966 | ||
967 | std::ostream& operator<<(std::ostream& out, const buffer::ptr& bp); | |
968 | ||
969 | std::ostream& operator<<(std::ostream& out, const raw &r); | |
970 | ||
971 | std::ostream& operator<<(std::ostream& out, const buffer::list& bl); | |
972 | ||
973 | std::ostream& operator<<(std::ostream& out, const buffer::error& e); | |
974 | ||
31f18b77 | 975 | inline bufferhash& operator<<(bufferhash& l, const bufferlist &r) { |
7c673cae FG |
976 | l.update(r); |
977 | return l; | |
978 | } | |
979 | ||
980 | } | |
981 | ||
982 | #if defined(HAVE_XIO) | |
983 | xio_reg_mem* get_xio_mp(const buffer::ptr& bp); | |
984 | #endif | |
985 | ||
986 | } | |
987 | ||
988 | #endif |