]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
31f18b77 | 15 | #include <atomic> |
9f95a23c | 16 | #include <cstring> |
31f18b77 FG |
17 | #include <errno.h> |
18 | #include <limits.h> | |
19 | ||
20 | #include <sys/uio.h> | |
21 | ||
11fdf7f2 TL |
22 | #include "include/ceph_assert.h" |
23 | #include "include/types.h" | |
24 | #include "include/buffer_raw.h" | |
7c673cae FG |
25 | #include "include/compat.h" |
26 | #include "include/mempool.h" | |
27 | #include "armor.h" | |
28 | #include "common/environment.h" | |
29 | #include "common/errno.h" | |
30 | #include "common/safe_io.h" | |
7c673cae FG |
31 | #include "common/strtol.h" |
32 | #include "common/likely.h" | |
33 | #include "common/valgrind.h" | |
34 | #include "common/deleter.h" | |
7c673cae | 35 | #include "common/RWLock.h" |
11fdf7f2 | 36 | #include "include/spinlock.h" |
7c673cae | 37 | #include "include/scope_guard.h" |
31f18b77 | 38 | |
31f18b77 | 39 | using namespace ceph; |
7c673cae | 40 | |
11fdf7f2 | 41 | #define CEPH_BUFFER_ALLOC_UNIT 4096u |
7c673cae FG |
42 | #define CEPH_BUFFER_APPEND_SIZE (CEPH_BUFFER_ALLOC_UNIT - sizeof(raw_combined)) |
43 | ||
44 | #ifdef BUFFER_DEBUG | |
11fdf7f2 TL |
45 | static ceph::spinlock debug_lock; |
46 | # define bdout { std::lock_guard<ceph::spinlock> lg(debug_lock); std::cout | |
47 | # define bendl std::endl; } | |
7c673cae FG |
48 | #else |
49 | # define bdout if (0) { std::cout | |
50 | # define bendl std::endl; } | |
51 | #endif | |
52 | ||
9f95a23c TL |
53 | static ceph::atomic<unsigned> buffer_cached_crc { 0 }; |
54 | static ceph::atomic<unsigned> buffer_cached_crc_adjusted { 0 }; | |
55 | static ceph::atomic<unsigned> buffer_missed_crc { 0 }; | |
31f18b77 | 56 | |
7c673cae FG |
57 | static bool buffer_track_crc = get_env_bool("CEPH_BUFFER_TRACK"); |
58 | ||
59 | void buffer::track_cached_crc(bool b) { | |
60 | buffer_track_crc = b; | |
61 | } | |
62 | int buffer::get_cached_crc() { | |
31f18b77 | 63 | return buffer_cached_crc; |
7c673cae FG |
64 | } |
65 | int buffer::get_cached_crc_adjusted() { | |
31f18b77 | 66 | return buffer_cached_crc_adjusted; |
7c673cae FG |
67 | } |
68 | ||
69 | int buffer::get_missed_crc() { | |
31f18b77 | 70 | return buffer_missed_crc; |
7c673cae FG |
71 | } |
72 | ||
7c673cae FG |
73 | const char * buffer::error::what() const throw () { |
74 | return "buffer::exception"; | |
75 | } | |
76 | const char * buffer::bad_alloc::what() const throw () { | |
77 | return "buffer::bad_alloc"; | |
78 | } | |
79 | const char * buffer::end_of_buffer::what() const throw () { | |
80 | return "buffer::end_of_buffer"; | |
81 | } | |
82 | const char * buffer::malformed_input::what() const throw () { | |
83 | return buf; | |
84 | } | |
85 | buffer::error_code::error_code(int error) : | |
86 | buffer::malformed_input(cpp_strerror(error).c_str()), code(error) {} | |
87 | ||
7c673cae FG |
88 | /* |
89 | * raw_combined is always placed within a single allocation along | |
90 | * with the data buffer. the data goes at the beginning, and | |
91 | * raw_combined at the end. | |
92 | */ | |
93 | class buffer::raw_combined : public buffer::raw { | |
94 | size_t alignment; | |
95 | public: | |
3efd9988 FG |
96 | raw_combined(char *dataptr, unsigned l, unsigned align, |
97 | int mempool) | |
98 | : raw(dataptr, l, mempool), | |
7c673cae | 99 | alignment(align) { |
7c673cae FG |
100 | } |
101 | raw* clone_empty() override { | |
9f95a23c | 102 | return create(len, alignment).release(); |
7c673cae FG |
103 | } |
104 | ||
9f95a23c TL |
105 | static ceph::unique_leakable_ptr<buffer::raw> |
106 | create(unsigned len, | |
107 | unsigned align, | |
108 | int mempool = mempool::mempool_buffer_anon) | |
109 | { | |
7c673cae FG |
110 | if (!align) |
111 | align = sizeof(size_t); | |
11fdf7f2 | 112 | size_t rawlen = round_up_to(sizeof(buffer::raw_combined), |
7c673cae | 113 | alignof(buffer::raw_combined)); |
11fdf7f2 | 114 | size_t datalen = round_up_to(len, alignof(buffer::raw_combined)); |
7c673cae | 115 | |
31f18b77 FG |
116 | #ifdef DARWIN |
117 | char *ptr = (char *) valloc(rawlen + datalen); | |
118 | #else | |
119 | char *ptr = 0; | |
120 | int r = ::posix_memalign((void**)(void*)&ptr, align, rawlen + datalen); | |
121 | if (r) | |
122 | throw bad_alloc(); | |
123 | #endif /* DARWIN */ | |
7c673cae FG |
124 | if (!ptr) |
125 | throw bad_alloc(); | |
126 | ||
127 | // actual data first, since it has presumably larger alignment restriction | |
128 | // then put the raw_combined at the end | |
9f95a23c TL |
129 | return ceph::unique_leakable_ptr<buffer::raw>( |
130 | new (ptr + datalen) raw_combined(ptr, len, align, mempool)); | |
7c673cae FG |
131 | } |
132 | ||
133 | static void operator delete(void *ptr) { | |
134 | raw_combined *raw = (raw_combined *)ptr; | |
31f18b77 | 135 | ::free((void *)raw->data); |
7c673cae FG |
136 | } |
137 | }; | |
138 | ||
139 | class buffer::raw_malloc : public buffer::raw { | |
140 | public: | |
141 | MEMPOOL_CLASS_HELPERS(); | |
142 | ||
143 | explicit raw_malloc(unsigned l) : raw(l) { | |
144 | if (len) { | |
145 | data = (char *)malloc(len); | |
146 | if (!data) | |
147 | throw bad_alloc(); | |
148 | } else { | |
149 | data = 0; | |
150 | } | |
11fdf7f2 | 151 | bdout << "raw_malloc " << this << " alloc " << (void *)data << " " << l << bendl; |
7c673cae FG |
152 | } |
153 | raw_malloc(unsigned l, char *b) : raw(b, l) { | |
11fdf7f2 | 154 | bdout << "raw_malloc " << this << " alloc " << (void *)data << " " << l << bendl; |
7c673cae FG |
155 | } |
156 | ~raw_malloc() override { | |
157 | free(data); | |
11fdf7f2 | 158 | bdout << "raw_malloc " << this << " free " << (void *)data << " " << bendl; |
7c673cae FG |
159 | } |
160 | raw* clone_empty() override { | |
161 | return new raw_malloc(len); | |
162 | } | |
163 | }; | |
164 | ||
165 | #ifndef __CYGWIN__ | |
7c673cae FG |
166 | class buffer::raw_posix_aligned : public buffer::raw { |
167 | unsigned align; | |
168 | public: | |
169 | MEMPOOL_CLASS_HELPERS(); | |
170 | ||
171 | raw_posix_aligned(unsigned l, unsigned _align) : raw(l) { | |
172 | align = _align; | |
11fdf7f2 | 173 | ceph_assert((align >= sizeof(void *)) && (align & (align - 1)) == 0); |
31f18b77 FG |
174 | #ifdef DARWIN |
175 | data = (char *) valloc(len); | |
176 | #else | |
177 | int r = ::posix_memalign((void**)(void*)&data, align, len); | |
178 | if (r) | |
179 | throw bad_alloc(); | |
180 | #endif /* DARWIN */ | |
7c673cae FG |
181 | if (!data) |
182 | throw bad_alloc(); | |
11fdf7f2 TL |
183 | bdout << "raw_posix_aligned " << this << " alloc " << (void *)data |
184 | << " l=" << l << ", align=" << align << bendl; | |
7c673cae FG |
185 | } |
186 | ~raw_posix_aligned() override { | |
31f18b77 | 187 | ::free(data); |
11fdf7f2 | 188 | bdout << "raw_posix_aligned " << this << " free " << (void *)data << bendl; |
7c673cae FG |
189 | } |
190 | raw* clone_empty() override { | |
191 | return new raw_posix_aligned(len, align); | |
192 | } | |
193 | }; | |
194 | #endif | |
195 | ||
196 | #ifdef __CYGWIN__ | |
197 | class buffer::raw_hack_aligned : public buffer::raw { | |
198 | unsigned align; | |
199 | char *realdata; | |
200 | public: | |
201 | raw_hack_aligned(unsigned l, unsigned _align) : raw(l) { | |
202 | align = _align; | |
203 | realdata = new char[len+align-1]; | |
204 | unsigned off = ((unsigned)realdata) & (align-1); | |
205 | if (off) | |
206 | data = realdata + align - off; | |
207 | else | |
208 | data = realdata; | |
7c673cae FG |
209 | //cout << "hack aligned " << (unsigned)data |
210 | //<< " in raw " << (unsigned)realdata | |
211 | //<< " off " << off << std::endl; | |
11fdf7f2 | 212 | ceph_assert(((unsigned)data & (align-1)) == 0); |
7c673cae FG |
213 | } |
214 | ~raw_hack_aligned() { | |
215 | delete[] realdata; | |
7c673cae FG |
216 | } |
217 | raw* clone_empty() { | |
218 | return new raw_hack_aligned(len, align); | |
219 | } | |
220 | }; | |
221 | #endif | |
222 | ||
7c673cae FG |
223 | /* |
224 | * primitive buffer types | |
225 | */ | |
226 | class buffer::raw_char : public buffer::raw { | |
227 | public: | |
228 | MEMPOOL_CLASS_HELPERS(); | |
229 | ||
230 | explicit raw_char(unsigned l) : raw(l) { | |
231 | if (len) | |
31f18b77 | 232 | data = new char[len]; |
7c673cae FG |
233 | else |
234 | data = 0; | |
11fdf7f2 | 235 | bdout << "raw_char " << this << " alloc " << (void *)data << " " << l << bendl; |
7c673cae FG |
236 | } |
237 | raw_char(unsigned l, char *b) : raw(b, l) { | |
11fdf7f2 | 238 | bdout << "raw_char " << this << " alloc " << (void *)data << " " << l << bendl; |
7c673cae FG |
239 | } |
240 | ~raw_char() override { | |
31f18b77 | 241 | delete[] data; |
11fdf7f2 | 242 | bdout << "raw_char " << this << " free " << (void *)data << bendl; |
7c673cae FG |
243 | } |
244 | raw* clone_empty() override { | |
245 | return new raw_char(len); | |
246 | } | |
247 | }; | |
248 | ||
31f18b77 FG |
249 | class buffer::raw_claimed_char : public buffer::raw { |
250 | public: | |
251 | MEMPOOL_CLASS_HELPERS(); | |
252 | ||
253 | explicit raw_claimed_char(unsigned l, char *b) : raw(b, l) { | |
31f18b77 | 254 | bdout << "raw_claimed_char " << this << " alloc " << (void *)data |
11fdf7f2 | 255 | << " " << l << bendl; |
31f18b77 FG |
256 | } |
257 | ~raw_claimed_char() override { | |
31f18b77 | 258 | bdout << "raw_claimed_char " << this << " free " << (void *)data |
11fdf7f2 | 259 | << bendl; |
31f18b77 FG |
260 | } |
261 | raw* clone_empty() override { | |
262 | return new raw_char(len); | |
263 | } | |
264 | }; | |
265 | ||
7c673cae FG |
266 | class buffer::raw_static : public buffer::raw { |
267 | public: | |
268 | MEMPOOL_CLASS_HELPERS(); | |
269 | ||
270 | raw_static(const char *d, unsigned l) : raw((char*)d, l) { } | |
271 | ~raw_static() override {} | |
272 | raw* clone_empty() override { | |
273 | return new buffer::raw_char(len); | |
274 | } | |
275 | }; | |
276 | ||
277 | class buffer::raw_claim_buffer : public buffer::raw { | |
278 | deleter del; | |
279 | public: | |
280 | raw_claim_buffer(const char *b, unsigned l, deleter d) | |
281 | : raw((char*)b, l), del(std::move(d)) { } | |
282 | ~raw_claim_buffer() override {} | |
283 | raw* clone_empty() override { | |
284 | return new buffer::raw_char(len); | |
285 | } | |
286 | }; | |
287 | ||
11fdf7f2 TL |
288 | ceph::unique_leakable_ptr<buffer::raw> buffer::copy(const char *c, unsigned len) { |
289 | auto r = buffer::create_aligned(len, sizeof(size_t)); | |
7c673cae FG |
290 | memcpy(r->data, c, len); |
291 | return r; | |
292 | } | |
293 | ||
11fdf7f2 | 294 | ceph::unique_leakable_ptr<buffer::raw> buffer::create(unsigned len) { |
7c673cae FG |
295 | return buffer::create_aligned(len, sizeof(size_t)); |
296 | } | |
9f95a23c TL |
297 | ceph::unique_leakable_ptr<buffer::raw> |
298 | buffer::create_in_mempool(unsigned len, int mempool) { | |
3efd9988 FG |
299 | return buffer::create_aligned_in_mempool(len, sizeof(size_t), mempool); |
300 | } | |
9f95a23c TL |
301 | ceph::unique_leakable_ptr<buffer::raw> |
302 | buffer::claim_char(unsigned len, char *buf) { | |
303 | return ceph::unique_leakable_ptr<buffer::raw>( | |
304 | new raw_claimed_char(len, buf)); | |
7c673cae | 305 | } |
9f95a23c TL |
306 | ceph::unique_leakable_ptr<buffer::raw> buffer::create_malloc(unsigned len) { |
307 | return ceph::unique_leakable_ptr<buffer::raw>(new raw_malloc(len)); | |
7c673cae | 308 | } |
9f95a23c TL |
309 | ceph::unique_leakable_ptr<buffer::raw> |
310 | buffer::claim_malloc(unsigned len, char *buf) { | |
311 | return ceph::unique_leakable_ptr<buffer::raw>(new raw_malloc(len, buf)); | |
7c673cae | 312 | } |
9f95a23c TL |
313 | ceph::unique_leakable_ptr<buffer::raw> |
314 | buffer::create_static(unsigned len, char *buf) { | |
315 | return ceph::unique_leakable_ptr<buffer::raw>(new raw_static(buf, len)); | |
7c673cae | 316 | } |
9f95a23c TL |
317 | ceph::unique_leakable_ptr<buffer::raw> |
318 | buffer::claim_buffer(unsigned len, char *buf, deleter del) { | |
319 | return ceph::unique_leakable_ptr<buffer::raw>( | |
320 | new raw_claim_buffer(buf, len, std::move(del))); | |
7c673cae FG |
321 | } |
322 | ||
11fdf7f2 | 323 | ceph::unique_leakable_ptr<buffer::raw> buffer::create_aligned_in_mempool( |
9f95a23c TL |
324 | unsigned len, unsigned align, int mempool) |
325 | { | |
7c673cae FG |
326 | // If alignment is a page multiple, use a separate buffer::raw to |
327 | // avoid fragmenting the heap. | |
328 | // | |
329 | // Somewhat unexpectedly, I see consistently better performance | |
330 | // from raw_combined than from raw even when the allocation size is | |
331 | // a page multiple (but alignment is not). | |
332 | // | |
333 | // I also see better performance from a separate buffer::raw once the | |
334 | // size passes 8KB. | |
335 | if ((align & ~CEPH_PAGE_MASK) == 0 || | |
336 | len >= CEPH_PAGE_SIZE * 2) { | |
337 | #ifndef __CYGWIN__ | |
11fdf7f2 | 338 | return ceph::unique_leakable_ptr<buffer::raw>(new raw_posix_aligned(len, align)); |
7c673cae | 339 | #else |
11fdf7f2 | 340 | return ceph::unique_leakable_ptr<buffer::raw>(new raw_hack_aligned(len, align)); |
7c673cae FG |
341 | #endif |
342 | } | |
9f95a23c | 343 | return raw_combined::create(len, align, mempool); |
3efd9988 | 344 | } |
11fdf7f2 | 345 | ceph::unique_leakable_ptr<buffer::raw> buffer::create_aligned( |
3efd9988 FG |
346 | unsigned len, unsigned align) { |
347 | return create_aligned_in_mempool(len, align, | |
348 | mempool::mempool_buffer_anon); | |
7c673cae FG |
349 | } |
350 | ||
11fdf7f2 | 351 | ceph::unique_leakable_ptr<buffer::raw> buffer::create_page_aligned(unsigned len) { |
7c673cae FG |
352 | return create_aligned(len, CEPH_PAGE_SIZE); |
353 | } | |
11fdf7f2 TL |
354 | ceph::unique_leakable_ptr<buffer::raw> buffer::create_small_page_aligned(unsigned len) { |
355 | if (len < CEPH_PAGE_SIZE) { | |
356 | return create_aligned(len, CEPH_BUFFER_ALLOC_UNIT); | |
9f95a23c | 357 | } else { |
11fdf7f2 | 358 | return create_aligned(len, CEPH_PAGE_SIZE); |
9f95a23c | 359 | } |
7c673cae FG |
360 | } |
361 | ||
11fdf7f2 TL |
362 | buffer::ptr::ptr(ceph::unique_leakable_ptr<raw> r) |
363 | : _raw(r.release()), | |
364 | _off(0), | |
365 | _len(_raw->len) | |
366 | { | |
367 | _raw->nref.store(1, std::memory_order_release); | |
368 | bdout << "ptr " << this << " get " << _raw << bendl; | |
369 | } | |
7c673cae FG |
370 | buffer::ptr::ptr(unsigned l) : _off(0), _len(l) |
371 | { | |
11fdf7f2 TL |
372 | _raw = buffer::create(l).release(); |
373 | _raw->nref.store(1, std::memory_order_release); | |
7c673cae FG |
374 | bdout << "ptr " << this << " get " << _raw << bendl; |
375 | } | |
376 | buffer::ptr::ptr(const char *d, unsigned l) : _off(0), _len(l) // ditto. | |
377 | { | |
11fdf7f2 TL |
378 | _raw = buffer::copy(d, l).release(); |
379 | _raw->nref.store(1, std::memory_order_release); | |
7c673cae FG |
380 | bdout << "ptr " << this << " get " << _raw << bendl; |
381 | } | |
382 | buffer::ptr::ptr(const ptr& p) : _raw(p._raw), _off(p._off), _len(p._len) | |
383 | { | |
384 | if (_raw) { | |
31f18b77 | 385 | _raw->nref++; |
7c673cae FG |
386 | bdout << "ptr " << this << " get " << _raw << bendl; |
387 | } | |
388 | } | |
389 | buffer::ptr::ptr(ptr&& p) noexcept : _raw(p._raw), _off(p._off), _len(p._len) | |
390 | { | |
391 | p._raw = nullptr; | |
392 | p._off = p._len = 0; | |
393 | } | |
394 | buffer::ptr::ptr(const ptr& p, unsigned o, unsigned l) | |
395 | : _raw(p._raw), _off(p._off + o), _len(l) | |
396 | { | |
11fdf7f2 TL |
397 | ceph_assert(o+l <= p._len); |
398 | ceph_assert(_raw); | |
31f18b77 | 399 | _raw->nref++; |
7c673cae FG |
400 | bdout << "ptr " << this << " get " << _raw << bendl; |
401 | } | |
11fdf7f2 TL |
402 | buffer::ptr::ptr(const ptr& p, ceph::unique_leakable_ptr<raw> r) |
403 | : _raw(r.release()), | |
404 | _off(p._off), | |
405 | _len(p._len) | |
406 | { | |
407 | _raw->nref.store(1, std::memory_order_release); | |
408 | bdout << "ptr " << this << " get " << _raw << bendl; | |
409 | } | |
7c673cae FG |
410 | buffer::ptr& buffer::ptr::operator= (const ptr& p) |
411 | { | |
412 | if (p._raw) { | |
31f18b77 | 413 | p._raw->nref++; |
7c673cae FG |
414 | bdout << "ptr " << this << " get " << _raw << bendl; |
415 | } | |
416 | buffer::raw *raw = p._raw; | |
417 | release(); | |
418 | if (raw) { | |
419 | _raw = raw; | |
420 | _off = p._off; | |
421 | _len = p._len; | |
422 | } else { | |
423 | _off = _len = 0; | |
424 | } | |
425 | return *this; | |
426 | } | |
427 | buffer::ptr& buffer::ptr::operator= (ptr&& p) noexcept | |
428 | { | |
429 | release(); | |
430 | buffer::raw *raw = p._raw; | |
431 | if (raw) { | |
432 | _raw = raw; | |
433 | _off = p._off; | |
434 | _len = p._len; | |
435 | p._raw = nullptr; | |
436 | p._off = p._len = 0; | |
437 | } else { | |
438 | _off = _len = 0; | |
439 | } | |
440 | return *this; | |
441 | } | |
442 | ||
11fdf7f2 | 443 | ceph::unique_leakable_ptr<buffer::raw> buffer::ptr::clone() |
7c673cae FG |
444 | { |
445 | return _raw->clone(); | |
446 | } | |
447 | ||
11fdf7f2 | 448 | void buffer::ptr::swap(ptr& other) noexcept |
7c673cae FG |
449 | { |
450 | raw *r = _raw; | |
451 | unsigned o = _off; | |
452 | unsigned l = _len; | |
453 | _raw = other._raw; | |
454 | _off = other._off; | |
455 | _len = other._len; | |
456 | other._raw = r; | |
457 | other._off = o; | |
458 | other._len = l; | |
459 | } | |
460 | ||
461 | void buffer::ptr::release() | |
462 | { | |
463 | if (_raw) { | |
464 | bdout << "ptr " << this << " release " << _raw << bendl; | |
11fdf7f2 TL |
465 | const bool last_one = (1 == _raw->nref.load(std::memory_order_acquire)); |
466 | if (likely(last_one) || --_raw->nref == 0) { | |
467 | // BE CAREFUL: this is called also for hypercombined ptr_node. After | |
468 | // freeing underlying raw, `*this` can become inaccessible as well! | |
469 | const auto* delete_raw = _raw; | |
470 | _raw = nullptr; | |
7c673cae | 471 | //cout << "hosing raw " << (void*)_raw << " len " << _raw->len << std::endl; |
9f95a23c TL |
472 | ANNOTATE_HAPPENS_AFTER(&delete_raw->nref); |
473 | ANNOTATE_HAPPENS_BEFORE_FORGET_ALL(&delete_raw->nref); | |
11fdf7f2 | 474 | delete delete_raw; // dealloc old (if any) |
7c673cae FG |
475 | } else { |
476 | ANNOTATE_HAPPENS_BEFORE(&_raw->nref); | |
11fdf7f2 | 477 | _raw = nullptr; |
7c673cae | 478 | } |
7c673cae FG |
479 | } |
480 | } | |
481 | ||
3efd9988 FG |
482 | int buffer::ptr::get_mempool() const { |
483 | if (_raw) { | |
484 | return _raw->mempool; | |
485 | } | |
486 | return mempool::mempool_buffer_anon; | |
487 | } | |
488 | ||
489 | void buffer::ptr::reassign_to_mempool(int pool) { | |
490 | if (_raw) { | |
491 | _raw->reassign_to_mempool(pool); | |
492 | } | |
493 | } | |
494 | void buffer::ptr::try_assign_to_mempool(int pool) { | |
495 | if (_raw) { | |
496 | _raw->try_assign_to_mempool(pool); | |
497 | } | |
498 | } | |
499 | ||
7c673cae | 500 | const char *buffer::ptr::c_str() const { |
11fdf7f2 | 501 | ceph_assert(_raw); |
7c673cae FG |
502 | return _raw->get_data() + _off; |
503 | } | |
504 | char *buffer::ptr::c_str() { | |
11fdf7f2 | 505 | ceph_assert(_raw); |
7c673cae FG |
506 | return _raw->get_data() + _off; |
507 | } | |
508 | const char *buffer::ptr::end_c_str() const { | |
11fdf7f2 | 509 | ceph_assert(_raw); |
7c673cae FG |
510 | return _raw->get_data() + _off + _len; |
511 | } | |
512 | char *buffer::ptr::end_c_str() { | |
11fdf7f2 | 513 | ceph_assert(_raw); |
7c673cae FG |
514 | return _raw->get_data() + _off + _len; |
515 | } | |
516 | ||
517 | unsigned buffer::ptr::unused_tail_length() const | |
518 | { | |
519 | if (_raw) | |
520 | return _raw->len - (_off+_len); | |
521 | else | |
522 | return 0; | |
523 | } | |
524 | const char& buffer::ptr::operator[](unsigned n) const | |
525 | { | |
11fdf7f2 TL |
526 | ceph_assert(_raw); |
527 | ceph_assert(n < _len); | |
7c673cae FG |
528 | return _raw->get_data()[_off + n]; |
529 | } | |
530 | char& buffer::ptr::operator[](unsigned n) | |
531 | { | |
11fdf7f2 TL |
532 | ceph_assert(_raw); |
533 | ceph_assert(n < _len); | |
7c673cae FG |
534 | return _raw->get_data()[_off + n]; |
535 | } | |
536 | ||
11fdf7f2 TL |
537 | const char *buffer::ptr::raw_c_str() const { ceph_assert(_raw); return _raw->data; } |
538 | unsigned buffer::ptr::raw_length() const { ceph_assert(_raw); return _raw->len; } | |
539 | int buffer::ptr::raw_nref() const { ceph_assert(_raw); return _raw->nref; } | |
7c673cae FG |
540 | |
541 | void buffer::ptr::copy_out(unsigned o, unsigned l, char *dest) const { | |
11fdf7f2 | 542 | ceph_assert(_raw); |
7c673cae FG |
543 | if (o+l > _len) |
544 | throw end_of_buffer(); | |
545 | char* src = _raw->data + _off + o; | |
546 | maybe_inline_memcpy(dest, src, l, 8); | |
547 | } | |
548 | ||
31f18b77 | 549 | unsigned buffer::ptr::wasted() const |
7c673cae | 550 | { |
7c673cae FG |
551 | return _raw->len - _len; |
552 | } | |
553 | ||
554 | int buffer::ptr::cmp(const ptr& o) const | |
555 | { | |
556 | int l = _len < o._len ? _len : o._len; | |
557 | if (l) { | |
558 | int r = memcmp(c_str(), o.c_str(), l); | |
559 | if (r) | |
560 | return r; | |
561 | } | |
562 | if (_len < o._len) | |
563 | return -1; | |
564 | if (_len > o._len) | |
565 | return 1; | |
566 | return 0; | |
567 | } | |
568 | ||
569 | bool buffer::ptr::is_zero() const | |
570 | { | |
571 | return mem_is_zero(c_str(), _len); | |
572 | } | |
573 | ||
574 | unsigned buffer::ptr::append(char c) | |
575 | { | |
11fdf7f2 TL |
576 | ceph_assert(_raw); |
577 | ceph_assert(1 <= unused_tail_length()); | |
7c673cae FG |
578 | char* ptr = _raw->data + _off + _len; |
579 | *ptr = c; | |
580 | _len++; | |
581 | return _len + _off; | |
582 | } | |
583 | ||
584 | unsigned buffer::ptr::append(const char *p, unsigned l) | |
585 | { | |
11fdf7f2 TL |
586 | ceph_assert(_raw); |
587 | ceph_assert(l <= unused_tail_length()); | |
7c673cae FG |
588 | char* c = _raw->data + _off + _len; |
589 | maybe_inline_memcpy(c, p, l, 32); | |
590 | _len += l; | |
591 | return _len + _off; | |
592 | } | |
593 | ||
11fdf7f2 | 594 | unsigned buffer::ptr::append_zeros(unsigned l) |
7c673cae | 595 | { |
11fdf7f2 TL |
596 | ceph_assert(_raw); |
597 | ceph_assert(l <= unused_tail_length()); | |
598 | char* c = _raw->data + _off + _len; | |
92f5a8d4 | 599 | // FIPS zeroization audit 20191115: this memset is not security related. |
11fdf7f2 TL |
600 | memset(c, 0, l); |
601 | _len += l; | |
602 | return _len + _off; | |
7c673cae FG |
603 | } |
604 | ||
605 | void buffer::ptr::copy_in(unsigned o, unsigned l, const char *src, bool crc_reset) | |
606 | { | |
11fdf7f2 TL |
607 | ceph_assert(_raw); |
608 | ceph_assert(o <= _len); | |
609 | ceph_assert(o+l <= _len); | |
7c673cae FG |
610 | char* dest = _raw->data + _off + o; |
611 | if (crc_reset) | |
612 | _raw->invalidate_crc(); | |
613 | maybe_inline_memcpy(dest, src, l, 64); | |
614 | } | |
615 | ||
7c673cae FG |
616 | void buffer::ptr::zero(bool crc_reset) |
617 | { | |
618 | if (crc_reset) | |
619 | _raw->invalidate_crc(); | |
92f5a8d4 | 620 | // FIPS zeroization audit 20191115: this memset is not security related. |
7c673cae FG |
621 | memset(c_str(), 0, _len); |
622 | } | |
623 | ||
7c673cae FG |
624 | void buffer::ptr::zero(unsigned o, unsigned l, bool crc_reset) |
625 | { | |
11fdf7f2 | 626 | ceph_assert(o+l <= _len); |
7c673cae FG |
627 | if (crc_reset) |
628 | _raw->invalidate_crc(); | |
92f5a8d4 | 629 | // FIPS zeroization audit 20191115: this memset is not security related. |
7c673cae FG |
630 | memset(c_str()+o, 0, l); |
631 | } | |
7c673cae FG |
632 | |
633 | // -- buffer::list::iterator -- | |
634 | /* | |
635 | buffer::list::iterator operator=(const buffer::list::iterator& other) | |
636 | { | |
637 | if (this != &other) { | |
638 | bl = other.bl; | |
639 | ls = other.ls; | |
640 | off = other.off; | |
641 | p = other.p; | |
642 | p_off = other.p_off; | |
643 | } | |
644 | return *this; | |
645 | }*/ | |
646 | ||
647 | template<bool is_const> | |
648 | buffer::list::iterator_impl<is_const>::iterator_impl(bl_t *l, unsigned o) | |
11fdf7f2 | 649 | : bl(l), ls(&bl->_buffers), p(ls->begin()), off(0), p_off(0) |
7c673cae | 650 | { |
9f95a23c | 651 | *this += o; |
7c673cae FG |
652 | } |
653 | ||
654 | template<bool is_const> | |
655 | buffer::list::iterator_impl<is_const>::iterator_impl(const buffer::list::iterator& i) | |
656 | : iterator_impl<is_const>(i.bl, i.off, i.p, i.p_off) {} | |
657 | ||
658 | template<bool is_const> | |
9f95a23c TL |
659 | auto buffer::list::iterator_impl<is_const>::operator +=(unsigned o) |
660 | -> iterator_impl& | |
11fdf7f2 TL |
661 | { |
662 | //cout << this << " advance " << o << " from " << off | |
663 | // << " (p_off " << p_off << " in " << p->length() << ")" | |
664 | // << std::endl; | |
665 | ||
666 | p_off +=o; | |
667 | while (p != ls->end()) { | |
668 | if (p_off >= p->length()) { | |
669 | // skip this buffer | |
670 | p_off -= p->length(); | |
671 | p++; | |
7c673cae | 672 | } else { |
11fdf7f2 TL |
673 | // somewhere in this buffer! |
674 | break; | |
7c673cae FG |
675 | } |
676 | } | |
11fdf7f2 TL |
677 | if (p == ls->end() && p_off) { |
678 | throw end_of_buffer(); | |
679 | } | |
680 | off += o; | |
9f95a23c | 681 | return *this; |
7c673cae FG |
682 | } |
683 | ||
684 | template<bool is_const> | |
685 | void buffer::list::iterator_impl<is_const>::seek(unsigned o) | |
686 | { | |
687 | p = ls->begin(); | |
688 | off = p_off = 0; | |
9f95a23c | 689 | *this += o; |
7c673cae FG |
690 | } |
691 | ||
692 | template<bool is_const> | |
693 | char buffer::list::iterator_impl<is_const>::operator*() const | |
694 | { | |
695 | if (p == ls->end()) | |
696 | throw end_of_buffer(); | |
697 | return (*p)[p_off]; | |
698 | } | |
699 | ||
700 | template<bool is_const> | |
701 | buffer::list::iterator_impl<is_const>& | |
702 | buffer::list::iterator_impl<is_const>::operator++() | |
703 | { | |
704 | if (p == ls->end()) | |
705 | throw end_of_buffer(); | |
9f95a23c | 706 | *this += 1; |
7c673cae FG |
707 | return *this; |
708 | } | |
709 | ||
710 | template<bool is_const> | |
711 | buffer::ptr buffer::list::iterator_impl<is_const>::get_current_ptr() const | |
712 | { | |
713 | if (p == ls->end()) | |
714 | throw end_of_buffer(); | |
715 | return ptr(*p, p_off, p->length() - p_off); | |
716 | } | |
717 | ||
11fdf7f2 TL |
718 | template<bool is_const> |
719 | bool buffer::list::iterator_impl<is_const>::is_pointing_same_raw( | |
720 | const ptr& other) const | |
721 | { | |
722 | if (p == ls->end()) | |
723 | throw end_of_buffer(); | |
9f95a23c | 724 | return p->_raw == other._raw; |
11fdf7f2 TL |
725 | } |
726 | ||
7c673cae FG |
727 | // copy data out. |
728 | // note that these all _append_ to dest! | |
729 | template<bool is_const> | |
730 | void buffer::list::iterator_impl<is_const>::copy(unsigned len, char *dest) | |
731 | { | |
732 | if (p == ls->end()) seek(off); | |
733 | while (len > 0) { | |
734 | if (p == ls->end()) | |
735 | throw end_of_buffer(); | |
7c673cae FG |
736 | |
737 | unsigned howmuch = p->length() - p_off; | |
738 | if (len < howmuch) howmuch = len; | |
739 | p->copy_out(p_off, howmuch, dest); | |
740 | dest += howmuch; | |
741 | ||
742 | len -= howmuch; | |
9f95a23c | 743 | *this += howmuch; |
7c673cae FG |
744 | } |
745 | } | |
746 | ||
747 | template<bool is_const> | |
748 | void buffer::list::iterator_impl<is_const>::copy(unsigned len, ptr &dest) | |
749 | { | |
750 | copy_deep(len, dest); | |
751 | } | |
752 | ||
753 | template<bool is_const> | |
754 | void buffer::list::iterator_impl<is_const>::copy_deep(unsigned len, ptr &dest) | |
755 | { | |
756 | if (!len) { | |
757 | return; | |
758 | } | |
759 | if (p == ls->end()) | |
760 | throw end_of_buffer(); | |
7c673cae FG |
761 | dest = create(len); |
762 | copy(len, dest.c_str()); | |
763 | } | |
764 | template<bool is_const> | |
765 | void buffer::list::iterator_impl<is_const>::copy_shallow(unsigned len, | |
766 | ptr &dest) | |
767 | { | |
768 | if (!len) { | |
769 | return; | |
770 | } | |
771 | if (p == ls->end()) | |
772 | throw end_of_buffer(); | |
7c673cae FG |
773 | unsigned howmuch = p->length() - p_off; |
774 | if (howmuch < len) { | |
775 | dest = create(len); | |
776 | copy(len, dest.c_str()); | |
777 | } else { | |
778 | dest = ptr(*p, p_off, len); | |
9f95a23c | 779 | *this += len; |
7c673cae FG |
780 | } |
781 | } | |
782 | ||
783 | template<bool is_const> | |
784 | void buffer::list::iterator_impl<is_const>::copy(unsigned len, list &dest) | |
785 | { | |
786 | if (p == ls->end()) | |
787 | seek(off); | |
788 | while (len > 0) { | |
789 | if (p == ls->end()) | |
790 | throw end_of_buffer(); | |
791 | ||
792 | unsigned howmuch = p->length() - p_off; | |
793 | if (len < howmuch) | |
794 | howmuch = len; | |
795 | dest.append(*p, p_off, howmuch); | |
796 | ||
797 | len -= howmuch; | |
9f95a23c | 798 | *this += howmuch; |
7c673cae FG |
799 | } |
800 | } | |
801 | ||
802 | template<bool is_const> | |
803 | void buffer::list::iterator_impl<is_const>::copy(unsigned len, std::string &dest) | |
804 | { | |
805 | if (p == ls->end()) | |
806 | seek(off); | |
807 | while (len > 0) { | |
808 | if (p == ls->end()) | |
809 | throw end_of_buffer(); | |
810 | ||
811 | unsigned howmuch = p->length() - p_off; | |
812 | const char *c_str = p->c_str(); | |
813 | if (len < howmuch) | |
814 | howmuch = len; | |
815 | dest.append(c_str + p_off, howmuch); | |
816 | ||
817 | len -= howmuch; | |
9f95a23c | 818 | *this += howmuch; |
7c673cae FG |
819 | } |
820 | } | |
821 | ||
822 | template<bool is_const> | |
823 | void buffer::list::iterator_impl<is_const>::copy_all(list &dest) | |
824 | { | |
825 | if (p == ls->end()) | |
826 | seek(off); | |
827 | while (1) { | |
828 | if (p == ls->end()) | |
829 | return; | |
7c673cae FG |
830 | |
831 | unsigned howmuch = p->length() - p_off; | |
832 | const char *c_str = p->c_str(); | |
833 | dest.append(c_str + p_off, howmuch); | |
834 | ||
9f95a23c | 835 | *this += howmuch; |
7c673cae FG |
836 | } |
837 | } | |
838 | ||
839 | template<bool is_const> | |
840 | size_t buffer::list::iterator_impl<is_const>::get_ptr_and_advance( | |
841 | size_t want, const char **data) | |
842 | { | |
843 | if (p == ls->end()) { | |
844 | seek(off); | |
845 | if (p == ls->end()) { | |
846 | return 0; | |
847 | } | |
848 | } | |
849 | *data = p->c_str() + p_off; | |
11fdf7f2 | 850 | size_t l = std::min<size_t>(p->length() - p_off, want); |
7c673cae FG |
851 | p_off += l; |
852 | if (p_off == p->length()) { | |
853 | ++p; | |
854 | p_off = 0; | |
855 | } | |
856 | off += l; | |
857 | return l; | |
858 | } | |
859 | ||
860 | template<bool is_const> | |
861 | uint32_t buffer::list::iterator_impl<is_const>::crc32c( | |
862 | size_t length, uint32_t crc) | |
863 | { | |
11fdf7f2 | 864 | length = std::min<size_t>(length, get_remaining()); |
7c673cae FG |
865 | while (length > 0) { |
866 | const char *p; | |
867 | size_t l = get_ptr_and_advance(length, &p); | |
868 | crc = ceph_crc32c(crc, (unsigned char*)p, l); | |
869 | length -= l; | |
870 | } | |
871 | return crc; | |
872 | } | |
873 | ||
874 | // explicitly instantiate only the iterator types we need, so we can hide the | |
875 | // details in this compilation unit without introducing unnecessary link time | |
876 | // dependencies. | |
877 | template class buffer::list::iterator_impl<true>; | |
878 | template class buffer::list::iterator_impl<false>; | |
879 | ||
880 | buffer::list::iterator::iterator(bl_t *l, unsigned o) | |
881 | : iterator_impl(l, o) | |
882 | {} | |
883 | ||
884 | buffer::list::iterator::iterator(bl_t *l, unsigned o, list_iter_t ip, unsigned po) | |
885 | : iterator_impl(l, o, ip, po) | |
886 | {} | |
887 | ||
7c673cae FG |
888 | // copy data in |
889 | void buffer::list::iterator::copy_in(unsigned len, const char *src, bool crc_reset) | |
890 | { | |
891 | // copy | |
892 | if (p == ls->end()) | |
893 | seek(off); | |
894 | while (len > 0) { | |
895 | if (p == ls->end()) | |
896 | throw end_of_buffer(); | |
897 | ||
898 | unsigned howmuch = p->length() - p_off; | |
899 | if (len < howmuch) | |
900 | howmuch = len; | |
901 | p->copy_in(p_off, howmuch, src, crc_reset); | |
902 | ||
903 | src += howmuch; | |
904 | len -= howmuch; | |
9f95a23c | 905 | *this += howmuch; |
7c673cae FG |
906 | } |
907 | } | |
908 | ||
909 | void buffer::list::iterator::copy_in(unsigned len, const list& otherl) | |
910 | { | |
911 | if (p == ls->end()) | |
912 | seek(off); | |
913 | unsigned left = len; | |
11fdf7f2 TL |
914 | for (const auto& node : otherl._buffers) { |
915 | unsigned l = node.length(); | |
7c673cae FG |
916 | if (left < l) |
917 | l = left; | |
11fdf7f2 | 918 | copy_in(l, node.c_str()); |
7c673cae FG |
919 | left -= l; |
920 | if (left == 0) | |
921 | break; | |
922 | } | |
923 | } | |
924 | ||
925 | // -- buffer::list -- | |
926 | ||
11fdf7f2 | 927 | void buffer::list::swap(list& other) noexcept |
7c673cae FG |
928 | { |
929 | std::swap(_len, other._len); | |
9f95a23c | 930 | std::swap(_num, other._num); |
11fdf7f2 | 931 | std::swap(_carriage, other._carriage); |
7c673cae | 932 | _buffers.swap(other._buffers); |
7c673cae FG |
933 | } |
934 | ||
7c673cae FG |
935 | bool buffer::list::contents_equal(const ceph::buffer::list& other) const |
936 | { | |
937 | if (length() != other.length()) | |
938 | return false; | |
939 | ||
940 | // buffer-wise comparison | |
941 | if (true) { | |
11fdf7f2 TL |
942 | auto a = std::cbegin(_buffers); |
943 | auto b = std::cbegin(other._buffers); | |
7c673cae | 944 | unsigned aoff = 0, boff = 0; |
11fdf7f2 | 945 | while (a != std::cend(_buffers)) { |
7c673cae FG |
946 | unsigned len = a->length() - aoff; |
947 | if (len > b->length() - boff) | |
948 | len = b->length() - boff; | |
949 | if (memcmp(a->c_str() + aoff, b->c_str() + boff, len) != 0) | |
950 | return false; | |
951 | aoff += len; | |
952 | if (aoff == a->length()) { | |
953 | aoff = 0; | |
954 | ++a; | |
955 | } | |
956 | boff += len; | |
957 | if (boff == b->length()) { | |
958 | boff = 0; | |
959 | ++b; | |
960 | } | |
961 | } | |
7c673cae FG |
962 | return true; |
963 | } | |
964 | ||
965 | // byte-wise comparison | |
966 | if (false) { | |
967 | bufferlist::const_iterator me = begin(); | |
968 | bufferlist::const_iterator him = other.begin(); | |
969 | while (!me.end()) { | |
970 | if (*me != *him) | |
971 | return false; | |
972 | ++me; | |
973 | ++him; | |
974 | } | |
975 | return true; | |
976 | } | |
977 | } | |
978 | ||
9f95a23c TL |
979 | bool buffer::list::contents_equal(const void* const other, |
980 | size_t length) const | |
981 | { | |
982 | if (this->length() != length) { | |
983 | return false; | |
984 | } | |
985 | ||
986 | const auto* other_buf = reinterpret_cast<const char*>(other); | |
987 | for (const auto& bp : buffers()) { | |
988 | const auto round_length = std::min<size_t>(length, bp.length()); | |
989 | if (std::memcmp(bp.c_str(), other_buf, round_length) != 0) { | |
990 | return false; | |
991 | } else { | |
992 | length -= round_length; | |
993 | other_buf += round_length; | |
994 | } | |
995 | } | |
996 | ||
997 | return true; | |
998 | } | |
999 | ||
11fdf7f2 | 1000 | bool buffer::list::is_provided_buffer(const char* const dst) const |
7c673cae | 1001 | { |
11fdf7f2 | 1002 | if (_buffers.empty()) { |
7c673cae | 1003 | return false; |
11fdf7f2 | 1004 | } |
7c673cae FG |
1005 | return (is_contiguous() && (_buffers.front().c_str() == dst)); |
1006 | } | |
1007 | ||
11fdf7f2 | 1008 | bool buffer::list::is_aligned(const unsigned align) const |
7c673cae | 1009 | { |
11fdf7f2 TL |
1010 | for (const auto& node : _buffers) { |
1011 | if (!node.is_aligned(align)) { | |
7c673cae | 1012 | return false; |
11fdf7f2 TL |
1013 | } |
1014 | } | |
7c673cae FG |
1015 | return true; |
1016 | } | |
1017 | ||
11fdf7f2 | 1018 | bool buffer::list::is_n_align_sized(const unsigned align) const |
7c673cae | 1019 | { |
11fdf7f2 TL |
1020 | for (const auto& node : _buffers) { |
1021 | if (!node.is_n_align_sized(align)) { | |
7c673cae | 1022 | return false; |
11fdf7f2 TL |
1023 | } |
1024 | } | |
7c673cae FG |
1025 | return true; |
1026 | } | |
1027 | ||
11fdf7f2 TL |
1028 | bool buffer::list::is_aligned_size_and_memory( |
1029 | const unsigned align_size, | |
1030 | const unsigned align_memory) const | |
7c673cae | 1031 | { |
11fdf7f2 TL |
1032 | for (const auto& node : _buffers) { |
1033 | if (!node.is_aligned(align_memory) || !node.is_n_align_sized(align_size)) { | |
7c673cae | 1034 | return false; |
11fdf7f2 | 1035 | } |
7c673cae FG |
1036 | } |
1037 | return true; | |
1038 | } | |
1039 | ||
1040 | bool buffer::list::is_zero() const { | |
11fdf7f2 TL |
1041 | for (const auto& node : _buffers) { |
1042 | if (!node.is_zero()) { | |
7c673cae FG |
1043 | return false; |
1044 | } | |
1045 | } | |
1046 | return true; | |
1047 | } | |
1048 | ||
1049 | void buffer::list::zero() | |
1050 | { | |
11fdf7f2 TL |
1051 | for (auto& node : _buffers) { |
1052 | node.zero(); | |
1053 | } | |
7c673cae FG |
1054 | } |
1055 | ||
11fdf7f2 | 1056 | void buffer::list::zero(const unsigned o, const unsigned l) |
7c673cae | 1057 | { |
11fdf7f2 | 1058 | ceph_assert(o+l <= _len); |
7c673cae | 1059 | unsigned p = 0; |
11fdf7f2 TL |
1060 | for (auto& node : _buffers) { |
1061 | if (p + node.length() > o) { | |
1062 | if (p >= o && p+node.length() <= o+l) { | |
7c673cae | 1063 | // 'o'------------- l -----------| |
11fdf7f2 TL |
1064 | // 'p'-- node.length() --| |
1065 | node.zero(); | |
7c673cae FG |
1066 | } else if (p >= o) { |
1067 | // 'o'------------- l -----------| | |
11fdf7f2 TL |
1068 | // 'p'------- node.length() -------| |
1069 | node.zero(0, o+l-p); | |
1070 | } else if (p + node.length() <= o+l) { | |
7c673cae | 1071 | // 'o'------------- l -----------| |
11fdf7f2 TL |
1072 | // 'p'------- node.length() -------| |
1073 | node.zero(o-p, node.length()-(o-p)); | |
7c673cae FG |
1074 | } else { |
1075 | // 'o'----------- l -----------| | |
11fdf7f2 TL |
1076 | // 'p'---------- node.length() ----------| |
1077 | node.zero(o-p, l); | |
7c673cae FG |
1078 | } |
1079 | } | |
11fdf7f2 TL |
1080 | p += node.length(); |
1081 | if (o+l <= p) { | |
7c673cae | 1082 | break; // done |
11fdf7f2 | 1083 | } |
7c673cae FG |
1084 | } |
1085 | } | |
1086 | ||
1087 | bool buffer::list::is_contiguous() const | |
1088 | { | |
9f95a23c | 1089 | return _num <= 1; |
7c673cae FG |
1090 | } |
1091 | ||
1092 | bool buffer::list::is_n_page_sized() const | |
1093 | { | |
1094 | return is_n_align_sized(CEPH_PAGE_SIZE); | |
1095 | } | |
1096 | ||
1097 | bool buffer::list::is_page_aligned() const | |
1098 | { | |
1099 | return is_aligned(CEPH_PAGE_SIZE); | |
1100 | } | |
1101 | ||
3efd9988 FG |
1102 | int buffer::list::get_mempool() const |
1103 | { | |
1104 | if (_buffers.empty()) { | |
1105 | return mempool::mempool_buffer_anon; | |
1106 | } | |
1107 | return _buffers.back().get_mempool(); | |
1108 | } | |
1109 | ||
31f18b77 FG |
1110 | void buffer::list::reassign_to_mempool(int pool) |
1111 | { | |
31f18b77 | 1112 | for (auto& p : _buffers) { |
9f95a23c | 1113 | p._raw->reassign_to_mempool(pool); |
31f18b77 FG |
1114 | } |
1115 | } | |
1116 | ||
1117 | void buffer::list::try_assign_to_mempool(int pool) | |
1118 | { | |
31f18b77 | 1119 | for (auto& p : _buffers) { |
9f95a23c | 1120 | p._raw->try_assign_to_mempool(pool); |
31f18b77 FG |
1121 | } |
1122 | } | |
1123 | ||
f64942e4 AA |
1124 | uint64_t buffer::list::get_wasted_space() const |
1125 | { | |
9f95a23c | 1126 | if (_num == 1) |
f64942e4 AA |
1127 | return _buffers.back().wasted(); |
1128 | ||
1129 | std::vector<const raw*> raw_vec; | |
9f95a23c | 1130 | raw_vec.reserve(_num); |
f64942e4 | 1131 | for (const auto& p : _buffers) |
9f95a23c | 1132 | raw_vec.push_back(p._raw); |
f64942e4 AA |
1133 | std::sort(raw_vec.begin(), raw_vec.end()); |
1134 | ||
1135 | uint64_t total = 0; | |
1136 | const raw *last = nullptr; | |
1137 | for (const auto r : raw_vec) { | |
1138 | if (r == last) | |
1139 | continue; | |
1140 | last = r; | |
1141 | total += r->len; | |
1142 | } | |
1143 | // If multiple buffers are sharing the same raw buffer and they overlap | |
1144 | // with each other, the wasted space will be underestimated. | |
1145 | if (total <= length()) | |
1146 | return 0; | |
1147 | return total - length(); | |
1148 | } | |
1149 | ||
7c673cae FG |
1150 | void buffer::list::rebuild() |
1151 | { | |
1152 | if (_len == 0) { | |
11fdf7f2 TL |
1153 | _carriage = &always_empty_bptr; |
1154 | _buffers.clear_and_dispose(); | |
9f95a23c | 1155 | _num = 0; |
7c673cae FG |
1156 | return; |
1157 | } | |
7c673cae | 1158 | if ((_len & ~CEPH_PAGE_MASK) == 0) |
11fdf7f2 | 1159 | rebuild(ptr_node::create(buffer::create_page_aligned(_len))); |
7c673cae | 1160 | else |
11fdf7f2 | 1161 | rebuild(ptr_node::create(buffer::create(_len))); |
7c673cae FG |
1162 | } |
1163 | ||
11fdf7f2 TL |
1164 | void buffer::list::rebuild( |
1165 | std::unique_ptr<buffer::ptr_node, buffer::ptr_node::disposer> nb) | |
7c673cae FG |
1166 | { |
1167 | unsigned pos = 0; | |
11fdf7f2 TL |
1168 | for (auto& node : _buffers) { |
1169 | nb->copy_in(pos, node.length(), node.c_str(), false); | |
1170 | pos += node.length(); | |
7c673cae | 1171 | } |
11fdf7f2 TL |
1172 | _buffers.clear_and_dispose(); |
1173 | if (likely(nb->length())) { | |
1174 | _carriage = nb.get(); | |
1175 | _buffers.push_back(*nb.release()); | |
9f95a23c TL |
1176 | _num = 1; |
1177 | } else { | |
1178 | _carriage = &always_empty_bptr; | |
1179 | _num = 0; | |
11fdf7f2 | 1180 | } |
7c673cae | 1181 | invalidate_crc(); |
7c673cae FG |
1182 | } |
1183 | ||
1184 | bool buffer::list::rebuild_aligned(unsigned align) | |
1185 | { | |
1186 | return rebuild_aligned_size_and_memory(align, align); | |
1187 | } | |
1188 | ||
1189 | bool buffer::list::rebuild_aligned_size_and_memory(unsigned align_size, | |
b32b8144 FG |
1190 | unsigned align_memory, |
1191 | unsigned max_buffers) | |
7c673cae | 1192 | { |
9f95a23c | 1193 | bool had_to_rebuild = false; |
b32b8144 | 1194 | |
9f95a23c | 1195 | if (max_buffers && _num > max_buffers && _len > (max_buffers * align_size)) { |
11fdf7f2 | 1196 | align_size = round_up_to(round_up_to(_len, max_buffers) / max_buffers, align_size); |
b32b8144 | 1197 | } |
11fdf7f2 TL |
1198 | auto p = std::begin(_buffers); |
1199 | auto p_prev = _buffers.before_begin(); | |
1200 | while (p != std::end(_buffers)) { | |
7c673cae FG |
1201 | // keep anything that's already align and sized aligned |
1202 | if (p->is_aligned(align_memory) && p->is_n_align_sized(align_size)) { | |
1203 | /*cout << " segment " << (void*)p->c_str() | |
1204 | << " offset " << ((unsigned long)p->c_str() & (align - 1)) | |
1205 | << " length " << p->length() | |
1206 | << " " << (p->length() & (align - 1)) << " ok" << std::endl; | |
1207 | */ | |
11fdf7f2 | 1208 | p_prev = p++; |
7c673cae FG |
1209 | continue; |
1210 | } | |
1211 | ||
1212 | // consolidate unaligned items, until we get something that is sized+aligned | |
1213 | list unaligned; | |
1214 | unsigned offset = 0; | |
1215 | do { | |
1216 | /*cout << " segment " << (void*)p->c_str() | |
1217 | << " offset " << ((unsigned long)p->c_str() & (align - 1)) | |
1218 | << " length " << p->length() << " " << (p->length() & (align - 1)) | |
1219 | << " overall offset " << offset << " " << (offset & (align - 1)) | |
1220 | << " not ok" << std::endl; | |
1221 | */ | |
1222 | offset += p->length(); | |
11fdf7f2 TL |
1223 | // no need to reallocate, relinking is enough thankfully to bi::list. |
1224 | auto p_after = _buffers.erase_after(p_prev); | |
9f95a23c | 1225 | _num -= 1; |
11fdf7f2 TL |
1226 | unaligned._buffers.push_back(*p); |
1227 | unaligned._len += p->length(); | |
9f95a23c | 1228 | unaligned._num += 1; |
11fdf7f2 TL |
1229 | p = p_after; |
1230 | } while (p != std::end(_buffers) && | |
7c673cae FG |
1231 | (!p->is_aligned(align_memory) || |
1232 | !p->is_n_align_sized(align_size) || | |
1233 | (offset % align_size))); | |
1234 | if (!(unaligned.is_contiguous() && unaligned._buffers.front().is_aligned(align_memory))) { | |
11fdf7f2 TL |
1235 | unaligned.rebuild( |
1236 | ptr_node::create( | |
1237 | buffer::create_aligned(unaligned._len, align_memory))); | |
9f95a23c | 1238 | had_to_rebuild = true; |
7c673cae | 1239 | } |
11fdf7f2 | 1240 | _buffers.insert_after(p_prev, *ptr_node::create(unaligned._buffers.front()).release()); |
9f95a23c | 1241 | _num += 1; |
11fdf7f2 | 1242 | ++p_prev; |
7c673cae | 1243 | } |
9f95a23c | 1244 | return had_to_rebuild; |
7c673cae FG |
1245 | } |
1246 | ||
1247 | bool buffer::list::rebuild_page_aligned() | |
1248 | { | |
1249 | return rebuild_aligned(CEPH_PAGE_SIZE); | |
1250 | } | |
1251 | ||
31f18b77 FG |
1252 | void buffer::list::reserve(size_t prealloc) |
1253 | { | |
11fdf7f2 TL |
1254 | if (get_append_buffer_unused_tail_length() < prealloc) { |
1255 | auto ptr = ptr_node::create(buffer::create_page_aligned(prealloc)); | |
1256 | ptr->set_length(0); // unused, so far. | |
1257 | _carriage = ptr.get(); | |
1258 | _buffers.push_back(*ptr.release()); | |
9f95a23c | 1259 | _num += 1; |
31f18b77 FG |
1260 | } |
1261 | } | |
1262 | ||
7c673cae | 1263 | // sort-of-like-assignment-op |
9f95a23c | 1264 | void buffer::list::claim(list& bl) |
7c673cae FG |
1265 | { |
1266 | // free my buffers | |
1267 | clear(); | |
9f95a23c | 1268 | claim_append(bl); |
7c673cae FG |
1269 | } |
1270 | ||
9f95a23c | 1271 | void buffer::list::claim_append(list& bl) |
7c673cae FG |
1272 | { |
1273 | // steal the other guy's buffers | |
1274 | _len += bl._len; | |
9f95a23c | 1275 | _num += bl._num; |
11fdf7f2 TL |
1276 | _buffers.splice_back(bl._buffers); |
1277 | bl._carriage = &always_empty_bptr; | |
1278 | bl._buffers.clear_and_dispose(); | |
7c673cae | 1279 | bl._len = 0; |
9f95a23c | 1280 | bl._num = 0; |
7c673cae FG |
1281 | } |
1282 | ||
31f18b77 FG |
1283 | void buffer::list::claim_append_piecewise(list& bl) |
1284 | { | |
1285 | // steal the other guy's buffers | |
11fdf7f2 TL |
1286 | for (const auto& node : bl.buffers()) { |
1287 | append(node, 0, node.length()); | |
31f18b77 FG |
1288 | } |
1289 | bl.clear(); | |
1290 | } | |
1291 | ||
7c673cae FG |
1292 | void buffer::list::append(char c) |
1293 | { | |
1294 | // put what we can into the existing append_buffer. | |
11fdf7f2 | 1295 | unsigned gap = get_append_buffer_unused_tail_length(); |
7c673cae | 1296 | if (!gap) { |
11fdf7f2 TL |
1297 | // make a new buffer! |
1298 | auto buf = ptr_node::create( | |
1299 | raw_combined::create(CEPH_BUFFER_APPEND_SIZE, 0, get_mempool())); | |
1300 | buf->set_length(0); // unused, so far. | |
1301 | _carriage = buf.get(); | |
1302 | _buffers.push_back(*buf.release()); | |
9f95a23c | 1303 | _num += 1; |
11fdf7f2 TL |
1304 | } else if (unlikely(_carriage != &_buffers.back())) { |
1305 | auto bptr = ptr_node::create(*_carriage, _carriage->length(), 0); | |
1306 | _carriage = bptr.get(); | |
1307 | _buffers.push_back(*bptr.release()); | |
9f95a23c | 1308 | _num += 1; |
11fdf7f2 TL |
1309 | } |
1310 | _carriage->append(c); | |
1311 | _len++; | |
1312 | } | |
1313 | ||
1314 | buffer::ptr buffer::list::always_empty_bptr; | |
1315 | ||
1316 | buffer::ptr_node& buffer::list::refill_append_space(const unsigned len) | |
1317 | { | |
1318 | // make a new buffer. fill out a complete page, factoring in the | |
1319 | // raw_combined overhead. | |
1320 | size_t need = round_up_to(len, sizeof(size_t)) + sizeof(raw_combined); | |
1321 | size_t alen = round_up_to(need, CEPH_BUFFER_ALLOC_UNIT) - | |
1322 | sizeof(raw_combined); | |
1323 | auto new_back = \ | |
1324 | ptr_node::create(raw_combined::create(alen, 0, get_mempool())); | |
1325 | new_back->set_length(0); // unused, so far. | |
1326 | _carriage = new_back.get(); | |
1327 | _buffers.push_back(*new_back.release()); | |
9f95a23c | 1328 | _num += 1; |
11fdf7f2 | 1329 | return _buffers.back(); |
7c673cae FG |
1330 | } |
1331 | ||
1332 | void buffer::list::append(const char *data, unsigned len) | |
1333 | { | |
11fdf7f2 TL |
1334 | _len += len; |
1335 | ||
1336 | const unsigned free_in_last = get_append_buffer_unused_tail_length(); | |
1337 | const unsigned first_round = std::min(len, free_in_last); | |
1338 | if (first_round) { | |
1339 | // _buffers and carriage can desynchronize when 1) a new ptr | |
1340 | // we don't own has been added into the _buffers 2) _buffers | |
1341 | // has been emptied as as a result of std::move or stolen by | |
1342 | // claim_append. | |
1343 | if (unlikely(_carriage != &_buffers.back())) { | |
1344 | auto bptr = ptr_node::create(*_carriage, _carriage->length(), 0); | |
1345 | _carriage = bptr.get(); | |
1346 | _buffers.push_back(*bptr.release()); | |
9f95a23c | 1347 | _num += 1; |
7c673cae | 1348 | } |
11fdf7f2 TL |
1349 | _carriage->append(data, first_round); |
1350 | } | |
1351 | ||
1352 | const unsigned second_round = len - first_round; | |
1353 | if (second_round) { | |
1354 | auto& new_back = refill_append_space(second_round); | |
1355 | new_back.append(data + first_round, second_round); | |
1356 | } | |
1357 | } | |
1358 | ||
1359 | buffer::list::reserve_t buffer::list::obtain_contiguous_space( | |
1360 | const unsigned len) | |
1361 | { | |
1362 | // note: if len < the normal append_buffer size it *might* | |
1363 | // be better to allocate a normal-sized append_buffer and | |
1364 | // use part of it. however, that optimizes for the case of | |
1365 | // old-style types including new-style types. and in most | |
1366 | // such cases, this won't be the very first thing encoded to | |
1367 | // the list, so append_buffer will already be allocated. | |
1368 | // OTOH if everything is new-style, we *should* allocate | |
1369 | // only what we need and conserve memory. | |
1370 | if (unlikely(get_append_buffer_unused_tail_length() < len)) { | |
1371 | auto new_back = \ | |
1372 | buffer::ptr_node::create(buffer::create(len)).release(); | |
1373 | new_back->set_length(0); // unused, so far. | |
1374 | _buffers.push_back(*new_back); | |
9f95a23c | 1375 | _num += 1; |
11fdf7f2 TL |
1376 | _carriage = new_back; |
1377 | return { new_back->c_str(), &new_back->_len, &_len }; | |
1378 | } else { | |
1379 | if (unlikely(_carriage != &_buffers.back())) { | |
1380 | auto bptr = ptr_node::create(*_carriage, _carriage->length(), 0); | |
1381 | _carriage = bptr.get(); | |
1382 | _buffers.push_back(*bptr.release()); | |
9f95a23c | 1383 | _num += 1; |
11fdf7f2 TL |
1384 | } |
1385 | return { _carriage->end_c_str(), &_carriage->_len, &_len }; | |
7c673cae FG |
1386 | } |
1387 | } | |
1388 | ||
1389 | void buffer::list::append(const ptr& bp) | |
1390 | { | |
7c673cae FG |
1391 | push_back(bp); |
1392 | } | |
1393 | ||
1394 | void buffer::list::append(ptr&& bp) | |
1395 | { | |
7c673cae FG |
1396 | push_back(std::move(bp)); |
1397 | } | |
1398 | ||
1399 | void buffer::list::append(const ptr& bp, unsigned off, unsigned len) | |
1400 | { | |
11fdf7f2 | 1401 | ceph_assert(len+off <= bp.length()); |
7c673cae FG |
1402 | if (!_buffers.empty()) { |
1403 | ptr &l = _buffers.back(); | |
9f95a23c | 1404 | if (l._raw == bp._raw && l.end() == bp.start() + off) { |
7c673cae FG |
1405 | // yay contiguous with tail bp! |
1406 | l.set_length(l.length()+len); | |
1407 | _len += len; | |
1408 | return; | |
1409 | } | |
1410 | } | |
1411 | // add new item to list | |
11fdf7f2 TL |
1412 | _buffers.push_back(*ptr_node::create(bp, off, len).release()); |
1413 | _len += len; | |
9f95a23c | 1414 | _num += 1; |
7c673cae FG |
1415 | } |
1416 | ||
1417 | void buffer::list::append(const list& bl) | |
1418 | { | |
1419 | _len += bl._len; | |
9f95a23c | 1420 | _num += bl._num; |
11fdf7f2 TL |
1421 | for (const auto& node : bl._buffers) { |
1422 | _buffers.push_back(*ptr_node::create(node).release()); | |
1423 | } | |
7c673cae FG |
1424 | } |
1425 | ||
1426 | void buffer::list::append(std::istream& in) | |
1427 | { | |
1428 | while (!in.eof()) { | |
1429 | std::string s; | |
1430 | getline(in, s); | |
1431 | append(s.c_str(), s.length()); | |
1432 | if (s.length()) | |
1433 | append("\n", 1); | |
1434 | } | |
1435 | } | |
1436 | ||
11fdf7f2 TL |
1437 | buffer::list::contiguous_filler buffer::list::append_hole(const unsigned len) |
1438 | { | |
1439 | _len += len; | |
1440 | ||
1441 | if (unlikely(get_append_buffer_unused_tail_length() < len)) { | |
1442 | // make a new append_buffer. fill out a complete page, factoring in | |
1443 | // the raw_combined overhead. | |
1444 | auto& new_back = refill_append_space(len); | |
1445 | new_back.set_length(len); | |
1446 | return { new_back.c_str() }; | |
1447 | } else if (unlikely(_carriage != &_buffers.back())) { | |
1448 | auto bptr = ptr_node::create(*_carriage, _carriage->length(), 0); | |
1449 | _carriage = bptr.get(); | |
1450 | _buffers.push_back(*bptr.release()); | |
9f95a23c | 1451 | _num += 1; |
11fdf7f2 TL |
1452 | } |
1453 | _carriage->set_length(_carriage->length() + len); | |
1454 | return { _carriage->end_c_str() - len }; | |
1455 | } | |
1456 | ||
7c673cae FG |
1457 | void buffer::list::prepend_zero(unsigned len) |
1458 | { | |
11fdf7f2 TL |
1459 | auto bp = ptr_node::create(len); |
1460 | bp->zero(false); | |
7c673cae | 1461 | _len += len; |
9f95a23c | 1462 | _num += 1; |
11fdf7f2 | 1463 | _buffers.push_front(*bp.release()); |
7c673cae FG |
1464 | } |
1465 | ||
1466 | void buffer::list::append_zero(unsigned len) | |
1467 | { | |
11fdf7f2 TL |
1468 | _len += len; |
1469 | ||
1470 | const unsigned free_in_last = get_append_buffer_unused_tail_length(); | |
1471 | const unsigned first_round = std::min(len, free_in_last); | |
1472 | if (first_round) { | |
1473 | if (unlikely(_carriage != &_buffers.back())) { | |
1474 | auto bptr = ptr_node::create(*_carriage, _carriage->length(), 0); | |
1475 | _carriage = bptr.get(); | |
1476 | _buffers.push_back(*bptr.release()); | |
9f95a23c | 1477 | _num += 1; |
11fdf7f2 TL |
1478 | } |
1479 | _carriage->append_zeros(first_round); | |
1480 | } | |
1481 | ||
1482 | const unsigned second_round = len - first_round; | |
1483 | if (second_round) { | |
1484 | auto& new_back = refill_append_space(second_round); | |
1485 | new_back.set_length(second_round); | |
1486 | new_back.zero(false); | |
1487 | } | |
7c673cae FG |
1488 | } |
1489 | ||
1490 | ||
1491 | /* | |
1492 | * get a char | |
1493 | */ | |
1494 | const char& buffer::list::operator[](unsigned n) const | |
1495 | { | |
1496 | if (n >= _len) | |
1497 | throw end_of_buffer(); | |
1498 | ||
11fdf7f2 TL |
1499 | for (const auto& node : _buffers) { |
1500 | if (n >= node.length()) { | |
1501 | n -= node.length(); | |
7c673cae FG |
1502 | continue; |
1503 | } | |
11fdf7f2 | 1504 | return node[n]; |
7c673cae FG |
1505 | } |
1506 | ceph_abort(); | |
1507 | } | |
1508 | ||
1509 | /* | |
1510 | * return a contiguous ptr to whole bufferlist contents. | |
1511 | */ | |
1512 | char *buffer::list::c_str() | |
1513 | { | |
1514 | if (_buffers.empty()) | |
1515 | return 0; // no buffers | |
1516 | ||
11fdf7f2 | 1517 | auto iter = std::cbegin(_buffers); |
7c673cae FG |
1518 | ++iter; |
1519 | ||
11fdf7f2 | 1520 | if (iter != std::cend(_buffers)) { |
7c673cae | 1521 | rebuild(); |
11fdf7f2 | 1522 | } |
7c673cae FG |
1523 | return _buffers.front().c_str(); // good, we're already contiguous. |
1524 | } | |
1525 | ||
1526 | string buffer::list::to_str() const { | |
1527 | string s; | |
1528 | s.reserve(length()); | |
11fdf7f2 TL |
1529 | for (const auto& node : _buffers) { |
1530 | if (node.length()) { | |
1531 | s.append(node.c_str(), node.length()); | |
7c673cae FG |
1532 | } |
1533 | } | |
1534 | return s; | |
1535 | } | |
1536 | ||
7c673cae FG |
1537 | void buffer::list::substr_of(const list& other, unsigned off, unsigned len) |
1538 | { | |
1539 | if (off + len > other.length()) | |
1540 | throw end_of_buffer(); | |
1541 | ||
1542 | clear(); | |
1543 | ||
1544 | // skip off | |
11fdf7f2 TL |
1545 | auto curbuf = std::cbegin(other._buffers); |
1546 | while (off > 0 && off >= curbuf->length()) { | |
7c673cae FG |
1547 | // skip this buffer |
1548 | //cout << "skipping over " << *curbuf << std::endl; | |
1549 | off -= (*curbuf).length(); | |
1550 | ++curbuf; | |
1551 | } | |
11fdf7f2 | 1552 | ceph_assert(len == 0 || curbuf != std::cend(other._buffers)); |
7c673cae FG |
1553 | |
1554 | while (len > 0) { | |
1555 | // partial? | |
1556 | if (off + len < curbuf->length()) { | |
1557 | //cout << "copying partial of " << *curbuf << std::endl; | |
11fdf7f2 | 1558 | _buffers.push_back(*ptr_node::create( *curbuf, off, len ).release()); |
7c673cae | 1559 | _len += len; |
9f95a23c | 1560 | _num += 1; |
7c673cae FG |
1561 | break; |
1562 | } | |
1563 | ||
1564 | // through end | |
1565 | //cout << "copying end (all?) of " << *curbuf << std::endl; | |
1566 | unsigned howmuch = curbuf->length() - off; | |
11fdf7f2 | 1567 | _buffers.push_back(*ptr_node::create( *curbuf, off, howmuch ).release()); |
7c673cae | 1568 | _len += howmuch; |
9f95a23c | 1569 | _num += 1; |
7c673cae FG |
1570 | len -= howmuch; |
1571 | off = 0; | |
1572 | ++curbuf; | |
1573 | } | |
1574 | } | |
1575 | ||
1576 | // funky modifer | |
1577 | void buffer::list::splice(unsigned off, unsigned len, list *claim_by /*, bufferlist& replace_with */) | |
1578 | { // fixme? | |
1579 | if (len == 0) | |
1580 | return; | |
1581 | ||
1582 | if (off >= length()) | |
1583 | throw end_of_buffer(); | |
1584 | ||
11fdf7f2 | 1585 | ceph_assert(len > 0); |
7c673cae FG |
1586 | //cout << "splice off " << off << " len " << len << " ... mylen = " << length() << std::endl; |
1587 | ||
1588 | // skip off | |
11fdf7f2 TL |
1589 | auto curbuf = std::begin(_buffers); |
1590 | auto curbuf_prev = _buffers.before_begin(); | |
7c673cae | 1591 | while (off > 0) { |
11fdf7f2 | 1592 | ceph_assert(curbuf != std::end(_buffers)); |
7c673cae FG |
1593 | if (off >= (*curbuf).length()) { |
1594 | // skip this buffer | |
1595 | //cout << "off = " << off << " skipping over " << *curbuf << std::endl; | |
1596 | off -= (*curbuf).length(); | |
11fdf7f2 | 1597 | curbuf_prev = curbuf++; |
7c673cae FG |
1598 | } else { |
1599 | // somewhere in this buffer! | |
1600 | //cout << "off = " << off << " somewhere in " << *curbuf << std::endl; | |
1601 | break; | |
1602 | } | |
1603 | } | |
1604 | ||
1605 | if (off) { | |
1606 | // add a reference to the front bit | |
1607 | // insert it before curbuf (which we'll hose) | |
1608 | //cout << "keeping front " << off << " of " << *curbuf << std::endl; | |
11fdf7f2 TL |
1609 | _buffers.insert_after(curbuf_prev, |
1610 | *ptr_node::create(*curbuf, 0, off).release()); | |
7c673cae | 1611 | _len += off; |
9f95a23c | 1612 | _num += 1; |
11fdf7f2 | 1613 | ++curbuf_prev; |
7c673cae FG |
1614 | } |
1615 | ||
11fdf7f2 TL |
1616 | _carriage = &always_empty_bptr; |
1617 | ||
7c673cae FG |
1618 | while (len > 0) { |
1619 | // partial? | |
1620 | if (off + len < (*curbuf).length()) { | |
1621 | //cout << "keeping end of " << *curbuf << ", losing first " << off+len << std::endl; | |
1622 | if (claim_by) | |
1623 | claim_by->append( *curbuf, off, len ); | |
1624 | (*curbuf).set_offset( off+len + (*curbuf).offset() ); // ignore beginning big | |
1625 | (*curbuf).set_length( (*curbuf).length() - (len+off) ); | |
1626 | _len -= off+len; | |
1627 | //cout << " now " << *curbuf << std::endl; | |
1628 | break; | |
1629 | } | |
1630 | ||
1631 | // hose though the end | |
1632 | unsigned howmuch = (*curbuf).length() - off; | |
1633 | //cout << "discarding " << howmuch << " of " << *curbuf << std::endl; | |
1634 | if (claim_by) | |
1635 | claim_by->append( *curbuf, off, howmuch ); | |
1636 | _len -= (*curbuf).length(); | |
9f95a23c | 1637 | _num -= 1; |
11fdf7f2 | 1638 | curbuf = _buffers.erase_after_and_dispose(curbuf_prev); |
7c673cae FG |
1639 | len -= howmuch; |
1640 | off = 0; | |
1641 | } | |
1642 | ||
1643 | // splice in *replace (implement me later?) | |
7c673cae FG |
1644 | } |
1645 | ||
1646 | void buffer::list::write(int off, int len, std::ostream& out) const | |
1647 | { | |
1648 | list s; | |
1649 | s.substr_of(*this, off, len); | |
11fdf7f2 TL |
1650 | for (const auto& node : s._buffers) { |
1651 | if (node.length()) { | |
1652 | out.write(node.c_str(), node.length()); | |
1653 | } | |
1654 | } | |
7c673cae FG |
1655 | } |
1656 | ||
1657 | void buffer::list::encode_base64(buffer::list& o) | |
1658 | { | |
1659 | bufferptr bp(length() * 4 / 3 + 3); | |
1660 | int l = ceph_armor(bp.c_str(), bp.c_str() + bp.length(), c_str(), c_str() + length()); | |
1661 | bp.set_length(l); | |
1662 | o.push_back(std::move(bp)); | |
1663 | } | |
1664 | ||
1665 | void buffer::list::decode_base64(buffer::list& e) | |
1666 | { | |
1667 | bufferptr bp(4 + ((e.length() * 3) / 4)); | |
1668 | int l = ceph_unarmor(bp.c_str(), bp.c_str() + bp.length(), e.c_str(), e.c_str() + e.length()); | |
1669 | if (l < 0) { | |
1670 | std::ostringstream oss; | |
1671 | oss << "decode_base64: decoding failed:\n"; | |
1672 | hexdump(oss); | |
1673 | throw buffer::malformed_input(oss.str().c_str()); | |
1674 | } | |
11fdf7f2 | 1675 | ceph_assert(l <= (int)bp.length()); |
7c673cae FG |
1676 | bp.set_length(l); |
1677 | push_back(std::move(bp)); | |
1678 | } | |
1679 | ||
9f95a23c TL |
1680 | ssize_t buffer::list::pread_file(const char *fn, uint64_t off, uint64_t len, std::string *error) |
1681 | { | |
1682 | int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY|O_CLOEXEC)); | |
1683 | if (fd < 0) { | |
1684 | int err = errno; | |
1685 | std::ostringstream oss; | |
1686 | oss << "can't open " << fn << ": " << cpp_strerror(err); | |
1687 | *error = oss.str(); | |
1688 | return -err; | |
1689 | } | |
1690 | ||
1691 | struct stat st; | |
1692 | // FIPS zeroization audit 20191115: this memset is not security related. | |
1693 | memset(&st, 0, sizeof(st)); | |
1694 | if (::fstat(fd, &st) < 0) { | |
1695 | int err = errno; | |
1696 | std::ostringstream oss; | |
1697 | oss << "bufferlist::read_file(" << fn << "): stat error: " | |
1698 | << cpp_strerror(err); | |
1699 | *error = oss.str(); | |
1700 | VOID_TEMP_FAILURE_RETRY(::close(fd)); | |
1701 | return -err; | |
1702 | } | |
1703 | ||
1704 | if (off > (uint64_t)st.st_size) { | |
1705 | std::ostringstream oss; | |
1706 | oss << "bufferlist::read_file(" << fn << "): read error: size < offset"; | |
1707 | *error = oss.str(); | |
1708 | VOID_TEMP_FAILURE_RETRY(::close(fd)); | |
1709 | return 0; | |
1710 | } | |
1711 | ||
1712 | if (len > st.st_size - off) { | |
1713 | len = st.st_size - off; | |
1714 | } | |
1715 | ssize_t ret = lseek64(fd, off, SEEK_SET); | |
1716 | if (ret != (ssize_t)off) { | |
1717 | return -errno; | |
1718 | } | |
1719 | ||
1720 | ret = read_fd(fd, len); | |
1721 | if (ret < 0) { | |
1722 | std::ostringstream oss; | |
1723 | oss << "bufferlist::read_file(" << fn << "): read error:" | |
1724 | << cpp_strerror(ret); | |
1725 | *error = oss.str(); | |
1726 | VOID_TEMP_FAILURE_RETRY(::close(fd)); | |
1727 | return ret; | |
1728 | } else if (ret != (ssize_t)len) { | |
1729 | // Premature EOF. | |
1730 | // Perhaps the file changed between stat() and read()? | |
1731 | std::ostringstream oss; | |
1732 | oss << "bufferlist::read_file(" << fn << "): warning: got premature EOF."; | |
1733 | *error = oss.str(); | |
1734 | // not actually an error, but weird | |
1735 | } | |
1736 | VOID_TEMP_FAILURE_RETRY(::close(fd)); | |
1737 | return 0; | |
1738 | } | |
7c673cae FG |
1739 | |
1740 | int buffer::list::read_file(const char *fn, std::string *error) | |
1741 | { | |
91327a77 | 1742 | int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY|O_CLOEXEC)); |
7c673cae FG |
1743 | if (fd < 0) { |
1744 | int err = errno; | |
1745 | std::ostringstream oss; | |
1746 | oss << "can't open " << fn << ": " << cpp_strerror(err); | |
1747 | *error = oss.str(); | |
1748 | return -err; | |
1749 | } | |
1750 | ||
1751 | struct stat st; | |
92f5a8d4 | 1752 | // FIPS zeroization audit 20191115: this memset is not security related. |
7c673cae FG |
1753 | memset(&st, 0, sizeof(st)); |
1754 | if (::fstat(fd, &st) < 0) { | |
1755 | int err = errno; | |
1756 | std::ostringstream oss; | |
1757 | oss << "bufferlist::read_file(" << fn << "): stat error: " | |
1758 | << cpp_strerror(err); | |
1759 | *error = oss.str(); | |
1760 | VOID_TEMP_FAILURE_RETRY(::close(fd)); | |
1761 | return -err; | |
1762 | } | |
1763 | ||
1764 | ssize_t ret = read_fd(fd, st.st_size); | |
1765 | if (ret < 0) { | |
1766 | std::ostringstream oss; | |
1767 | oss << "bufferlist::read_file(" << fn << "): read error:" | |
1768 | << cpp_strerror(ret); | |
1769 | *error = oss.str(); | |
1770 | VOID_TEMP_FAILURE_RETRY(::close(fd)); | |
1771 | return ret; | |
1772 | } | |
1773 | else if (ret != st.st_size) { | |
1774 | // Premature EOF. | |
1775 | // Perhaps the file changed between stat() and read()? | |
1776 | std::ostringstream oss; | |
1777 | oss << "bufferlist::read_file(" << fn << "): warning: got premature EOF."; | |
1778 | *error = oss.str(); | |
1779 | // not actually an error, but weird | |
1780 | } | |
1781 | VOID_TEMP_FAILURE_RETRY(::close(fd)); | |
1782 | return 0; | |
1783 | } | |
1784 | ||
1785 | ssize_t buffer::list::read_fd(int fd, size_t len) | |
1786 | { | |
11fdf7f2 TL |
1787 | auto bp = ptr_node::create(buffer::create(len)); |
1788 | ssize_t ret = safe_read(fd, (void*)bp->c_str(), len); | |
7c673cae | 1789 | if (ret >= 0) { |
11fdf7f2 TL |
1790 | bp->set_length(ret); |
1791 | push_back(std::move(bp)); | |
7c673cae FG |
1792 | } |
1793 | return ret; | |
1794 | } | |
1795 | ||
7c673cae FG |
1796 | int buffer::list::write_file(const char *fn, int mode) |
1797 | { | |
91327a77 | 1798 | int fd = TEMP_FAILURE_RETRY(::open(fn, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC, mode)); |
7c673cae FG |
1799 | if (fd < 0) { |
1800 | int err = errno; | |
1801 | cerr << "bufferlist::write_file(" << fn << "): failed to open file: " | |
1802 | << cpp_strerror(err) << std::endl; | |
1803 | return -err; | |
1804 | } | |
1805 | int ret = write_fd(fd); | |
1806 | if (ret) { | |
1807 | cerr << "bufferlist::write_fd(" << fn << "): write_fd error: " | |
1808 | << cpp_strerror(ret) << std::endl; | |
1809 | VOID_TEMP_FAILURE_RETRY(::close(fd)); | |
1810 | return ret; | |
1811 | } | |
1812 | if (TEMP_FAILURE_RETRY(::close(fd))) { | |
1813 | int err = errno; | |
1814 | cerr << "bufferlist::write_file(" << fn << "): close error: " | |
1815 | << cpp_strerror(err) << std::endl; | |
1816 | return -err; | |
1817 | } | |
1818 | return 0; | |
1819 | } | |
1820 | ||
1821 | static int do_writev(int fd, struct iovec *vec, uint64_t offset, unsigned veclen, unsigned bytes) | |
1822 | { | |
7c673cae | 1823 | while (bytes > 0) { |
11fdf7f2 | 1824 | ssize_t r = 0; |
7c673cae FG |
1825 | #ifdef HAVE_PWRITEV |
1826 | r = ::pwritev(fd, vec, veclen, offset); | |
1827 | #else | |
1828 | r = ::lseek64(fd, offset, SEEK_SET); | |
1829 | if (r != offset) { | |
11fdf7f2 | 1830 | return -errno; |
7c673cae FG |
1831 | } |
1832 | r = ::writev(fd, vec, veclen); | |
1833 | #endif | |
1834 | if (r < 0) { | |
1835 | if (errno == EINTR) | |
1836 | continue; | |
1837 | return -errno; | |
1838 | } | |
1839 | ||
1840 | bytes -= r; | |
1841 | offset += r; | |
1842 | if (bytes == 0) break; | |
1843 | ||
1844 | while (r > 0) { | |
1845 | if (vec[0].iov_len <= (size_t)r) { | |
1846 | // drain this whole item | |
1847 | r -= vec[0].iov_len; | |
1848 | ++vec; | |
1849 | --veclen; | |
1850 | } else { | |
1851 | vec[0].iov_base = (char *)vec[0].iov_base + r; | |
1852 | vec[0].iov_len -= r; | |
1853 | break; | |
1854 | } | |
1855 | } | |
1856 | } | |
1857 | return 0; | |
1858 | } | |
1859 | ||
1860 | int buffer::list::write_fd(int fd) const | |
1861 | { | |
7c673cae FG |
1862 | // use writev! |
1863 | iovec iov[IOV_MAX]; | |
1864 | int iovlen = 0; | |
1865 | ssize_t bytes = 0; | |
1866 | ||
11fdf7f2 TL |
1867 | auto p = std::cbegin(_buffers); |
1868 | while (p != std::cend(_buffers)) { | |
7c673cae FG |
1869 | if (p->length() > 0) { |
1870 | iov[iovlen].iov_base = (void *)p->c_str(); | |
1871 | iov[iovlen].iov_len = p->length(); | |
1872 | bytes += p->length(); | |
1873 | iovlen++; | |
1874 | } | |
1875 | ++p; | |
1876 | ||
d2e6a577 | 1877 | if (iovlen == IOV_MAX || |
7c673cae FG |
1878 | p == _buffers.end()) { |
1879 | iovec *start = iov; | |
1880 | int num = iovlen; | |
1881 | ssize_t wrote; | |
1882 | retry: | |
1883 | wrote = ::writev(fd, start, num); | |
1884 | if (wrote < 0) { | |
1885 | int err = errno; | |
1886 | if (err == EINTR) | |
1887 | goto retry; | |
1888 | return -err; | |
1889 | } | |
1890 | if (wrote < bytes) { | |
1891 | // partial write, recover! | |
1892 | while ((size_t)wrote >= start[0].iov_len) { | |
1893 | wrote -= start[0].iov_len; | |
1894 | bytes -= start[0].iov_len; | |
1895 | start++; | |
1896 | num--; | |
1897 | } | |
1898 | if (wrote > 0) { | |
1899 | start[0].iov_len -= wrote; | |
1900 | start[0].iov_base = (char *)start[0].iov_base + wrote; | |
1901 | bytes -= wrote; | |
1902 | } | |
1903 | goto retry; | |
1904 | } | |
1905 | iovlen = 0; | |
1906 | bytes = 0; | |
1907 | } | |
1908 | } | |
1909 | return 0; | |
1910 | } | |
1911 | ||
1912 | int buffer::list::write_fd(int fd, uint64_t offset) const | |
1913 | { | |
1914 | iovec iov[IOV_MAX]; | |
1915 | ||
11fdf7f2 | 1916 | auto p = std::cbegin(_buffers); |
9f95a23c | 1917 | uint64_t left_pbrs = get_num_buffers(); |
7c673cae FG |
1918 | while (left_pbrs) { |
1919 | ssize_t bytes = 0; | |
1920 | unsigned iovlen = 0; | |
11fdf7f2 | 1921 | uint64_t size = std::min<uint64_t>(left_pbrs, IOV_MAX); |
7c673cae FG |
1922 | left_pbrs -= size; |
1923 | while (size > 0) { | |
1924 | iov[iovlen].iov_base = (void *)p->c_str(); | |
1925 | iov[iovlen].iov_len = p->length(); | |
1926 | iovlen++; | |
1927 | bytes += p->length(); | |
1928 | ++p; | |
1929 | size--; | |
1930 | } | |
1931 | ||
1932 | int r = do_writev(fd, iov, offset, iovlen, bytes); | |
1933 | if (r < 0) | |
1934 | return r; | |
1935 | offset += bytes; | |
1936 | } | |
1937 | return 0; | |
1938 | } | |
1939 | ||
7c673cae FG |
1940 | __u32 buffer::list::crc32c(__u32 crc) const |
1941 | { | |
11fdf7f2 TL |
1942 | int cache_misses = 0; |
1943 | int cache_hits = 0; | |
1944 | int cache_adjusts = 0; | |
1945 | ||
1946 | for (const auto& node : _buffers) { | |
1947 | if (node.length()) { | |
9f95a23c | 1948 | raw* const r = node._raw; |
11fdf7f2 | 1949 | pair<size_t, size_t> ofs(node.offset(), node.offset() + node.length()); |
7c673cae FG |
1950 | pair<uint32_t, uint32_t> ccrc; |
1951 | if (r->get_crc(ofs, &ccrc)) { | |
1952 | if (ccrc.first == crc) { | |
1953 | // got it already | |
1954 | crc = ccrc.second; | |
11fdf7f2 | 1955 | cache_hits++; |
7c673cae FG |
1956 | } else { |
1957 | /* If we have cached crc32c(buf, v) for initial value v, | |
1958 | * we can convert this to a different initial value v' by: | |
1959 | * crc32c(buf, v') = crc32c(buf, v) ^ adjustment | |
1960 | * where adjustment = crc32c(0*len(buf), v ^ v') | |
1961 | * | |
1962 | * http://crcutil.googlecode.com/files/crc-doc.1.0.pdf | |
1963 | * note, u for our crc32c implementation is 0 | |
1964 | */ | |
11fdf7f2 TL |
1965 | crc = ccrc.second ^ ceph_crc32c(ccrc.first ^ crc, NULL, node.length()); |
1966 | cache_adjusts++; | |
7c673cae FG |
1967 | } |
1968 | } else { | |
11fdf7f2 | 1969 | cache_misses++; |
7c673cae | 1970 | uint32_t base = crc; |
11fdf7f2 | 1971 | crc = ceph_crc32c(crc, (unsigned char*)node.c_str(), node.length()); |
7c673cae FG |
1972 | r->set_crc(ofs, make_pair(base, crc)); |
1973 | } | |
1974 | } | |
1975 | } | |
11fdf7f2 TL |
1976 | |
1977 | if (buffer_track_crc) { | |
1978 | if (cache_adjusts) | |
1979 | buffer_cached_crc_adjusted += cache_adjusts; | |
1980 | if (cache_hits) | |
1981 | buffer_cached_crc += cache_hits; | |
1982 | if (cache_misses) | |
1983 | buffer_missed_crc += cache_misses; | |
1984 | } | |
1985 | ||
7c673cae FG |
1986 | return crc; |
1987 | } | |
1988 | ||
1989 | void buffer::list::invalidate_crc() | |
1990 | { | |
11fdf7f2 | 1991 | for (const auto& node : _buffers) { |
9f95a23c TL |
1992 | if (node._raw) { |
1993 | node._raw->invalidate_crc(); | |
7c673cae FG |
1994 | } |
1995 | } | |
1996 | } | |
1997 | ||
1998 | /** | |
1999 | * Binary write all contents to a C++ stream | |
2000 | */ | |
2001 | void buffer::list::write_stream(std::ostream &out) const | |
2002 | { | |
11fdf7f2 TL |
2003 | for (const auto& node : _buffers) { |
2004 | if (node.length() > 0) { | |
2005 | out.write(node.c_str(), node.length()); | |
7c673cae FG |
2006 | } |
2007 | } | |
2008 | } | |
2009 | ||
2010 | ||
2011 | void buffer::list::hexdump(std::ostream &out, bool trailing_newline) const | |
2012 | { | |
2013 | if (!length()) | |
2014 | return; | |
2015 | ||
2016 | std::ios_base::fmtflags original_flags = out.flags(); | |
2017 | ||
2018 | // do our best to match the output of hexdump -C, for better | |
2019 | // diff'ing! | |
2020 | ||
2021 | out.setf(std::ios::right); | |
2022 | out.fill('0'); | |
2023 | ||
2024 | unsigned per = 16; | |
9f95a23c TL |
2025 | char last_row_char = '\0'; |
2026 | bool was_same = false, did_star = false; | |
7c673cae | 2027 | for (unsigned o=0; o<length(); o += per) { |
9f95a23c TL |
2028 | if (o == 0) { |
2029 | last_row_char = (*this)[o]; | |
2030 | } | |
2031 | ||
7c673cae | 2032 | if (o + per < length()) { |
9f95a23c | 2033 | bool row_is_same = true; |
7c673cae | 2034 | for (unsigned i=0; i<per && o+i<length(); i++) { |
9f95a23c TL |
2035 | char current_char = (*this)[o+i]; |
2036 | if (current_char != last_row_char) { | |
2037 | if (i == 0) { | |
2038 | last_row_char = current_char; | |
2039 | was_same = false; | |
2040 | did_star = false; | |
2041 | } else { | |
2042 | row_is_same = false; | |
2043 | } | |
7c673cae FG |
2044 | } |
2045 | } | |
9f95a23c TL |
2046 | if (row_is_same) { |
2047 | if (was_same) { | |
7c673cae FG |
2048 | if (!did_star) { |
2049 | out << "\n*"; | |
2050 | did_star = true; | |
2051 | } | |
2052 | continue; | |
2053 | } | |
9f95a23c | 2054 | was_same = true; |
7c673cae | 2055 | } else { |
9f95a23c | 2056 | was_same = false; |
7c673cae FG |
2057 | did_star = false; |
2058 | } | |
2059 | } | |
2060 | if (o) | |
2061 | out << "\n"; | |
2062 | out << std::hex << std::setw(8) << o << " "; | |
2063 | ||
2064 | unsigned i; | |
2065 | for (i=0; i<per && o+i<length(); i++) { | |
2066 | if (i == 8) | |
2067 | out << ' '; | |
2068 | out << " " << std::setw(2) << ((unsigned)(*this)[o+i] & 0xff); | |
2069 | } | |
2070 | for (; i<per; i++) { | |
2071 | if (i == 8) | |
2072 | out << ' '; | |
2073 | out << " "; | |
2074 | } | |
2075 | ||
2076 | out << " |"; | |
2077 | for (i=0; i<per && o+i<length(); i++) { | |
2078 | char c = (*this)[o+i]; | |
2079 | if (isupper(c) || islower(c) || isdigit(c) || c == ' ' || ispunct(c)) | |
2080 | out << c; | |
2081 | else | |
2082 | out << '.'; | |
2083 | } | |
2084 | out << '|' << std::dec; | |
2085 | } | |
2086 | if (trailing_newline) { | |
2087 | out << "\n" << std::hex << std::setw(8) << length(); | |
2088 | out << "\n"; | |
2089 | } | |
2090 | ||
2091 | out.flags(original_flags); | |
2092 | } | |
2093 | ||
31f18b77 FG |
2094 | |
2095 | buffer::list buffer::list::static_from_mem(char* c, size_t l) { | |
2096 | list bl; | |
11fdf7f2 | 2097 | bl.push_back(ptr_node::create(create_static(l, c))); |
31f18b77 FG |
2098 | return bl; |
2099 | } | |
2100 | ||
2101 | buffer::list buffer::list::static_from_cstring(char* c) { | |
2102 | return static_from_mem(c, std::strlen(c)); | |
2103 | } | |
2104 | ||
2105 | buffer::list buffer::list::static_from_string(string& s) { | |
2106 | // C++14 just has string::data return a char* from a non-const | |
2107 | // string. | |
2108 | return static_from_mem(const_cast<char*>(s.data()), s.length()); | |
2109 | // But the way buffer::list mostly doesn't work in a sane way with | |
2110 | // const makes me generally sad. | |
2111 | } | |
2112 | ||
11fdf7f2 TL |
2113 | bool buffer::ptr_node::dispose_if_hypercombined( |
2114 | buffer::ptr_node* const delete_this) | |
2115 | { | |
2116 | const bool is_hypercombined = static_cast<void*>(delete_this) == \ | |
9f95a23c | 2117 | static_cast<void*>(&delete_this->_raw->bptr_storage); |
11fdf7f2 TL |
2118 | if (is_hypercombined) { |
2119 | ceph_assert_always("hypercombining is currently disabled" == nullptr); | |
2120 | delete_this->~ptr_node(); | |
2121 | } | |
2122 | return is_hypercombined; | |
2123 | } | |
2124 | ||
2125 | std::unique_ptr<buffer::ptr_node, buffer::ptr_node::disposer> | |
2126 | buffer::ptr_node::create_hypercombined(ceph::unique_leakable_ptr<buffer::raw> r) | |
2127 | { | |
2128 | // FIXME: we don't currently hypercombine buffers due to crashes | |
2129 | // observed in the rados suite. After fixing we'll use placement | |
2130 | // new to create ptr_node on buffer::raw::bptr_storage. | |
2131 | return std::unique_ptr<buffer::ptr_node, buffer::ptr_node::disposer>( | |
2132 | new ptr_node(std::move(r))); | |
2133 | } | |
2134 | ||
11fdf7f2 TL |
2135 | buffer::ptr_node* buffer::ptr_node::cloner::operator()( |
2136 | const buffer::ptr_node& clone_this) | |
2137 | { | |
9f95a23c | 2138 | return new ptr_node(clone_this); |
11fdf7f2 TL |
2139 | } |
2140 | ||
7c673cae | 2141 | std::ostream& buffer::operator<<(std::ostream& out, const buffer::raw &r) { |
31f18b77 | 2142 | return out << "buffer::raw(" << (void*)r.data << " len " << r.len << " nref " << r.nref.load() << ")"; |
7c673cae FG |
2143 | } |
2144 | ||
2145 | std::ostream& buffer::operator<<(std::ostream& out, const buffer::ptr& bp) { | |
2146 | if (bp.have_raw()) | |
2147 | out << "buffer::ptr(" << bp.offset() << "~" << bp.length() | |
2148 | << " " << (void*)bp.c_str() | |
2149 | << " in raw " << (void*)bp.raw_c_str() | |
2150 | << " len " << bp.raw_length() | |
2151 | << " nref " << bp.raw_nref() << ")"; | |
2152 | else | |
2153 | out << "buffer:ptr(" << bp.offset() << "~" << bp.length() << " no raw)"; | |
2154 | return out; | |
2155 | } | |
2156 | ||
2157 | std::ostream& buffer::operator<<(std::ostream& out, const buffer::list& bl) { | |
2158 | out << "buffer::list(len=" << bl.length() << "," << std::endl; | |
2159 | ||
11fdf7f2 TL |
2160 | for (const auto& node : bl.buffers()) { |
2161 | out << "\t" << node; | |
2162 | if (&node != &bl.buffers().back()) { | |
2163 | out << "," << std::endl; | |
2164 | } | |
7c673cae FG |
2165 | } |
2166 | out << std::endl << ")"; | |
2167 | return out; | |
2168 | } | |
2169 | ||
2170 | std::ostream& buffer::operator<<(std::ostream& out, const buffer::error& e) | |
2171 | { | |
2172 | return out << e.what(); | |
2173 | } | |
2174 | ||
2175 | MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_malloc, buffer_raw_malloc, | |
2176 | buffer_meta); | |
7c673cae FG |
2177 | MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_posix_aligned, |
2178 | buffer_raw_posix_aligned, buffer_meta); | |
7c673cae | 2179 | MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_char, buffer_raw_char, buffer_meta); |
31f18b77 FG |
2180 | MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_claimed_char, buffer_raw_claimed_char, |
2181 | buffer_meta); | |
7c673cae FG |
2182 | MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_static, buffer_raw_static, |
2183 | buffer_meta); | |
2184 |