]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | #ifndef CEPH_INLINE_MEMORY_H | |
15 | #define CEPH_INLINE_MEMORY_H | |
16 | ||
17 | #if defined(__GNUC__) | |
18 | ||
19 | // optimize for the common case, which is very small copies | |
20 | static inline void *maybe_inline_memcpy(void *dest, const void *src, size_t l, | |
21 | size_t inline_len) | |
22 | __attribute__((always_inline)); | |
23 | ||
24 | void *maybe_inline_memcpy(void *dest, const void *src, size_t l, | |
25 | size_t inline_len) | |
26 | { | |
27 | if (l > inline_len) { | |
28 | return memcpy(dest, src, l); | |
29 | } | |
30 | switch (l) { | |
31 | case 8: | |
32 | return __builtin_memcpy(dest, src, 8); | |
33 | case 4: | |
34 | return __builtin_memcpy(dest, src, 4); | |
35 | case 3: | |
36 | return __builtin_memcpy(dest, src, 3); | |
37 | case 2: | |
38 | return __builtin_memcpy(dest, src, 2); | |
39 | case 1: | |
40 | return __builtin_memcpy(dest, src, 1); | |
41 | default: | |
42 | int cursor = 0; | |
43 | while (l >= sizeof(uint64_t)) { | |
44 | __builtin_memcpy((char*)dest + cursor, (char*)src + cursor, | |
45 | sizeof(uint64_t)); | |
46 | cursor += sizeof(uint64_t); | |
47 | l -= sizeof(uint64_t); | |
48 | } | |
49 | while (l >= sizeof(uint32_t)) { | |
50 | __builtin_memcpy((char*)dest + cursor, (char*)src + cursor, | |
51 | sizeof(uint32_t)); | |
52 | cursor += sizeof(uint32_t); | |
53 | l -= sizeof(uint32_t); | |
54 | } | |
55 | while (l > 0) { | |
56 | *((char*)dest + cursor) = *((char*)src + cursor); | |
57 | cursor++; | |
58 | l--; | |
59 | } | |
60 | } | |
61 | return dest; | |
62 | } | |
63 | ||
64 | #else | |
65 | ||
66 | #define maybe_inline_memcpy(d, s, l, x) memcpy(d, s, l) | |
67 | ||
68 | #endif | |
69 | ||
70 | ||
71 | #if defined(__GNUC__) && defined(__x86_64__) | |
72 | ||
73 | typedef unsigned uint128_t __attribute__ ((mode (TI))); | |
74 | ||
75 | static inline bool mem_is_zero(const char *data, size_t len) | |
76 | __attribute__((always_inline)); | |
77 | ||
78 | bool mem_is_zero(const char *data, size_t len) | |
79 | { | |
80 | // we do have XMM registers in x86-64, so if we need to check at least | |
81 | // 16 bytes, make use of them | |
82 | if (len / sizeof(uint128_t) > 0) { | |
83 | // align data pointer to 16 bytes, otherwise it'll segfault due to bug | |
84 | // in (at least some) GCC versions (using MOVAPS instead of MOVUPS). | |
85 | // check up to 15 first bytes while at it. | |
86 | while (((unsigned long long)data) & 15) { | |
87 | if (*(uint8_t*)data != 0) { | |
88 | return false; | |
89 | } | |
90 | data += sizeof(uint8_t); | |
91 | --len; | |
92 | } | |
93 | ||
94 | const char* data_start = data; | |
95 | const char* max128 = data + (len / sizeof(uint128_t))*sizeof(uint128_t); | |
96 | ||
97 | while (data < max128) { | |
98 | if (*(uint128_t*)data != 0) { | |
99 | return false; | |
100 | } | |
101 | data += sizeof(uint128_t); | |
102 | } | |
103 | len -= (data - data_start); | |
104 | } | |
105 | ||
106 | const char* max = data + len; | |
107 | const char* max32 = data + (len / sizeof(uint32_t))*sizeof(uint32_t); | |
108 | while (data < max32) { | |
109 | if (*(uint32_t*)data != 0) { | |
110 | return false; | |
111 | } | |
112 | data += sizeof(uint32_t); | |
113 | } | |
114 | while (data < max) { | |
115 | if (*(uint8_t*)data != 0) { | |
116 | return false; | |
117 | } | |
118 | data += sizeof(uint8_t); | |
119 | } | |
120 | return true; | |
121 | } | |
122 | ||
123 | #else // gcc and x86_64 | |
124 | ||
125 | static inline bool mem_is_zero(const char *data, size_t len) { | |
126 | const char *end = data + len; | |
31f18b77 FG |
127 | const char* end64 = data + (len / sizeof(uint64_t))*sizeof(uint64_t); |
128 | ||
129 | while (data < end64) { | |
130 | if (*(uint64_t*)data != 0) { | |
131 | return false; | |
132 | } | |
133 | data += sizeof(uint64_t); | |
134 | } | |
135 | ||
7c673cae FG |
136 | while (data < end) { |
137 | if (*data != 0) { | |
138 | return false; | |
139 | } | |
140 | ++data; | |
141 | } | |
142 | return true; | |
143 | } | |
144 | ||
145 | #endif // !x86_64 | |
146 | ||
147 | #endif |