]>
git.proxmox.com Git - rustc.git/blob - src/jemalloc/src/quarantine.c
1 #define JEMALLOC_QUARANTINE_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
5 * Quarantine pointers close to NULL are used to encode state information that
6 * is used for cleaning up during thread shutdown.
8 #define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1)
9 #define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2)
10 #define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY
12 /******************************************************************************/
13 /* Function prototypes for non-inline static functions. */
15 static quarantine_t
*quarantine_grow(tsd_t
*tsd
, quarantine_t
*quarantine
);
16 static void quarantine_drain_one(tsdn_t
*tsdn
, quarantine_t
*quarantine
);
17 static void quarantine_drain(tsdn_t
*tsdn
, quarantine_t
*quarantine
,
20 /******************************************************************************/
23 quarantine_init(tsdn_t
*tsdn
, size_t lg_maxobjs
)
25 quarantine_t
*quarantine
;
28 size
= offsetof(quarantine_t
, objs
) + ((ZU(1) << lg_maxobjs
) *
29 sizeof(quarantine_obj_t
));
30 quarantine
= (quarantine_t
*)iallocztm(tsdn
, size
, size2index(size
),
31 false, NULL
, true, arena_get(TSDN_NULL
, 0, true), true);
32 if (quarantine
== NULL
)
34 quarantine
->curbytes
= 0;
35 quarantine
->curobjs
= 0;
36 quarantine
->first
= 0;
37 quarantine
->lg_maxobjs
= lg_maxobjs
;
43 quarantine_alloc_hook_work(tsd_t
*tsd
)
45 quarantine_t
*quarantine
;
47 if (!tsd_nominal(tsd
))
50 quarantine
= quarantine_init(tsd_tsdn(tsd
), LG_MAXOBJS_INIT
);
52 * Check again whether quarantine has been initialized, because
53 * quarantine_init() may have triggered recursive initialization.
55 if (tsd_quarantine_get(tsd
) == NULL
)
56 tsd_quarantine_set(tsd
, quarantine
);
58 idalloctm(tsd_tsdn(tsd
), quarantine
, NULL
, true, true);
62 quarantine_grow(tsd_t
*tsd
, quarantine_t
*quarantine
)
66 ret
= quarantine_init(tsd_tsdn(tsd
), quarantine
->lg_maxobjs
+ 1);
68 quarantine_drain_one(tsd_tsdn(tsd
), quarantine
);
72 ret
->curbytes
= quarantine
->curbytes
;
73 ret
->curobjs
= quarantine
->curobjs
;
74 if (quarantine
->first
+ quarantine
->curobjs
<= (ZU(1) <<
75 quarantine
->lg_maxobjs
)) {
76 /* objs ring buffer data are contiguous. */
77 memcpy(ret
->objs
, &quarantine
->objs
[quarantine
->first
],
78 quarantine
->curobjs
* sizeof(quarantine_obj_t
));
80 /* objs ring buffer data wrap around. */
81 size_t ncopy_a
= (ZU(1) << quarantine
->lg_maxobjs
) -
83 size_t ncopy_b
= quarantine
->curobjs
- ncopy_a
;
85 memcpy(ret
->objs
, &quarantine
->objs
[quarantine
->first
], ncopy_a
86 * sizeof(quarantine_obj_t
));
87 memcpy(&ret
->objs
[ncopy_a
], quarantine
->objs
, ncopy_b
*
88 sizeof(quarantine_obj_t
));
90 idalloctm(tsd_tsdn(tsd
), quarantine
, NULL
, true, true);
92 tsd_quarantine_set(tsd
, ret
);
97 quarantine_drain_one(tsdn_t
*tsdn
, quarantine_t
*quarantine
)
99 quarantine_obj_t
*obj
= &quarantine
->objs
[quarantine
->first
];
100 assert(obj
->usize
== isalloc(tsdn
, obj
->ptr
, config_prof
));
101 idalloctm(tsdn
, obj
->ptr
, NULL
, false, true);
102 quarantine
->curbytes
-= obj
->usize
;
103 quarantine
->curobjs
--;
104 quarantine
->first
= (quarantine
->first
+ 1) & ((ZU(1) <<
105 quarantine
->lg_maxobjs
) - 1);
109 quarantine_drain(tsdn_t
*tsdn
, quarantine_t
*quarantine
, size_t upper_bound
)
112 while (quarantine
->curbytes
> upper_bound
&& quarantine
->curobjs
> 0)
113 quarantine_drain_one(tsdn
, quarantine
);
117 quarantine(tsd_t
*tsd
, void *ptr
)
119 quarantine_t
*quarantine
;
120 size_t usize
= isalloc(tsd_tsdn(tsd
), ptr
, config_prof
);
122 cassert(config_fill
);
123 assert(opt_quarantine
);
125 if ((quarantine
= tsd_quarantine_get(tsd
)) == NULL
) {
126 idalloctm(tsd_tsdn(tsd
), ptr
, NULL
, false, true);
130 * Drain one or more objects if the quarantine size limit would be
131 * exceeded by appending ptr.
133 if (quarantine
->curbytes
+ usize
> opt_quarantine
) {
134 size_t upper_bound
= (opt_quarantine
>= usize
) ? opt_quarantine
136 quarantine_drain(tsd_tsdn(tsd
), quarantine
, upper_bound
);
138 /* Grow the quarantine ring buffer if it's full. */
139 if (quarantine
->curobjs
== (ZU(1) << quarantine
->lg_maxobjs
))
140 quarantine
= quarantine_grow(tsd
, quarantine
);
141 /* quarantine_grow() must free a slot if it fails to grow. */
142 assert(quarantine
->curobjs
< (ZU(1) << quarantine
->lg_maxobjs
));
143 /* Append ptr if its size doesn't exceed the quarantine size. */
144 if (quarantine
->curbytes
+ usize
<= opt_quarantine
) {
145 size_t offset
= (quarantine
->first
+ quarantine
->curobjs
) &
146 ((ZU(1) << quarantine
->lg_maxobjs
) - 1);
147 quarantine_obj_t
*obj
= &quarantine
->objs
[offset
];
150 quarantine
->curbytes
+= usize
;
151 quarantine
->curobjs
++;
152 if (config_fill
&& unlikely(opt_junk_free
)) {
154 * Only do redzone validation if Valgrind isn't in
157 if ((!config_valgrind
|| likely(!in_valgrind
))
158 && usize
<= SMALL_MAXCLASS
)
159 arena_quarantine_junk_small(ptr
, usize
);
161 memset(ptr
, JEMALLOC_FREE_JUNK
, usize
);
164 assert(quarantine
->curbytes
== 0);
165 idalloctm(tsd_tsdn(tsd
), ptr
, NULL
, false, true);
170 quarantine_cleanup(tsd_t
*tsd
)
172 quarantine_t
*quarantine
;
177 quarantine
= tsd_quarantine_get(tsd
);
178 if (quarantine
!= NULL
) {
179 quarantine_drain(tsd_tsdn(tsd
), quarantine
, 0);
180 idalloctm(tsd_tsdn(tsd
), quarantine
, NULL
, true, true);
181 tsd_quarantine_set(tsd
, NULL
);