[rustc.git] / src / jemalloc / src / pages.c

#define	JEMALLOC_PAGES_C_
#include "jemalloc/internal/jemalloc_internal.h"

#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
#include <sys/sysctl.h>
#endif

/******************************************************************************/
/* Data. */

#ifndef _WIN32
#  define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE)
#  define PAGES_PROT_DECOMMIT (PROT_NONE)
static int	mmap_flags;
#endif
static bool	os_overcommits;

/******************************************************************************/

void *
pages_map(void *addr, size_t size, bool *commit)
{
	void *ret;

	assert(size != 0);

	if (os_overcommits)
		*commit = true;

#ifdef _WIN32
	/*
	 * If VirtualAlloc can't allocate at the given address when one is
	 * given, it fails and returns NULL.
	 */
	ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),
	    PAGE_READWRITE);
#else
	/*
	 * We don't use MAP_FIXED here, because it can cause the *replacement*
	 * of existing mappings, and we only want to create new mappings.
	 */
	{
		int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;

		ret = mmap(addr, size, prot, mmap_flags, -1, 0);
	}
	assert(ret != NULL);

	if (ret == MAP_FAILED)
		ret = NULL;
	else if (addr != NULL && ret != addr) {
		/*
		 * We succeeded in mapping memory, but not in the right place.
		 */
		pages_unmap(ret, size);
		ret = NULL;
	}
#endif
	assert(ret == NULL || (addr == NULL && ret != addr)
	    || (addr != NULL && ret == addr));
	return (ret);
}

void
pages_unmap(void *addr, size_t size)
{

#ifdef _WIN32
	if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
#else
	if (munmap(addr, size) == -1)
#endif
	{
		char buf[BUFERROR_BUF];

		buferror(get_errno(), buf, sizeof(buf));
		malloc_printf("<jemalloc>: Error in "
#ifdef _WIN32
		              "VirtualFree"
#else
		              "munmap"
#endif
		              "(): %s\n", buf);
		if (opt_abort)
			abort();
	}
}

void *
pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
    bool *commit)
{
	void *ret = (void *)((uintptr_t)addr + leadsize);

	assert(alloc_size >= leadsize + size);
#ifdef _WIN32
	{
		void *new_addr;

		pages_unmap(addr, alloc_size);
		new_addr = pages_map(ret, size, commit);
		if (new_addr == ret)
			return (ret);
		if (new_addr)
			pages_unmap(new_addr, size);
		return (NULL);
	}
#else
	{
		size_t trailsize = alloc_size - leadsize - size;

		if (leadsize != 0)
			pages_unmap(addr, leadsize);
		if (trailsize != 0)
			pages_unmap((void *)((uintptr_t)ret + size), trailsize);
		return (ret);
	}
#endif
}

static bool
pages_commit_impl(void *addr, size_t size, bool commit)
{

	if (os_overcommits)
		return (true);

#ifdef _WIN32
	return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,
	    PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT)));
#else
	{
		int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
		void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED,
		    -1, 0);
		if (result == MAP_FAILED)
			return (true);
		if (result != addr) {
			/*
			 * We succeeded in mapping memory, but not in the right
			 * place.
			 */
			pages_unmap(result, size);
			return (true);
		}
		return (false);
	}
#endif
}

bool
pages_commit(void *addr, size_t size)
{

	return (pages_commit_impl(addr, size, true));
}

bool
pages_decommit(void *addr, size_t size)
{

	return (pages_commit_impl(addr, size, false));
}

bool
pages_purge(void *addr, size_t size)
{
	bool unzeroed;

#ifdef _WIN32
	VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
	unzeroed = true;
#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \
    defined(JEMALLOC_PURGE_MADVISE_DONTNEED))
#  if defined(JEMALLOC_PURGE_MADVISE_FREE)
#    define JEMALLOC_MADV_PURGE MADV_FREE
#    define JEMALLOC_MADV_ZEROS false
#  elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
#    define JEMALLOC_MADV_PURGE MADV_DONTNEED
#    define JEMALLOC_MADV_ZEROS true
#  else
#    error No madvise(2) flag defined for purging unused dirty pages
#  endif
	int err = madvise(addr, size, JEMALLOC_MADV_PURGE);
	unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0);
#  undef JEMALLOC_MADV_PURGE
#  undef JEMALLOC_MADV_ZEROS
#else
	/* Last resort no-op. */
	unzeroed = true;
#endif
	return (unzeroed);
}

bool
pages_huge(void *addr, size_t size)
{

	assert(PAGE_ADDR2BASE(addr) == addr);
	assert(PAGE_CEILING(size) == size);

#ifdef JEMALLOC_HAVE_MADVISE_HUGE
	return (madvise(addr, size, MADV_HUGEPAGE) != 0);
#else
	return (false);
#endif
}

bool
pages_nohuge(void *addr, size_t size)
{

	assert(PAGE_ADDR2BASE(addr) == addr);
	assert(PAGE_CEILING(size) == size);

#ifdef JEMALLOC_HAVE_MADVISE_HUGE
	return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);
#else
	return (false);
#endif
}

#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
static bool
os_overcommits_sysctl(void)
{
	int vm_overcommit;
	size_t sz;

	sz = sizeof(vm_overcommit);
	if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0)
		return (false); /* Error. */

	return ((vm_overcommit & 0x3) == 0);
}
#endif

#ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
/*
 * Use syscall(2) rather than {open,read,close}(2) when possible to avoid
 * reentry during bootstrapping if another library has interposed system call
 * wrappers.
 */
static bool
os_overcommits_proc(void)
{
	int fd;
	char buf[1];
	ssize_t nread;

#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
	fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY);
#elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat)
	fd = (int)syscall(SYS_openat,
	    AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY);
#else
	fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
#endif
	if (fd == -1)
		return (false); /* Error. */

#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read)
	nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf));
#else
	nread = read(fd, &buf, sizeof(buf));
#endif

#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
	syscall(SYS_close, fd);
#else
	close(fd);
#endif

	if (nread < 1)
		return (false); /* Error. */
	/*
	 * /proc/sys/vm/overcommit_memory meanings:
	 * 0: Heuristic overcommit.
	 * 1: Always overcommit.
	 * 2: Never overcommit.
	 */
	return (buf[0] == '0' || buf[0] == '1');
}
#endif

void
pages_boot(void)
{

#ifndef _WIN32
	mmap_flags = MAP_PRIVATE | MAP_ANON;
#endif

#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
	os_overcommits = os_overcommits_sysctl();
#elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY)
	os_overcommits = os_overcommits_proc();
#  ifdef MAP_NORESERVE
	if (os_overcommits)
		mmap_flags |= MAP_NORESERVE;
#  endif
#else
	os_overcommits = false;
#endif
}
Commit	Line	Data
54a0048b SL	1	#define JEMALLOC_PAGES_C_
	2	#include "jemalloc/internal/jemalloc_internal.h"
	3
3b2f2976 XL	4	#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
	5	#include <sys/sysctl.h>
	6	#endif
	7
	8	/******************************************************************************/
	9	/* Data. */
	10
	11	#ifndef _WIN32
	12	# define PAGES_PROT_COMMIT (PROT_READ \| PROT_WRITE)
	13	# define PAGES_PROT_DECOMMIT (PROT_NONE)
	14	static int mmap_flags;
	15	#endif
	16	static bool os_overcommits;
	17
54a0048b SL	18	/******************************************************************************/
	19
	20	void *
3b2f2976	21	pages_map(void addr, size_t size, bool commit)
54a0048b SL	22	{
	23	void *ret;
	24
	25	assert(size != 0);
	26
3b2f2976 XL	27	if (os_overcommits)
	28	*commit = true;
	29
54a0048b SL	30	#ifdef _WIN32
	31	/*
	32	* If VirtualAlloc can't allocate at the given address when one is
	33	* given, it fails and returns NULL.
	34	*/
3b2f2976	35	ret = VirtualAlloc(addr, size, MEM_RESERVE \| (*commit ? MEM_COMMIT : 0),
54a0048b SL	36	PAGE_READWRITE);
	37	#else
	38	/*
	39	* We don't use MAP_FIXED here, because it can cause the replacement
	40	* of existing mappings, and we only want to create new mappings.
	41	*/
3b2f2976 XL	42	{
	43	int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
	44
	45	ret = mmap(addr, size, prot, mmap_flags, -1, 0);
	46	}
54a0048b SL	47	assert(ret != NULL);
	48
	49	if (ret == MAP_FAILED)
	50	ret = NULL;
	51	else if (addr != NULL && ret != addr) {
	52	/*
	53	* We succeeded in mapping memory, but not in the right place.
	54	*/
	55	pages_unmap(ret, size);
	56	ret = NULL;
	57	}
	58	#endif
	59	assert(ret == NULL \|\| (addr == NULL && ret != addr)
	60	\|\| (addr != NULL && ret == addr));
	61	return (ret);
	62	}
	63
	64	void
	65	pages_unmap(void *addr, size_t size)
	66	{
	67
	68	#ifdef _WIN32
	69	if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
	70	#else
	71	if (munmap(addr, size) == -1)
	72	#endif
	73	{
	74	char buf[BUFERROR_BUF];
	75
	76	buferror(get_errno(), buf, sizeof(buf));
	77	malloc_printf("<jemalloc>: Error in "
	78	#ifdef _WIN32
	79	"VirtualFree"
	80	#else
	81	"munmap"
	82	#endif
	83	"(): %s\n", buf);
	84	if (opt_abort)
	85	abort();
	86	}
	87	}
	88
	89	void *
3b2f2976 XL	90	pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
3b2f2976 XL	91	bool *commit)
54a0048b SL	92	{
	93	void ret = (void )((uintptr_t)addr + leadsize);
	94
	95	assert(alloc_size >= leadsize + size);
	96	#ifdef _WIN32
	97	{
	98	void *new_addr;
	99
	100	pages_unmap(addr, alloc_size);
3b2f2976	101	new_addr = pages_map(ret, size, commit);
54a0048b SL	102	if (new_addr == ret)
	103	return (ret);
	104	if (new_addr)
	105	pages_unmap(new_addr, size);
	106	return (NULL);
	107	}
	108	#else
	109	{
	110	size_t trailsize = alloc_size - leadsize - size;
	111
	112	if (leadsize != 0)
	113	pages_unmap(addr, leadsize);
	114	if (trailsize != 0)
	115	pages_unmap((void *)((uintptr_t)ret + size), trailsize);
	116	return (ret);
	117	}
	118	#endif
	119	}
	120
	121	static bool
	122	pages_commit_impl(void *addr, size_t size, bool commit)
	123	{
	124
3b2f2976 XL	125	if (os_overcommits)
	126	return (true);
	127
	128	#ifdef _WIN32
	129	return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,
	130	PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT)));
	131	#else
	132	{
	133	int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
	134	void *result = mmap(addr, size, prot, mmap_flags \| MAP_FIXED,
	135	-1, 0);
54a0048b SL	136	if (result == MAP_FAILED)
	137	return (true);
	138	if (result != addr) {
	139	/*
	140	* We succeeded in mapping memory, but not in the right
	141	* place.
	142	*/
	143	pages_unmap(result, size);
	144	return (true);
	145	}
	146	return (false);
	147	}
	148	#endif
54a0048b SL	149	}
	150
	151	bool
	152	pages_commit(void *addr, size_t size)
	153	{
	154
	155	return (pages_commit_impl(addr, size, true));
	156	}
	157
	158	bool
	159	pages_decommit(void *addr, size_t size)
	160	{
	161
	162	return (pages_commit_impl(addr, size, false));
	163	}
	164
	165	bool
	166	pages_purge(void *addr, size_t size)
	167	{
	168	bool unzeroed;
	169
	170	#ifdef _WIN32
	171	VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
	172	unzeroed = true;
3b2f2976 XL	173	#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) \|\| \
	174	defined(JEMALLOC_PURGE_MADVISE_DONTNEED))
	175	# if defined(JEMALLOC_PURGE_MADVISE_FREE)
54a0048b SL	176	# define JEMALLOC_MADV_PURGE MADV_FREE
54a0048b SL	177	# define JEMALLOC_MADV_ZEROS false
3b2f2976 XL	178	# elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
	179	# define JEMALLOC_MADV_PURGE MADV_DONTNEED
	180	# define JEMALLOC_MADV_ZEROS true
54a0048b	181	# else
3b2f2976	182	# error No madvise(2) flag defined for purging unused dirty pages
54a0048b SL	183	# endif
	184	int err = madvise(addr, size, JEMALLOC_MADV_PURGE);
	185	unzeroed = (!JEMALLOC_MADV_ZEROS \|\| err != 0);
	186	# undef JEMALLOC_MADV_PURGE
	187	# undef JEMALLOC_MADV_ZEROS
	188	#else
	189	/* Last resort no-op. */
	190	unzeroed = true;
	191	#endif
	192	return (unzeroed);
	193	}
	194
3b2f2976 XL	195	bool
	196	pages_huge(void *addr, size_t size)
	197	{
	198
	199	assert(PAGE_ADDR2BASE(addr) == addr);
	200	assert(PAGE_CEILING(size) == size);
	201
	202	#ifdef JEMALLOC_HAVE_MADVISE_HUGE
	203	return (madvise(addr, size, MADV_HUGEPAGE) != 0);
	204	#else
	205	return (false);
	206	#endif
	207	}
	208
	209	bool
	210	pages_nohuge(void *addr, size_t size)
	211	{
	212
	213	assert(PAGE_ADDR2BASE(addr) == addr);
	214	assert(PAGE_CEILING(size) == size);
	215
	216	#ifdef JEMALLOC_HAVE_MADVISE_HUGE
	217	return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);
	218	#else
	219	return (false);
	220	#endif
	221	}
	222
	223	#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
	224	static bool
	225	os_overcommits_sysctl(void)
	226	{
	227	int vm_overcommit;
	228	size_t sz;
	229
	230	sz = sizeof(vm_overcommit);
	231	if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0)
	232	return (false); /* Error. */
	233
	234	return ((vm_overcommit & 0x3) == 0);
	235	}
	236	#endif
	237
	238	#ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
	239	/*
	240	* Use syscall(2) rather than {open,read,close}(2) when possible to avoid
	241	* reentry during bootstrapping if another library has interposed system call
	242	* wrappers.
	243	*/
	244	static bool
	245	os_overcommits_proc(void)
	246	{
	247	int fd;
	248	char buf[1];
	249	ssize_t nread;
	250
	251	#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
	252	fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY);
	253	#elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat)
	254	fd = (int)syscall(SYS_openat,
	255	AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY);
	256	#else
	257	fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
	258	#endif
259	if (fd == -1)
260	return (false); /* Error. */
261
262	#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read)
263	nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf));
264	#else
265	nread = read(fd, &buf, sizeof(buf));
266	#endif
267
268	#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
269	syscall(SYS_close, fd);
270	#else
271	close(fd);
272	#endif
273
274	if (nread < 1)
275	return (false); /* Error. */
276	/*
277	* /proc/sys/vm/overcommit_memory meanings:
278	* 0: Heuristic overcommit.
279	* 1: Always overcommit.
280	* 2: Never overcommit.
281	*/
282	return (buf[0] == '0' \|\| buf[0] == '1');
283	}
284	#endif
285
286	void
287	pages_boot(void)
288	{
289
290	#ifndef _WIN32
291	mmap_flags = MAP_PRIVATE \| MAP_ANON;
292	#endif
293
294	#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
295	os_overcommits = os_overcommits_sysctl();
296	#elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY)
297	os_overcommits = os_overcommits_proc();
298	# ifdef MAP_NORESERVE
299	if (os_overcommits)
300	mmap_flags \|= MAP_NORESERVE;
301	# endif
302	#else
303	os_overcommits = false;
304	#endif
305	}