1 /* $Id: memobj-r0drv-linux.c $ */
3 * IPRT - Ring-0 Memory Objects, Linux.
7 * Copyright (C) 2006-2016 Oracle Corporation
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
28 /*********************************************************************************************************************************
30 *********************************************************************************************************************************/
31 #include "the-linux-kernel.h"
33 #include <iprt/memobj.h>
34 #include <iprt/alloc.h>
35 #include <iprt/assert.h>
37 #include <iprt/process.h>
38 #include <iprt/string.h>
39 #include "internal/memobj.h"
42 /*********************************************************************************************************************************
43 * Defined Constants And Macros *
44 *********************************************************************************************************************************/
45 /* early 2.6 kernels */
46 #ifndef PAGE_SHARED_EXEC
47 # define PAGE_SHARED_EXEC PAGE_SHARED
49 #ifndef PAGE_READONLY_EXEC
50 # define PAGE_READONLY_EXEC PAGE_READONLY
54 * 2.6.29+ kernels don't work with remap_pfn_range() anymore because
55 * track_pfn_vma_new() is apparently not defined for non-RAM pages.
56 * It should be safe to use vm_insert_page() older kernels as well.
58 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
59 # define VBOX_USE_INSERT_PAGE
61 #if defined(CONFIG_X86_PAE) \
62 && ( defined(HAVE_26_STYLE_REMAP_PAGE_RANGE) \
63 || ( LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) \
64 && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)))
65 # define VBOX_USE_PAE_HACK
69 /*********************************************************************************************************************************
70 * Structures and Typedefs *
71 *********************************************************************************************************************************/
73 * The Darwin version of the memory object structure.
75 typedef struct RTR0MEMOBJLNX
77 /** The core structure. */
78 RTR0MEMOBJINTERNAL Core
;
79 /** Set if the allocation is contiguous.
80 * This means it has to be given back as one chunk. */
82 /** Set if we've vmap'ed the memory into ring-0. */
84 /** The pages in the apPages array. */
86 /** Array of struct page pointers. (variable size) */
87 struct page
*apPages
[1];
88 } RTR0MEMOBJLNX
, *PRTR0MEMOBJLNX
;
91 static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx
);
95 * Helper that converts from a RTR0PROCESS handle to a linux task.
97 * @returns The corresponding Linux task.
98 * @param R0Process IPRT ring-0 process handle.
100 static struct task_struct
*rtR0ProcessToLinuxTask(RTR0PROCESS R0Process
)
102 /** @todo fix rtR0ProcessToLinuxTask!! */
103 /** @todo many (all?) callers currently assume that we return 'current'! */
104 return R0Process
== RTR0ProcHandleSelf() ? current
: NULL
;
109 * Compute order. Some functions allocate 2^order pages.
112 * @param cPages Number of pages.
114 static int rtR0MemObjLinuxOrder(size_t cPages
)
119 for (iOrder
= 0, cTmp
= cPages
; cTmp
>>= 1; ++iOrder
)
121 if (cPages
& ~((size_t)1 << iOrder
))
129 * Converts from RTMEM_PROT_* to Linux PAGE_*.
131 * @returns Linux page protection constant.
132 * @param fProt The IPRT protection mask.
133 * @param fKernel Whether it applies to kernel or user space.
135 static pgprot_t
rtR0MemObjLinuxConvertProt(unsigned fProt
, bool fKernel
)
140 AssertMsgFailed(("%#x %d\n", fProt
, fKernel
));
141 case RTMEM_PROT_NONE
:
144 case RTMEM_PROT_READ
:
145 return fKernel
? PAGE_KERNEL_RO
: PAGE_READONLY
;
147 case RTMEM_PROT_WRITE
:
148 case RTMEM_PROT_WRITE
| RTMEM_PROT_READ
:
149 return fKernel
? PAGE_KERNEL
: PAGE_SHARED
;
151 case RTMEM_PROT_EXEC
:
152 case RTMEM_PROT_EXEC
| RTMEM_PROT_READ
:
153 #if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
156 pgprot_t fPg
= MY_PAGE_KERNEL_EXEC
;
157 pgprot_val(fPg
) &= ~_PAGE_RW
;
160 return PAGE_READONLY_EXEC
;
162 return fKernel
? MY_PAGE_KERNEL_EXEC
: PAGE_READONLY_EXEC
;
165 case RTMEM_PROT_WRITE
| RTMEM_PROT_EXEC
:
166 case RTMEM_PROT_WRITE
| RTMEM_PROT_EXEC
| RTMEM_PROT_READ
:
167 return fKernel
? MY_PAGE_KERNEL_EXEC
: PAGE_SHARED_EXEC
;
173 * Worker for rtR0MemObjNativeReserveUser and rtR0MemObjNativerMapUser that creates
174 * an empty user space mapping.
176 * We acquire the mmap_sem of the task!
178 * @returns Pointer to the mapping.
179 * (void *)-1 on failure.
180 * @param R3PtrFixed (RTR3PTR)-1 if anywhere, otherwise a specific location.
181 * @param cb The size of the mapping.
182 * @param uAlignment The alignment of the mapping.
183 * @param pTask The Linux task to create this mapping in.
184 * @param fProt The RTMEM_PROT_* mask.
186 static void *rtR0MemObjLinuxDoMmap(RTR3PTR R3PtrFixed
, size_t cb
, size_t uAlignment
, struct task_struct
*pTask
, unsigned fProt
)
189 unsigned long ulAddr
;
191 Assert(pTask
== current
); /* do_mmap */
195 * Convert from IPRT protection to mman.h PROT_ and call do_mmap.
197 fProt
&= (RTMEM_PROT_NONE
| RTMEM_PROT_READ
| RTMEM_PROT_WRITE
| RTMEM_PROT_EXEC
);
198 if (fProt
== RTMEM_PROT_NONE
)
199 fLnxProt
= PROT_NONE
;
203 if (fProt
& RTMEM_PROT_READ
)
204 fLnxProt
|= PROT_READ
;
205 if (fProt
& RTMEM_PROT_WRITE
)
206 fLnxProt
|= PROT_WRITE
;
207 if (fProt
& RTMEM_PROT_EXEC
)
208 fLnxProt
|= PROT_EXEC
;
211 if (R3PtrFixed
!= (RTR3PTR
)-1)
213 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
214 ulAddr
= vm_mmap(NULL
, R3PtrFixed
, cb
, fLnxProt
, MAP_SHARED
| MAP_ANONYMOUS
| MAP_FIXED
, 0);
216 down_write(&pTask
->mm
->mmap_sem
);
217 ulAddr
= do_mmap(NULL
, R3PtrFixed
, cb
, fLnxProt
, MAP_SHARED
| MAP_ANONYMOUS
| MAP_FIXED
, 0);
218 up_write(&pTask
->mm
->mmap_sem
);
223 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
224 ulAddr
= vm_mmap(NULL
, 0, cb
, fLnxProt
, MAP_SHARED
| MAP_ANONYMOUS
, 0);
226 down_write(&pTask
->mm
->mmap_sem
);
227 ulAddr
= do_mmap(NULL
, 0, cb
, fLnxProt
, MAP_SHARED
| MAP_ANONYMOUS
, 0);
228 up_write(&pTask
->mm
->mmap_sem
);
230 if ( !(ulAddr
& ~PAGE_MASK
)
231 && (ulAddr
& (uAlignment
- 1)))
233 /** @todo implement uAlignment properly... We'll probably need to make some dummy mappings to fill
234 * up alignment gaps. This is of course complicated by fragmentation (which we might have cause
235 * ourselves) and further by there begin two mmap strategies (top / bottom). */
236 /* For now, just ignore uAlignment requirements... */
241 if (ulAddr
& ~PAGE_MASK
) /* ~PAGE_MASK == PAGE_OFFSET_MASK */
243 return (void *)ulAddr
;
248 * Worker that destroys a user space mapping.
249 * Undoes what rtR0MemObjLinuxDoMmap did.
251 * We acquire the mmap_sem of the task!
253 * @param pv The ring-3 mapping.
254 * @param cb The size of the mapping.
255 * @param pTask The Linux task to destroy this mapping in.
257 static void rtR0MemObjLinuxDoMunmap(void *pv
, size_t cb
, struct task_struct
*pTask
)
259 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
260 Assert(pTask
== current
); RT_NOREF_PV(pTask
);
261 vm_munmap((unsigned long)pv
, cb
);
262 #elif defined(USE_RHEL4_MUNMAP)
263 down_write(&pTask
->mm
->mmap_sem
);
264 do_munmap(pTask
->mm
, (unsigned long)pv
, cb
, 0); /* should it be 1 or 0? */
265 up_write(&pTask
->mm
->mmap_sem
);
267 down_write(&pTask
->mm
->mmap_sem
);
268 do_munmap(pTask
->mm
, (unsigned long)pv
, cb
);
269 up_write(&pTask
->mm
->mmap_sem
);
275 * Internal worker that allocates physical pages and creates the memory object for them.
277 * @returns IPRT status code.
278 * @param ppMemLnx Where to store the memory object pointer.
279 * @param enmType The object type.
280 * @param cb The number of bytes to allocate.
281 * @param uAlignment The alignment of the physical memory.
282 * Only valid if fContiguous == true, ignored otherwise.
283 * @param fFlagsLnx The page allocation flags (GPFs).
284 * @param fContiguous Whether the allocation must be contiguous.
285 * @param rcNoMem What to return when we're out of pages.
287 static int rtR0MemObjLinuxAllocPages(PRTR0MEMOBJLNX
*ppMemLnx
, RTR0MEMOBJTYPE enmType
, size_t cb
,
288 size_t uAlignment
, unsigned fFlagsLnx
, bool fContiguous
, int rcNoMem
)
291 size_t const cPages
= cb
>> PAGE_SHIFT
;
292 struct page
*paPages
;
295 * Allocate a memory object structure that's large enough to contain
296 * the page pointer array.
298 PRTR0MEMOBJLNX pMemLnx
= (PRTR0MEMOBJLNX
)rtR0MemObjNew(RT_OFFSETOF(RTR0MEMOBJLNX
, apPages
[cPages
]), enmType
, NULL
, cb
);
300 return VERR_NO_MEMORY
;
301 pMemLnx
->cPages
= cPages
;
306 /* Try hard to allocate the memory, but the allocation attempt might fail. */
307 fFlagsLnx
|= __GFP_REPEAT
;
309 # ifdef __GFP_NOMEMALLOC
310 /* Introduced with Linux 2.6.12: Don't use emergency reserves */
311 fFlagsLnx
|= __GFP_NOMEMALLOC
;
316 * Allocate the pages.
317 * For small allocations we'll try contiguous first and then fall back on page by page.
319 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
321 || cb
<= PAGE_SIZE
* 2)
323 # ifdef VBOX_USE_INSERT_PAGE
324 paPages
= alloc_pages(fFlagsLnx
| __GFP_COMP
| __GFP_NOWARN
, rtR0MemObjLinuxOrder(cPages
));
326 paPages
= alloc_pages(fFlagsLnx
| __GFP_NOWARN
, rtR0MemObjLinuxOrder(cPages
));
331 for (iPage
= 0; iPage
< cPages
; iPage
++)
332 pMemLnx
->apPages
[iPage
] = &paPages
[iPage
];
334 else if (fContiguous
)
336 rtR0MemObjDelete(&pMemLnx
->Core
);
343 for (iPage
= 0; iPage
< cPages
; iPage
++)
345 pMemLnx
->apPages
[iPage
] = alloc_page(fFlagsLnx
| __GFP_NOWARN
);
346 if (RT_UNLIKELY(!pMemLnx
->apPages
[iPage
]))
349 __free_page(pMemLnx
->apPages
[iPage
]);
350 rtR0MemObjDelete(&pMemLnx
->Core
);
357 /** @todo figure out why we didn't allocate page-by-page on 2.4.21 and older... */
358 paPages
= alloc_pages(fFlagsLnx
, rtR0MemObjLinuxOrder(cPages
));
361 rtR0MemObjDelete(&pMemLnx
->Core
);
364 for (iPage
= 0; iPage
< cPages
; iPage
++)
366 pMemLnx
->apPages
[iPage
] = &paPages
[iPage
];
367 MY_SET_PAGES_EXEC(pMemLnx
->apPages
[iPage
], 1);
368 if (PageHighMem(pMemLnx
->apPages
[iPage
]))
373 #endif /* < 2.4.22 */
374 pMemLnx
->fContiguous
= fContiguous
;
376 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
380 * Linux >= 4.5 with CONFIG_DEBUG_VM panics when setting PG_reserved on compound
381 * pages. According to Michal Hocko this shouldn't be necessary anyway because
382 * as pages which are not on the LRU list are never evictable.
384 for (iPage
= 0; iPage
< cPages
; iPage
++)
385 SetPageReserved(pMemLnx
->apPages
[iPage
]);
389 * Note that the physical address of memory allocated with alloc_pages(flags, order)
390 * is always 2^(PAGE_SHIFT+order)-aligned.
393 && uAlignment
> PAGE_SIZE
)
396 * Check for alignment constraints. The physical address of memory allocated with
397 * alloc_pages(flags, order) is always 2^(PAGE_SHIFT+order)-aligned.
399 if (RT_UNLIKELY(page_to_phys(pMemLnx
->apPages
[0]) & (uAlignment
- 1)))
402 * This should never happen!
404 printk("rtR0MemObjLinuxAllocPages(cb=0x%lx, uAlignment=0x%lx): alloc_pages(..., %d) returned physical memory at 0x%lx!\n",
405 (unsigned long)cb
, (unsigned long)uAlignment
, rtR0MemObjLinuxOrder(cPages
), (unsigned long)page_to_phys(pMemLnx
->apPages
[0]));
406 rtR0MemObjLinuxFreePages(pMemLnx
);
417 * Frees the physical pages allocated by the rtR0MemObjLinuxAllocPages() call.
419 * This method does NOT free the object.
421 * @param pMemLnx The object which physical pages should be freed.
423 static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx
)
425 size_t iPage
= pMemLnx
->cPages
;
429 * Restore the page flags.
433 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
435 * See SetPageReserved() in rtR0MemObjLinuxAllocPages()
437 ClearPageReserved(pMemLnx
->apPages
[iPage
]);
439 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
441 MY_SET_PAGES_NOEXEC(pMemLnx
->apPages
[iPage
], 1);
448 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
449 if (!pMemLnx
->fContiguous
)
451 iPage
= pMemLnx
->cPages
;
453 __free_page(pMemLnx
->apPages
[iPage
]);
457 __free_pages(pMemLnx
->apPages
[0], rtR0MemObjLinuxOrder(pMemLnx
->cPages
));
465 * Maps the allocation into ring-0.
467 * This will update the RTR0MEMOBJLNX::Core.pv and RTR0MEMOBJ::fMappedToRing0 members.
469 * Contiguous mappings that isn't in 'high' memory will already be mapped into kernel
470 * space, so we'll use that mapping if possible. If execute access is required, we'll
471 * play safe and do our own mapping.
473 * @returns IPRT status code.
474 * @param pMemLnx The linux memory object to map.
475 * @param fExecutable Whether execute access is required.
477 static int rtR0MemObjLinuxVMap(PRTR0MEMOBJLNX pMemLnx
, bool fExecutable
)
479 int rc
= VINF_SUCCESS
;
482 * Choose mapping strategy.
484 bool fMustMap
= fExecutable
485 || !pMemLnx
->fContiguous
;
488 size_t iPage
= pMemLnx
->cPages
;
490 if (PageHighMem(pMemLnx
->apPages
[iPage
]))
497 Assert(!pMemLnx
->Core
.pv
);
498 Assert(!pMemLnx
->fMappedToRing0
);
503 * Use vmap - 2.4.22 and later.
505 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
507 pgprot_val(fPg
) = _PAGE_PRESENT
| _PAGE_RW
;
510 pgprot_val(fPg
) |= _PAGE_NX
;
514 pMemLnx
->Core
.pv
= vmap(&pMemLnx
->apPages
[0], pMemLnx
->cPages
, VM_MAP
, fPg
);
516 pMemLnx
->Core
.pv
= vmap(&pMemLnx
->apPages
[0], pMemLnx
->cPages
, VM_ALLOC
, fPg
);
518 if (pMemLnx
->Core
.pv
)
519 pMemLnx
->fMappedToRing0
= true;
521 rc
= VERR_MAP_FAILED
;
523 rc
= VERR_NOT_SUPPORTED
;
529 * Use the kernel RAM mapping.
531 pMemLnx
->Core
.pv
= phys_to_virt(page_to_phys(pMemLnx
->apPages
[0]));
532 Assert(pMemLnx
->Core
.pv
);
540 * Undoes what rtR0MemObjLinuxVMap() did.
542 * @param pMemLnx The linux memory object.
544 static void rtR0MemObjLinuxVUnmap(PRTR0MEMOBJLNX pMemLnx
)
546 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
547 if (pMemLnx
->fMappedToRing0
)
549 Assert(pMemLnx
->Core
.pv
);
550 vunmap(pMemLnx
->Core
.pv
);
551 pMemLnx
->fMappedToRing0
= false;
554 Assert(!pMemLnx
->fMappedToRing0
);
556 pMemLnx
->Core
.pv
= NULL
;
560 DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem
)
562 IPRT_LINUX_SAVE_EFL_AC();
563 PRTR0MEMOBJLNX pMemLnx
= (PRTR0MEMOBJLNX
)pMem
;
566 * Release any memory that we've allocated or locked.
568 switch (pMemLnx
->Core
.enmType
)
570 case RTR0MEMOBJTYPE_LOW
:
571 case RTR0MEMOBJTYPE_PAGE
:
572 case RTR0MEMOBJTYPE_CONT
:
573 case RTR0MEMOBJTYPE_PHYS
:
574 case RTR0MEMOBJTYPE_PHYS_NC
:
575 rtR0MemObjLinuxVUnmap(pMemLnx
);
576 rtR0MemObjLinuxFreePages(pMemLnx
);
579 case RTR0MEMOBJTYPE_LOCK
:
580 if (pMemLnx
->Core
.u
.Lock
.R0Process
!= NIL_RTR0PROCESS
)
582 struct task_struct
*pTask
= rtR0ProcessToLinuxTask(pMemLnx
->Core
.u
.Lock
.R0Process
);
585 if (pTask
&& pTask
->mm
)
586 down_read(&pTask
->mm
->mmap_sem
);
588 iPage
= pMemLnx
->cPages
;
591 if (!PageReserved(pMemLnx
->apPages
[iPage
]))
592 SetPageDirty(pMemLnx
->apPages
[iPage
]);
593 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
594 put_page(pMemLnx
->apPages
[iPage
]);
596 page_cache_release(pMemLnx
->apPages
[iPage
]);
600 if (pTask
&& pTask
->mm
)
601 up_read(&pTask
->mm
->mmap_sem
);
603 /* else: kernel memory - nothing to do here. */
606 case RTR0MEMOBJTYPE_RES_VIRT
:
607 Assert(pMemLnx
->Core
.pv
);
608 if (pMemLnx
->Core
.u
.ResVirt
.R0Process
!= NIL_RTR0PROCESS
)
610 struct task_struct
*pTask
= rtR0ProcessToLinuxTask(pMemLnx
->Core
.u
.Lock
.R0Process
);
612 if (pTask
&& pTask
->mm
)
613 rtR0MemObjLinuxDoMunmap(pMemLnx
->Core
.pv
, pMemLnx
->Core
.cb
, pTask
);
617 vunmap(pMemLnx
->Core
.pv
);
619 Assert(pMemLnx
->cPages
== 1 && pMemLnx
->apPages
[0] != NULL
);
620 __free_page(pMemLnx
->apPages
[0]);
621 pMemLnx
->apPages
[0] = NULL
;
624 pMemLnx
->Core
.pv
= NULL
;
627 case RTR0MEMOBJTYPE_MAPPING
:
628 Assert(pMemLnx
->cPages
== 0); Assert(pMemLnx
->Core
.pv
);
629 if (pMemLnx
->Core
.u
.ResVirt
.R0Process
!= NIL_RTR0PROCESS
)
631 struct task_struct
*pTask
= rtR0ProcessToLinuxTask(pMemLnx
->Core
.u
.Lock
.R0Process
);
633 if (pTask
&& pTask
->mm
)
634 rtR0MemObjLinuxDoMunmap(pMemLnx
->Core
.pv
, pMemLnx
->Core
.cb
, pTask
);
637 vunmap(pMemLnx
->Core
.pv
);
638 pMemLnx
->Core
.pv
= NULL
;
642 AssertMsgFailed(("enmType=%d\n", pMemLnx
->Core
.enmType
));
643 return VERR_INTERNAL_ERROR
;
645 IPRT_LINUX_RESTORE_EFL_ONLY_AC();
650 DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem
, size_t cb
, bool fExecutable
)
652 IPRT_LINUX_SAVE_EFL_AC();
653 PRTR0MEMOBJLNX pMemLnx
;
656 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
657 rc
= rtR0MemObjLinuxAllocPages(&pMemLnx
, RTR0MEMOBJTYPE_PAGE
, cb
, PAGE_SIZE
, GFP_HIGHUSER
,
658 false /* non-contiguous */, VERR_NO_MEMORY
);
660 rc
= rtR0MemObjLinuxAllocPages(&pMemLnx
, RTR0MEMOBJTYPE_PAGE
, cb
, PAGE_SIZE
, GFP_USER
,
661 false /* non-contiguous */, VERR_NO_MEMORY
);
665 rc
= rtR0MemObjLinuxVMap(pMemLnx
, fExecutable
);
668 *ppMem
= &pMemLnx
->Core
;
669 IPRT_LINUX_RESTORE_EFL_AC();
673 rtR0MemObjLinuxFreePages(pMemLnx
);
674 rtR0MemObjDelete(&pMemLnx
->Core
);
677 IPRT_LINUX_RESTORE_EFL_AC();
682 DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem
, size_t cb
, bool fExecutable
)
684 IPRT_LINUX_SAVE_EFL_AC();
685 PRTR0MEMOBJLNX pMemLnx
;
688 /* Try to avoid GFP_DMA. GFM_DMA32 was introduced with Linux 2.6.15. */
689 #if (defined(RT_ARCH_AMD64) || defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
690 /* ZONE_DMA32: 0-4GB */
691 rc
= rtR0MemObjLinuxAllocPages(&pMemLnx
, RTR0MEMOBJTYPE_LOW
, cb
, PAGE_SIZE
, GFP_DMA32
,
692 false /* non-contiguous */, VERR_NO_LOW_MEMORY
);
696 /* ZONE_DMA: 0-16MB */
697 rc
= rtR0MemObjLinuxAllocPages(&pMemLnx
, RTR0MEMOBJTYPE_LOW
, cb
, PAGE_SIZE
, GFP_DMA
,
698 false /* non-contiguous */, VERR_NO_LOW_MEMORY
);
700 # ifdef CONFIG_X86_PAE
702 /* ZONE_NORMAL: 0-896MB */
703 rc
= rtR0MemObjLinuxAllocPages(&pMemLnx
, RTR0MEMOBJTYPE_LOW
, cb
, PAGE_SIZE
, GFP_USER
,
704 false /* non-contiguous */, VERR_NO_LOW_MEMORY
);
708 rc
= rtR0MemObjLinuxVMap(pMemLnx
, fExecutable
);
711 *ppMem
= &pMemLnx
->Core
;
712 IPRT_LINUX_RESTORE_EFL_AC();
716 rtR0MemObjLinuxFreePages(pMemLnx
);
717 rtR0MemObjDelete(&pMemLnx
->Core
);
720 IPRT_LINUX_RESTORE_EFL_AC();
725 DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem
, size_t cb
, bool fExecutable
)
727 IPRT_LINUX_SAVE_EFL_AC();
728 PRTR0MEMOBJLNX pMemLnx
;
731 #if (defined(RT_ARCH_AMD64) || defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
732 /* ZONE_DMA32: 0-4GB */
733 rc
= rtR0MemObjLinuxAllocPages(&pMemLnx
, RTR0MEMOBJTYPE_CONT
, cb
, PAGE_SIZE
, GFP_DMA32
,
734 true /* contiguous */, VERR_NO_CONT_MEMORY
);
738 /* ZONE_DMA: 0-16MB */
739 rc
= rtR0MemObjLinuxAllocPages(&pMemLnx
, RTR0MEMOBJTYPE_CONT
, cb
, PAGE_SIZE
, GFP_DMA
,
740 true /* contiguous */, VERR_NO_CONT_MEMORY
);
742 /* ZONE_NORMAL (32-bit hosts): 0-896MB */
743 rc
= rtR0MemObjLinuxAllocPages(&pMemLnx
, RTR0MEMOBJTYPE_CONT
, cb
, PAGE_SIZE
, GFP_USER
,
744 true /* contiguous */, VERR_NO_CONT_MEMORY
);
748 rc
= rtR0MemObjLinuxVMap(pMemLnx
, fExecutable
);
751 #if defined(RT_STRICT) && (defined(RT_ARCH_AMD64) || defined(CONFIG_HIGHMEM64G))
752 size_t iPage
= pMemLnx
->cPages
;
754 Assert(page_to_phys(pMemLnx
->apPages
[iPage
]) < _4G
);
756 pMemLnx
->Core
.u
.Cont
.Phys
= page_to_phys(pMemLnx
->apPages
[0]);
757 *ppMem
= &pMemLnx
->Core
;
758 IPRT_LINUX_RESTORE_EFL_AC();
762 rtR0MemObjLinuxFreePages(pMemLnx
);
763 rtR0MemObjDelete(&pMemLnx
->Core
);
766 IPRT_LINUX_RESTORE_EFL_AC();
772 * Worker for rtR0MemObjLinuxAllocPhysSub that tries one allocation strategy.
774 * @returns IPRT status code.
775 * @param ppMemLnx Where to
776 * @param enmType The object type.
777 * @param cb The size of the allocation.
778 * @param uAlignment The alignment of the physical memory.
779 * Only valid for fContiguous == true, ignored otherwise.
780 * @param PhysHighest See rtR0MemObjNativeAllocPhys.
781 * @param fGfp The Linux GFP flags to use for the allocation.
783 static int rtR0MemObjLinuxAllocPhysSub2(PPRTR0MEMOBJINTERNAL ppMem
, RTR0MEMOBJTYPE enmType
,
784 size_t cb
, size_t uAlignment
, RTHCPHYS PhysHighest
, unsigned fGfp
)
786 PRTR0MEMOBJLNX pMemLnx
;
789 rc
= rtR0MemObjLinuxAllocPages(&pMemLnx
, enmType
, cb
, uAlignment
, fGfp
,
790 enmType
== RTR0MEMOBJTYPE_PHYS
/* contiguous / non-contiguous */,
791 VERR_NO_PHYS_MEMORY
);
796 * Check the addresses if necessary. (Can be optimized a bit for PHYS.)
798 if (PhysHighest
!= NIL_RTHCPHYS
)
800 size_t iPage
= pMemLnx
->cPages
;
802 if (page_to_phys(pMemLnx
->apPages
[iPage
]) > PhysHighest
)
804 rtR0MemObjLinuxFreePages(pMemLnx
);
805 rtR0MemObjDelete(&pMemLnx
->Core
);
806 return VERR_NO_MEMORY
;
811 * Complete the object.
813 if (enmType
== RTR0MEMOBJTYPE_PHYS
)
815 pMemLnx
->Core
.u
.Phys
.PhysBase
= page_to_phys(pMemLnx
->apPages
[0]);
816 pMemLnx
->Core
.u
.Phys
.fAllocated
= true;
818 *ppMem
= &pMemLnx
->Core
;
824 * Worker for rtR0MemObjNativeAllocPhys and rtR0MemObjNativeAllocPhysNC.
826 * @returns IPRT status code.
827 * @param ppMem Where to store the memory object pointer on success.
828 * @param enmType The object type.
829 * @param cb The size of the allocation.
830 * @param uAlignment The alignment of the physical memory.
831 * Only valid for enmType == RTR0MEMOBJTYPE_PHYS, ignored otherwise.
832 * @param PhysHighest See rtR0MemObjNativeAllocPhys.
834 static int rtR0MemObjLinuxAllocPhysSub(PPRTR0MEMOBJINTERNAL ppMem
, RTR0MEMOBJTYPE enmType
,
835 size_t cb
, size_t uAlignment
, RTHCPHYS PhysHighest
)
838 IPRT_LINUX_SAVE_EFL_AC();
841 * There are two clear cases and that's the <=16MB and anything-goes ones.
842 * When the physical address limit is somewhere in-between those two we'll
843 * just have to try, starting with HIGHUSER and working our way thru the
844 * different types, hoping we'll get lucky.
846 * We should probably move this physical address restriction logic up to
847 * the page alloc function as it would be more efficient there. But since
848 * we don't expect this to be a performance issue just yet it can wait.
850 if (PhysHighest
== NIL_RTHCPHYS
)
851 /* ZONE_HIGHMEM: the whole physical memory */
852 rc
= rtR0MemObjLinuxAllocPhysSub2(ppMem
, enmType
, cb
, uAlignment
, PhysHighest
, GFP_HIGHUSER
);
853 else if (PhysHighest
<= _1M
* 16)
854 /* ZONE_DMA: 0-16MB */
855 rc
= rtR0MemObjLinuxAllocPhysSub2(ppMem
, enmType
, cb
, uAlignment
, PhysHighest
, GFP_DMA
);
860 /* ZONE_HIGHMEM: the whole physical memory */
861 rc
= rtR0MemObjLinuxAllocPhysSub2(ppMem
, enmType
, cb
, uAlignment
, PhysHighest
, GFP_HIGHUSER
);
863 /* ZONE_NORMAL: 0-896MB */
864 rc
= rtR0MemObjLinuxAllocPhysSub2(ppMem
, enmType
, cb
, uAlignment
, PhysHighest
, GFP_USER
);
867 /* ZONE_DMA32: 0-4GB */
868 rc
= rtR0MemObjLinuxAllocPhysSub2(ppMem
, enmType
, cb
, uAlignment
, PhysHighest
, GFP_DMA32
);
871 /* ZONE_DMA: 0-16MB */
872 rc
= rtR0MemObjLinuxAllocPhysSub2(ppMem
, enmType
, cb
, uAlignment
, PhysHighest
, GFP_DMA
);
874 IPRT_LINUX_RESTORE_EFL_AC();
880 * Translates a kernel virtual address to a linux page structure by walking the
883 * @note We do assume that the page tables will not change as we are walking
884 * them. This assumption is rather forced by the fact that I could not
885 * immediately see any way of preventing this from happening. So, we
886 * take some extra care when accessing them.
888 * Because of this, we don't want to use this function on memory where
889 * attribute changes to nearby pages is likely to cause large pages to
890 * be used or split up. So, don't use this for the linear mapping of
893 * @returns Pointer to the page structur or NULL if it could not be found.
894 * @param pv The kernel virtual address.
896 static struct page
*rtR0MemObjLinuxVirtToPage(void *pv
)
898 unsigned long ulAddr
= (unsigned long)pv
;
905 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
912 /* Should this happen in a situation this code will be called in? And if
913 * so, can it change under our feet? See also
914 * "Documentation/vm/active_mm.txt" in the kernel sources. */
915 if (RT_UNLIKELY(!current
->active_mm
))
917 u
.Global
= *pgd_offset(current
->active_mm
, ulAddr
);
918 if (RT_UNLIKELY(pgd_none(u
.Global
)))
921 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
922 u
.Upper
= *pud_offset(&u
.Global
, ulAddr
);
923 if (RT_UNLIKELY(pud_none(u
.Upper
)))
925 # if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)
926 if (pud_large(u
.Upper
))
928 pPage
= pud_page(u
.Upper
);
929 AssertReturn(pPage
, NULL
);
930 pfn
= page_to_pfn(pPage
); /* doing the safe way... */
931 pfn
+= (ulAddr
>> PAGE_SHIFT
) & ((UINT32_C(1) << (PUD_SHIFT
- PAGE_SHIFT
)) - 1);
932 return pfn_to_page(pfn
);
936 u
.Middle
= *pmd_offset(&u
.Upper
, ulAddr
);
938 u
.Middle
= *pmd_offset(&u
.Global
, ulAddr
);
939 #endif /* < 2.6.11 */
940 if (RT_UNLIKELY(pmd_none(u
.Middle
)))
942 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0)
943 if (pmd_large(u
.Middle
))
945 pPage
= pmd_page(u
.Middle
);
946 AssertReturn(pPage
, NULL
);
947 pfn
= page_to_pfn(pPage
); /* doing the safe way... */
948 pfn
+= (ulAddr
>> PAGE_SHIFT
) & ((UINT32_C(1) << (PMD_SHIFT
- PAGE_SHIFT
)) - 1);
949 return pfn_to_page(pfn
);
953 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 5) || defined(pte_offset_map) /* As usual, RHEL 3 had pte_offset_map earlier. */
954 pEntry
= pte_offset_map(&u
.Middle
, ulAddr
);
956 pEntry
= pte_offset(&u
.Middle
, ulAddr
);
958 if (RT_UNLIKELY(!pEntry
))
961 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 5) || defined(pte_offset_map)
965 if (RT_UNLIKELY(!pte_present(u
.Entry
)))
967 return pte_page(u
.Entry
);
971 DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem
, size_t cb
, RTHCPHYS PhysHighest
, size_t uAlignment
)
973 return rtR0MemObjLinuxAllocPhysSub(ppMem
, RTR0MEMOBJTYPE_PHYS
, cb
, uAlignment
, PhysHighest
);
977 DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem
, size_t cb
, RTHCPHYS PhysHighest
)
979 return rtR0MemObjLinuxAllocPhysSub(ppMem
, RTR0MEMOBJTYPE_PHYS_NC
, cb
, PAGE_SIZE
, PhysHighest
);
983 DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem
, RTHCPHYS Phys
, size_t cb
, uint32_t uCachePolicy
)
985 IPRT_LINUX_SAVE_EFL_AC();
988 * All we need to do here is to validate that we can use
989 * ioremap on the specified address (32/64-bit dma_addr_t).
991 PRTR0MEMOBJLNX pMemLnx
;
992 dma_addr_t PhysAddr
= Phys
;
993 AssertMsgReturn(PhysAddr
== Phys
, ("%#llx\n", (unsigned long long)Phys
), VERR_ADDRESS_TOO_BIG
);
995 pMemLnx
= (PRTR0MEMOBJLNX
)rtR0MemObjNew(sizeof(*pMemLnx
), RTR0MEMOBJTYPE_PHYS
, NULL
, cb
);
998 IPRT_LINUX_RESTORE_EFL_AC();
999 return VERR_NO_MEMORY
;
1002 pMemLnx
->Core
.u
.Phys
.PhysBase
= PhysAddr
;
1003 pMemLnx
->Core
.u
.Phys
.fAllocated
= false;
1004 pMemLnx
->Core
.u
.Phys
.uCachePolicy
= uCachePolicy
;
1005 Assert(!pMemLnx
->cPages
);
1006 *ppMem
= &pMemLnx
->Core
;
1007 IPRT_LINUX_RESTORE_EFL_AC();
1008 return VINF_SUCCESS
;
1012 DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem
, RTR3PTR R3Ptr
, size_t cb
, uint32_t fAccess
, RTR0PROCESS R0Process
)
1014 IPRT_LINUX_SAVE_EFL_AC();
1015 const int cPages
= cb
>> PAGE_SHIFT
;
1016 struct task_struct
*pTask
= rtR0ProcessToLinuxTask(R0Process
);
1017 struct vm_area_struct
**papVMAs
;
1018 PRTR0MEMOBJLNX pMemLnx
;
1019 int rc
= VERR_NO_MEMORY
;
1020 int const fWrite
= fAccess
& RTMEM_PROT_WRITE
? 1 : 0;
1023 * Check for valid task and size overflows.
1026 return VERR_NOT_SUPPORTED
;
1027 if (((size_t)cPages
<< PAGE_SHIFT
) != cb
)
1028 return VERR_OUT_OF_RANGE
;
1031 * Allocate the memory object and a temporary buffer for the VMAs.
1033 pMemLnx
= (PRTR0MEMOBJLNX
)rtR0MemObjNew(RT_OFFSETOF(RTR0MEMOBJLNX
, apPages
[cPages
]), RTR0MEMOBJTYPE_LOCK
, (void *)R3Ptr
, cb
);
1036 IPRT_LINUX_RESTORE_EFL_AC();
1037 return VERR_NO_MEMORY
;
1040 papVMAs
= (struct vm_area_struct
**)RTMemAlloc(sizeof(*papVMAs
) * cPages
);
1043 down_read(&pTask
->mm
->mmap_sem
);
1048 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
1049 if (R0Process
== RTR0ProcHandleSelf())
1050 rc
= get_user_pages(R3Ptr
, /* Where from. */
1051 cPages
, /* How many pages. */
1052 # if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
1053 fWrite
? FOLL_WRITE
| /* Write to memory. */
1054 FOLL_FORCE
/* force write access. */
1055 : 0, /* Write to memory. */
1057 fWrite
, /* Write to memory. */
1058 fWrite
, /* force write access. */
1060 &pMemLnx
->apPages
[0], /* Page array. */
1061 papVMAs
); /* vmas */
1063 * Actually this should not happen at the moment as call this function
1064 * only for our own process.
1067 rc
= get_user_pages_remote(
1068 pTask
, /* Task for fault accounting. */
1069 pTask
->mm
, /* Whose pages. */
1070 R3Ptr
, /* Where from. */
1071 cPages
, /* How many pages. */
1072 # if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
1073 fWrite
? FOLL_WRITE
| /* Write to memory. */
1074 FOLL_FORCE
/* force write access. */
1075 : 0, /* Write to memory. */
1077 fWrite
, /* Write to memory. */
1078 fWrite
, /* force write access. */
1080 &pMemLnx
->apPages
[0], /* Page array. */
1082 # if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
1086 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) */
1087 rc
= get_user_pages(pTask
, /* Task for fault accounting. */
1088 pTask
->mm
, /* Whose pages. */
1089 R3Ptr
, /* Where from. */
1090 cPages
, /* How many pages. */
1091 # if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
1092 fWrite
? FOLL_WRITE
| /* Write to memory. */
1093 FOLL_FORCE
/* force write access. */
1094 : 0, /* Write to memory. */
1096 fWrite
, /* Write to memory. */
1097 fWrite
, /* force write access. */
1099 &pMemLnx
->apPages
[0], /* Page array. */
1100 papVMAs
); /* vmas */
1101 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) */
1105 * Flush dcache (required?), protect against fork and _really_ pin the page
1106 * table entries. get_user_pages() will protect against swapping out the
1107 * pages but it will NOT protect against removing page table entries. This
1108 * can be achieved with
1109 * - using mlock / mmap(..., MAP_LOCKED, ...) from userland. This requires
1110 * an appropriate limit set up with setrlimit(..., RLIMIT_MEMLOCK, ...).
1111 * Usual Linux distributions support only a limited size of locked pages
1113 * - setting the PageReserved bit (as we do in rtR0MemObjLinuxAllocPages()
1115 * - setting the VM_LOCKED flag. This is the same as doing mlock() without
1118 /** @todo The Linux fork() protection will require more work if this API
1119 * is to be used for anything but locking VM pages. */
1122 flush_dcache_page(pMemLnx
->apPages
[rc
]);
1123 papVMAs
[rc
]->vm_flags
|= (VM_DONTCOPY
| VM_LOCKED
);
1126 up_read(&pTask
->mm
->mmap_sem
);
1130 pMemLnx
->Core
.u
.Lock
.R0Process
= R0Process
;
1131 pMemLnx
->cPages
= cPages
;
1132 Assert(!pMemLnx
->fMappedToRing0
);
1133 *ppMem
= &pMemLnx
->Core
;
1135 IPRT_LINUX_RESTORE_EFL_AC();
1136 return VINF_SUCCESS
;
1140 * Failed - we need to unlock any pages that we succeeded to lock.
1144 if (!PageReserved(pMemLnx
->apPages
[rc
]))
1145 SetPageDirty(pMemLnx
->apPages
[rc
]);
1146 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
1147 put_page(pMemLnx
->apPages
[rc
]);
1149 page_cache_release(pMemLnx
->apPages
[rc
]);
1153 up_read(&pTask
->mm
->mmap_sem
);
1156 rc
= VERR_LOCK_FAILED
;
1159 rtR0MemObjDelete(&pMemLnx
->Core
);
1160 IPRT_LINUX_RESTORE_EFL_AC();
1165 DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem
, void *pv
, size_t cb
, uint32_t fAccess
)
1167 IPRT_LINUX_SAVE_EFL_AC();
1168 void *pvLast
= (uint8_t *)pv
+ cb
- 1;
1169 size_t const cPages
= cb
>> PAGE_SHIFT
;
1170 PRTR0MEMOBJLNX pMemLnx
;
1171 bool fLinearMapping
;
1177 if ( !RTR0MemKernelIsValidAddr(pv
)
1178 || !RTR0MemKernelIsValidAddr(pv
+ cb
))
1179 return VERR_INVALID_PARAMETER
;
1182 * The lower part of the kernel memory has a linear mapping between
1183 * physical and virtual addresses. So we take a short cut here. This is
1184 * assumed to be the cleanest way to handle those addresses (and the code
1185 * is well tested, though the test for determining it is not very nice).
1186 * If we ever decide it isn't we can still remove it.
1189 fLinearMapping
= (unsigned long)pvLast
< VMALLOC_START
;
1191 fLinearMapping
= (unsigned long)pv
>= (unsigned long)__va(0)
1192 && (unsigned long)pvLast
< (unsigned long)high_memory
;
1196 * Allocate the memory object.
1198 pMemLnx
= (PRTR0MEMOBJLNX
)rtR0MemObjNew(RT_OFFSETOF(RTR0MEMOBJLNX
, apPages
[cPages
]), RTR0MEMOBJTYPE_LOCK
, pv
, cb
);
1201 IPRT_LINUX_RESTORE_EFL_AC();
1202 return VERR_NO_MEMORY
;
1207 * We ASSUME all kernel pages are non-swappable and non-movable.
1210 pbPage
= (uint8_t *)pvLast
;
1212 if (!fLinearMapping
)
1216 struct page
*pPage
= rtR0MemObjLinuxVirtToPage(pbPage
);
1217 if (RT_UNLIKELY(!pPage
))
1219 rc
= VERR_LOCK_FAILED
;
1222 pMemLnx
->apPages
[iPage
] = pPage
;
1223 pbPage
-= PAGE_SIZE
;
1230 pMemLnx
->apPages
[iPage
] = virt_to_page(pbPage
);
1231 pbPage
-= PAGE_SIZE
;
1237 * Complete the memory object and return.
1239 pMemLnx
->Core
.u
.Lock
.R0Process
= NIL_RTR0PROCESS
;
1240 pMemLnx
->cPages
= cPages
;
1241 Assert(!pMemLnx
->fMappedToRing0
);
1242 *ppMem
= &pMemLnx
->Core
;
1244 IPRT_LINUX_RESTORE_EFL_AC();
1245 return VINF_SUCCESS
;
1248 rtR0MemObjDelete(&pMemLnx
->Core
);
1249 IPRT_LINUX_RESTORE_EFL_AC();
1254 DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem
, void *pvFixed
, size_t cb
, size_t uAlignment
)
1256 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
1257 IPRT_LINUX_SAVE_EFL_AC();
1258 const size_t cPages
= cb
>> PAGE_SHIFT
;
1259 struct page
*pDummyPage
;
1260 struct page
**papPages
;
1262 /* check for unsupported stuff. */
1263 AssertMsgReturn(pvFixed
== (void *)-1, ("%p\n", pvFixed
), VERR_NOT_SUPPORTED
);
1264 if (uAlignment
> PAGE_SIZE
)
1265 return VERR_NOT_SUPPORTED
;
1268 * Allocate a dummy page and create a page pointer array for vmap such that
1269 * the dummy page is mapped all over the reserved area.
1271 pDummyPage
= alloc_page(GFP_HIGHUSER
| __GFP_NOWARN
);
1274 papPages
= RTMemAlloc(sizeof(*papPages
) * cPages
);
1278 size_t iPage
= cPages
;
1280 papPages
[iPage
] = pDummyPage
;
1282 pv
= vmap(papPages
, cPages
, VM_MAP
, PAGE_KERNEL_RO
);
1284 pv
= vmap(papPages
, cPages
, VM_ALLOC
, PAGE_KERNEL_RO
);
1286 RTMemFree(papPages
);
1289 PRTR0MEMOBJLNX pMemLnx
= (PRTR0MEMOBJLNX
)rtR0MemObjNew(sizeof(*pMemLnx
), RTR0MEMOBJTYPE_RES_VIRT
, pv
, cb
);
1292 pMemLnx
->Core
.u
.ResVirt
.R0Process
= NIL_RTR0PROCESS
;
1293 pMemLnx
->cPages
= 1;
1294 pMemLnx
->apPages
[0] = pDummyPage
;
1295 *ppMem
= &pMemLnx
->Core
;
1296 IPRT_LINUX_RESTORE_EFL_AC();
1297 return VINF_SUCCESS
;
1302 __free_page(pDummyPage
);
1304 IPRT_LINUX_RESTORE_EFL_AC();
1305 return VERR_NO_MEMORY
;
1307 #else /* < 2.4.22 */
1309 * Could probably use ioremap here, but the caller is in a better position than us
1310 * to select some safe physical memory.
1312 return VERR_NOT_SUPPORTED
;
1317 DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem
, RTR3PTR R3PtrFixed
, size_t cb
, size_t uAlignment
, RTR0PROCESS R0Process
)
1319 IPRT_LINUX_SAVE_EFL_AC();
1320 PRTR0MEMOBJLNX pMemLnx
;
1322 struct task_struct
*pTask
= rtR0ProcessToLinuxTask(R0Process
);
1324 return VERR_NOT_SUPPORTED
;
1327 * Check that the specified alignment is supported.
1329 if (uAlignment
> PAGE_SIZE
)
1330 return VERR_NOT_SUPPORTED
;
1333 * Let rtR0MemObjLinuxDoMmap do the difficult bits.
1335 pv
= rtR0MemObjLinuxDoMmap(R3PtrFixed
, cb
, uAlignment
, pTask
, RTMEM_PROT_NONE
);
1336 if (pv
== (void *)-1)
1338 IPRT_LINUX_RESTORE_EFL_AC();
1339 return VERR_NO_MEMORY
;
1342 pMemLnx
= (PRTR0MEMOBJLNX
)rtR0MemObjNew(sizeof(*pMemLnx
), RTR0MEMOBJTYPE_RES_VIRT
, pv
, cb
);
1345 rtR0MemObjLinuxDoMunmap(pv
, cb
, pTask
);
1346 IPRT_LINUX_RESTORE_EFL_AC();
1347 return VERR_NO_MEMORY
;
1350 pMemLnx
->Core
.u
.ResVirt
.R0Process
= R0Process
;
1351 *ppMem
= &pMemLnx
->Core
;
1352 IPRT_LINUX_RESTORE_EFL_AC();
1353 return VINF_SUCCESS
;
1357 DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem
, RTR0MEMOBJ pMemToMap
,
1358 void *pvFixed
, size_t uAlignment
,
1359 unsigned fProt
, size_t offSub
, size_t cbSub
)
1361 int rc
= VERR_NO_MEMORY
;
1362 PRTR0MEMOBJLNX pMemLnxToMap
= (PRTR0MEMOBJLNX
)pMemToMap
;
1363 PRTR0MEMOBJLNX pMemLnx
;
1364 IPRT_LINUX_SAVE_EFL_AC();
1366 /* Fail if requested to do something we can't. */
1367 AssertMsgReturn(!offSub
&& !cbSub
, ("%#x %#x\n", offSub
, cbSub
), VERR_NOT_SUPPORTED
);
1368 AssertMsgReturn(pvFixed
== (void *)-1, ("%p\n", pvFixed
), VERR_NOT_SUPPORTED
);
1369 if (uAlignment
> PAGE_SIZE
)
1370 return VERR_NOT_SUPPORTED
;
1373 * Create the IPRT memory object.
1375 pMemLnx
= (PRTR0MEMOBJLNX
)rtR0MemObjNew(sizeof(*pMemLnx
), RTR0MEMOBJTYPE_MAPPING
, NULL
, pMemLnxToMap
->Core
.cb
);
1378 if (pMemLnxToMap
->cPages
)
1380 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 22)
1382 * Use vmap - 2.4.22 and later.
1384 pgprot_t fPg
= rtR0MemObjLinuxConvertProt(fProt
, true /* kernel */);
1386 pMemLnx
->Core
.pv
= vmap(&pMemLnxToMap
->apPages
[0], pMemLnxToMap
->cPages
, VM_MAP
, fPg
);
1388 pMemLnx
->Core
.pv
= vmap(&pMemLnxToMap
->apPages
[0], pMemLnxToMap
->cPages
, VM_ALLOC
, fPg
);
1390 if (pMemLnx
->Core
.pv
)
1392 pMemLnx
->fMappedToRing0
= true;
1396 rc
= VERR_MAP_FAILED
;
1398 #else /* < 2.4.22 */
1400 * Only option here is to share mappings if possible and forget about fProt.
1402 if (rtR0MemObjIsRing3(pMemToMap
))
1403 rc
= VERR_NOT_SUPPORTED
;
1407 if (!pMemLnxToMap
->Core
.pv
)
1408 rc
= rtR0MemObjLinuxVMap(pMemLnxToMap
, !!(fProt
& RTMEM_PROT_EXEC
));
1411 Assert(pMemLnxToMap
->Core
.pv
);
1412 pMemLnx
->Core
.pv
= pMemLnxToMap
->Core
.pv
;
1420 * MMIO / physical memory.
1422 Assert(pMemLnxToMap
->Core
.enmType
== RTR0MEMOBJTYPE_PHYS
&& !pMemLnxToMap
->Core
.u
.Phys
.fAllocated
);
1423 pMemLnx
->Core
.pv
= pMemLnxToMap
->Core
.u
.Phys
.uCachePolicy
== RTMEM_CACHE_POLICY_MMIO
1424 ? ioremap_nocache(pMemLnxToMap
->Core
.u
.Phys
.PhysBase
, pMemLnxToMap
->Core
.cb
)
1425 : ioremap(pMemLnxToMap
->Core
.u
.Phys
.PhysBase
, pMemLnxToMap
->Core
.cb
);
1426 if (pMemLnx
->Core
.pv
)
1428 /** @todo fix protection. */
1434 pMemLnx
->Core
.u
.Mapping
.R0Process
= NIL_RTR0PROCESS
;
1435 *ppMem
= &pMemLnx
->Core
;
1436 IPRT_LINUX_RESTORE_EFL_AC();
1437 return VINF_SUCCESS
;
1439 rtR0MemObjDelete(&pMemLnx
->Core
);
1442 IPRT_LINUX_RESTORE_EFL_AC();
1447 #ifdef VBOX_USE_PAE_HACK
1449 * Replace the PFN of a PTE with the address of the actual page.
1451 * The caller maps a reserved dummy page at the address with the desired access
1454 * This hack is required for older Linux kernels which don't provide
1455 * remap_pfn_range().
1457 * @returns 0 on success, -ENOMEM on failure.
1458 * @param mm The memory context.
1459 * @param ulAddr The mapping address.
1460 * @param Phys The physical address of the page to map.
1462 static int rtR0MemObjLinuxFixPte(struct mm_struct
*mm
, unsigned long ulAddr
, RTHCPHYS Phys
)
1467 spin_lock(&mm
->page_table_lock
);
1469 pgd
= pgd_offset(mm
, ulAddr
);
1470 if (!pgd_none(*pgd
) && !pgd_bad(*pgd
))
1472 pmd_t
*pmd
= pmd_offset(pgd
, ulAddr
);
1473 if (!pmd_none(*pmd
))
1475 pte_t
*ptep
= pte_offset_map(pmd
, ulAddr
);
1479 pte
.pte_high
&= 0xfff00000;
1480 pte
.pte_high
|= ((Phys
>> 32) & 0x000fffff);
1481 pte
.pte_low
&= 0x00000fff;
1482 pte
.pte_low
|= (Phys
& 0xfffff000);
1490 spin_unlock(&mm
->page_table_lock
);
1493 #endif /* VBOX_USE_PAE_HACK */
1496 DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem
, RTR0MEMOBJ pMemToMap
, RTR3PTR R3PtrFixed
,
1497 size_t uAlignment
, unsigned fProt
, RTR0PROCESS R0Process
)
1499 struct task_struct
*pTask
= rtR0ProcessToLinuxTask(R0Process
);
1500 PRTR0MEMOBJLNX pMemLnxToMap
= (PRTR0MEMOBJLNX
)pMemToMap
;
1501 int rc
= VERR_NO_MEMORY
;
1502 PRTR0MEMOBJLNX pMemLnx
;
1503 #ifdef VBOX_USE_PAE_HACK
1504 struct page
*pDummyPage
;
1507 IPRT_LINUX_SAVE_EFL_AC();
1510 * Check for restrictions.
1513 return VERR_NOT_SUPPORTED
;
1514 if (uAlignment
> PAGE_SIZE
)
1515 return VERR_NOT_SUPPORTED
;
1517 #ifdef VBOX_USE_PAE_HACK
1519 * Allocate a dummy page for use when mapping the memory.
1521 pDummyPage
= alloc_page(GFP_USER
| __GFP_NOWARN
);
1524 IPRT_LINUX_RESTORE_EFL_AC();
1525 return VERR_NO_MEMORY
;
1527 SetPageReserved(pDummyPage
);
1528 DummyPhys
= page_to_phys(pDummyPage
);
1532 * Create the IPRT memory object.
1534 pMemLnx
= (PRTR0MEMOBJLNX
)rtR0MemObjNew(sizeof(*pMemLnx
), RTR0MEMOBJTYPE_MAPPING
, NULL
, pMemLnxToMap
->Core
.cb
);
1538 * Allocate user space mapping.
1541 pv
= rtR0MemObjLinuxDoMmap(R3PtrFixed
, pMemLnxToMap
->Core
.cb
, uAlignment
, pTask
, fProt
);
1542 if (pv
!= (void *)-1)
1545 * Map page by page into the mmap area.
1546 * This is generic, paranoid and not very efficient.
1548 pgprot_t fPg
= rtR0MemObjLinuxConvertProt(fProt
, false /* user */);
1549 unsigned long ulAddrCur
= (unsigned long)pv
;
1550 const size_t cPages
= pMemLnxToMap
->Core
.cb
>> PAGE_SHIFT
;
1553 down_write(&pTask
->mm
->mmap_sem
);
1556 if (pMemLnxToMap
->cPages
)
1558 for (iPage
= 0; iPage
< cPages
; iPage
++, ulAddrCur
+= PAGE_SIZE
)
1560 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)
1561 RTHCPHYS Phys
= page_to_phys(pMemLnxToMap
->apPages
[iPage
]);
1563 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1564 struct vm_area_struct
*vma
= find_vma(pTask
->mm
, ulAddrCur
); /* this is probably the same for all the pages... */
1565 AssertBreakStmt(vma
, rc
= VERR_INTERNAL_ERROR
);
1567 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0) && defined(RT_ARCH_X86)
1568 /* remap_page_range() limitation on x86 */
1569 AssertBreakStmt(Phys
< _4G
, rc
= VERR_NO_MEMORY
);
1572 #if defined(VBOX_USE_INSERT_PAGE) && LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
1573 rc
= vm_insert_page(vma
, ulAddrCur
, pMemLnxToMap
->apPages
[iPage
]);
1574 /* Thes flags help making 100% sure some bad stuff wont happen (swap, core, ++).
1575 * See remap_pfn_range() in mm/memory.c */
1576 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)
1577 vma
->vm_flags
|= VM_DONTEXPAND
| VM_DONTDUMP
;
1579 vma
->vm_flags
|= VM_RESERVED
;
1581 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
1582 rc
= remap_pfn_range(vma
, ulAddrCur
, page_to_pfn(pMemLnxToMap
->apPages
[iPage
]), PAGE_SIZE
, fPg
);
1583 #elif defined(VBOX_USE_PAE_HACK)
1584 rc
= remap_page_range(vma
, ulAddrCur
, DummyPhys
, PAGE_SIZE
, fPg
);
1586 rc
= rtR0MemObjLinuxFixPte(pTask
->mm
, ulAddrCur
, Phys
);
1587 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1588 rc
= remap_page_range(vma
, ulAddrCur
, Phys
, PAGE_SIZE
, fPg
);
1590 rc
= remap_page_range(ulAddrCur
, Phys
, PAGE_SIZE
, fPg
);
1594 rc
= VERR_NO_MEMORY
;
1602 if (pMemLnxToMap
->Core
.enmType
== RTR0MEMOBJTYPE_PHYS
)
1603 Phys
= pMemLnxToMap
->Core
.u
.Phys
.PhysBase
;
1604 else if (pMemLnxToMap
->Core
.enmType
== RTR0MEMOBJTYPE_CONT
)
1605 Phys
= pMemLnxToMap
->Core
.u
.Cont
.Phys
;
1608 AssertMsgFailed(("%d\n", pMemLnxToMap
->Core
.enmType
));
1609 Phys
= NIL_RTHCPHYS
;
1611 if (Phys
!= NIL_RTHCPHYS
)
1613 for (iPage
= 0; iPage
< cPages
; iPage
++, ulAddrCur
+= PAGE_SIZE
, Phys
+= PAGE_SIZE
)
1615 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1616 struct vm_area_struct
*vma
= find_vma(pTask
->mm
, ulAddrCur
); /* this is probably the same for all the pages... */
1617 AssertBreakStmt(vma
, rc
= VERR_INTERNAL_ERROR
);
1619 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0) && defined(RT_ARCH_X86)
1620 /* remap_page_range() limitation on x86 */
1621 AssertBreakStmt(Phys
< _4G
, rc
= VERR_NO_MEMORY
);
1624 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
1625 rc
= remap_pfn_range(vma
, ulAddrCur
, Phys
, PAGE_SIZE
, fPg
);
1626 #elif defined(VBOX_USE_PAE_HACK)
1627 rc
= remap_page_range(vma
, ulAddrCur
, DummyPhys
, PAGE_SIZE
, fPg
);
1629 rc
= rtR0MemObjLinuxFixPte(pTask
->mm
, ulAddrCur
, Phys
);
1630 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
1631 rc
= remap_page_range(vma
, ulAddrCur
, Phys
, PAGE_SIZE
, fPg
);
1633 rc
= remap_page_range(ulAddrCur
, Phys
, PAGE_SIZE
, fPg
);
1637 rc
= VERR_NO_MEMORY
;
1644 #ifdef CONFIG_NUMA_BALANCING
1645 # if LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0)
1646 # ifdef RHEL_RELEASE_CODE
1647 # if RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7, 0)
1648 # define VBOX_NUMA_HACK_OLD
1654 /** @todo Ugly hack! But right now we have no other means to
1655 * disable automatic NUMA page balancing. */
1657 # ifdef VBOX_NUMA_HACK_OLD
1658 pTask
->mm
->numa_next_reset
= jiffies
+ 0x7fffffffUL
;
1660 pTask
->mm
->numa_next_scan
= jiffies
+ 0x7fffffffUL
;
1662 # ifdef VBOX_NUMA_HACK_OLD
1663 pTask
->mm
->numa_next_reset
= jiffies
+ 0x7fffffffffffffffUL
;
1665 pTask
->mm
->numa_next_scan
= jiffies
+ 0x7fffffffffffffffUL
;
1668 #endif /* CONFIG_NUMA_BALANCING */
1670 up_write(&pTask
->mm
->mmap_sem
);
1674 #ifdef VBOX_USE_PAE_HACK
1675 __free_page(pDummyPage
);
1677 pMemLnx
->Core
.pv
= pv
;
1678 pMemLnx
->Core
.u
.Mapping
.R0Process
= R0Process
;
1679 *ppMem
= &pMemLnx
->Core
;
1680 IPRT_LINUX_RESTORE_EFL_AC();
1681 return VINF_SUCCESS
;
1687 rtR0MemObjLinuxDoMunmap(pv
, pMemLnxToMap
->Core
.cb
, pTask
);
1689 rtR0MemObjDelete(&pMemLnx
->Core
);
1691 #ifdef VBOX_USE_PAE_HACK
1692 __free_page(pDummyPage
);
1695 IPRT_LINUX_RESTORE_EFL_AC();
1700 DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem
, size_t offSub
, size_t cbSub
, uint32_t fProt
)
1706 return VERR_NOT_SUPPORTED
;
1710 DECLHIDDEN(RTHCPHYS
) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem
, size_t iPage
)
1712 PRTR0MEMOBJLNX pMemLnx
= (PRTR0MEMOBJLNX
)pMem
;
1714 if (pMemLnx
->cPages
)
1715 return page_to_phys(pMemLnx
->apPages
[iPage
]);
1717 switch (pMemLnx
->Core
.enmType
)
1719 case RTR0MEMOBJTYPE_CONT
:
1720 return pMemLnx
->Core
.u
.Cont
.Phys
+ (iPage
<< PAGE_SHIFT
);
1722 case RTR0MEMOBJTYPE_PHYS
:
1723 return pMemLnx
->Core
.u
.Phys
.PhysBase
+ (iPage
<< PAGE_SHIFT
);
1725 /* the parent knows */
1726 case RTR0MEMOBJTYPE_MAPPING
:
1727 return rtR0MemObjNativeGetPagePhysAddr(pMemLnx
->Core
.uRel
.Child
.pParent
, iPage
);
1730 case RTR0MEMOBJTYPE_LOW
:
1731 case RTR0MEMOBJTYPE_LOCK
:
1732 case RTR0MEMOBJTYPE_PHYS_NC
:
1733 case RTR0MEMOBJTYPE_PAGE
:
1735 AssertMsgFailed(("%d\n", pMemLnx
->Core
.enmType
));
1738 case RTR0MEMOBJTYPE_RES_VIRT
:
1739 return NIL_RTHCPHYS
;