1 // SPDX-License-Identifier: GPL-2.0-only
2 /**************************************************************************
3 * Copyright (c) 2007, Intel Corporation.
5 **************************************************************************/
12 * Code for the SGX MMU:
16 * clflush on one processor only:
17 * clflush should apparently flush the cache line on all processors in an
23 * The usage of the slots must be completely encapsulated within a spinlock, and
24 * no other functions that may be using the locks for other purposed may be
25 * called from within the locked region.
26 * Since the slots are per processor, this will guarantee that we are the only
31 * TODO: Inserting ptes from an interrupt handler:
32 * This may be desirable for some SGX functionality where the GPU can fault in
33 * needed pages. For that, we need to make an atomic insert_pages function, that
35 * If it fails, the caller need to insert the page using a workqueue function,
36 * but on average it should be fast.
39 static inline uint32_t psb_mmu_pt_index(uint32_t offset
)
41 return (offset
>> PSB_PTE_SHIFT
) & 0x3FF;
44 static inline uint32_t psb_mmu_pd_index(uint32_t offset
)
46 return offset
>> PSB_PDE_SHIFT
;
49 #if defined(CONFIG_X86)
50 static inline void psb_clflush(void *addr
)
52 __asm__
__volatile__("clflush (%0)\n" : : "r"(addr
) : "memory");
55 static inline void psb_mmu_clflush(struct psb_mmu_driver
*driver
, void *addr
)
57 if (!driver
->has_clflush
)
66 static inline void psb_mmu_clflush(struct psb_mmu_driver
*driver
, void *addr
)
72 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver
*driver
, int force
)
74 struct drm_device
*dev
= driver
->dev
;
75 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
77 if (atomic_read(&driver
->needs_tlbflush
) || force
) {
78 uint32_t val
= PSB_RSGX32(PSB_CR_BIF_CTRL
);
79 PSB_WSGX32(val
| _PSB_CB_CTRL_INVALDC
, PSB_CR_BIF_CTRL
);
81 /* Make sure data cache is turned off before enabling it */
83 PSB_WSGX32(val
& ~_PSB_CB_CTRL_INVALDC
, PSB_CR_BIF_CTRL
);
84 (void)PSB_RSGX32(PSB_CR_BIF_CTRL
);
85 if (driver
->msvdx_mmu_invaldc
)
86 atomic_set(driver
->msvdx_mmu_invaldc
, 1);
88 atomic_set(&driver
->needs_tlbflush
, 0);
92 static void psb_mmu_flush_pd(struct psb_mmu_driver
*driver
, int force
)
94 down_write(&driver
->sem
);
95 psb_mmu_flush_pd_locked(driver
, force
);
96 up_write(&driver
->sem
);
100 void psb_mmu_flush(struct psb_mmu_driver
*driver
)
102 struct drm_device
*dev
= driver
->dev
;
103 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
106 down_write(&driver
->sem
);
107 val
= PSB_RSGX32(PSB_CR_BIF_CTRL
);
108 if (atomic_read(&driver
->needs_tlbflush
))
109 PSB_WSGX32(val
| _PSB_CB_CTRL_INVALDC
, PSB_CR_BIF_CTRL
);
111 PSB_WSGX32(val
| _PSB_CB_CTRL_FLUSH
, PSB_CR_BIF_CTRL
);
113 /* Make sure data cache is turned off and MMU is flushed before
114 restoring bank interface control register */
116 PSB_WSGX32(val
& ~(_PSB_CB_CTRL_FLUSH
| _PSB_CB_CTRL_INVALDC
),
118 (void)PSB_RSGX32(PSB_CR_BIF_CTRL
);
120 atomic_set(&driver
->needs_tlbflush
, 0);
121 if (driver
->msvdx_mmu_invaldc
)
122 atomic_set(driver
->msvdx_mmu_invaldc
, 1);
123 up_write(&driver
->sem
);
126 void psb_mmu_set_pd_context(struct psb_mmu_pd
*pd
, int hw_context
)
128 struct drm_device
*dev
= pd
->driver
->dev
;
129 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
130 uint32_t offset
= (hw_context
== 0) ? PSB_CR_BIF_DIR_LIST_BASE0
:
131 PSB_CR_BIF_DIR_LIST_BASE1
+ hw_context
* 4;
133 down_write(&pd
->driver
->sem
);
134 PSB_WSGX32(page_to_pfn(pd
->p
) << PAGE_SHIFT
, offset
);
136 psb_mmu_flush_pd_locked(pd
->driver
, 1);
137 pd
->hw_context
= hw_context
;
138 up_write(&pd
->driver
->sem
);
142 static inline unsigned long psb_pd_addr_end(unsigned long addr
,
145 addr
= (addr
+ PSB_PDE_MASK
+ 1) & ~PSB_PDE_MASK
;
146 return (addr
< end
) ? addr
: end
;
149 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn
, int type
)
151 uint32_t mask
= PSB_PTE_VALID
;
153 if (type
& PSB_MMU_CACHED_MEMORY
)
154 mask
|= PSB_PTE_CACHED
;
155 if (type
& PSB_MMU_RO_MEMORY
)
157 if (type
& PSB_MMU_WO_MEMORY
)
160 return (pfn
<< PAGE_SHIFT
) | mask
;
163 struct psb_mmu_pd
*psb_mmu_alloc_pd(struct psb_mmu_driver
*driver
,
164 int trap_pagefaults
, int invalid_type
)
166 struct psb_mmu_pd
*pd
= kmalloc(sizeof(*pd
), GFP_KERNEL
);
173 pd
->p
= alloc_page(GFP_DMA32
);
176 pd
->dummy_pt
= alloc_page(GFP_DMA32
);
179 pd
->dummy_page
= alloc_page(GFP_DMA32
);
183 if (!trap_pagefaults
) {
184 pd
->invalid_pde
= psb_mmu_mask_pte(page_to_pfn(pd
->dummy_pt
),
186 pd
->invalid_pte
= psb_mmu_mask_pte(page_to_pfn(pd
->dummy_page
),
193 v
= kmap(pd
->dummy_pt
);
194 for (i
= 0; i
< (PAGE_SIZE
/ sizeof(uint32_t)); ++i
)
195 v
[i
] = pd
->invalid_pte
;
197 kunmap(pd
->dummy_pt
);
200 for (i
= 0; i
< (PAGE_SIZE
/ sizeof(uint32_t)); ++i
)
201 v
[i
] = pd
->invalid_pde
;
205 clear_page(kmap(pd
->dummy_page
));
206 kunmap(pd
->dummy_page
);
208 pd
->tables
= vmalloc_user(sizeof(struct psb_mmu_pt
*) * 1024);
213 pd
->pd_mask
= PSB_PTE_VALID
;
219 __free_page(pd
->dummy_page
);
221 __free_page(pd
->dummy_pt
);
229 static void psb_mmu_free_pt(struct psb_mmu_pt
*pt
)
235 void psb_mmu_free_pagedir(struct psb_mmu_pd
*pd
)
237 struct psb_mmu_driver
*driver
= pd
->driver
;
238 struct drm_device
*dev
= driver
->dev
;
239 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
240 struct psb_mmu_pt
*pt
;
243 down_write(&driver
->sem
);
244 if (pd
->hw_context
!= -1) {
245 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0
+ pd
->hw_context
* 4);
246 psb_mmu_flush_pd_locked(driver
, 1);
249 /* Should take the spinlock here, but we don't need to do that
250 since we have the semaphore in write mode. */
252 for (i
= 0; i
< 1024; ++i
) {
259 __free_page(pd
->dummy_page
);
260 __free_page(pd
->dummy_pt
);
263 up_write(&driver
->sem
);
266 static struct psb_mmu_pt
*psb_mmu_alloc_pt(struct psb_mmu_pd
*pd
)
268 struct psb_mmu_pt
*pt
= kmalloc(sizeof(*pt
), GFP_KERNEL
);
270 uint32_t clflush_add
= pd
->driver
->clflush_add
>> PAGE_SHIFT
;
271 uint32_t clflush_count
= PAGE_SIZE
/ clflush_add
;
272 spinlock_t
*lock
= &pd
->driver
->lock
;
280 pt
->p
= alloc_page(GFP_DMA32
);
288 v
= kmap_atomic(pt
->p
);
290 ptes
= (uint32_t *) v
;
291 for (i
= 0; i
< (PAGE_SIZE
/ sizeof(uint32_t)); ++i
)
292 *ptes
++ = pd
->invalid_pte
;
294 #if defined(CONFIG_X86)
295 if (pd
->driver
->has_clflush
&& pd
->hw_context
!= -1) {
297 for (i
= 0; i
< clflush_count
; ++i
) {
314 struct psb_mmu_pt
*psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd
*pd
,
317 uint32_t index
= psb_mmu_pd_index(addr
);
318 struct psb_mmu_pt
*pt
;
320 spinlock_t
*lock
= &pd
->driver
->lock
;
323 pt
= pd
->tables
[index
];
326 pt
= psb_mmu_alloc_pt(pd
);
331 if (pd
->tables
[index
]) {
335 pt
= pd
->tables
[index
];
339 v
= kmap_atomic(pd
->p
);
340 pd
->tables
[index
] = pt
;
341 v
[index
] = (page_to_pfn(pt
->p
) << 12) | pd
->pd_mask
;
343 kunmap_atomic((void *) v
);
345 if (pd
->hw_context
!= -1) {
346 psb_mmu_clflush(pd
->driver
, (void *)&v
[index
]);
347 atomic_set(&pd
->driver
->needs_tlbflush
, 1);
350 pt
->v
= kmap_atomic(pt
->p
);
354 static struct psb_mmu_pt
*psb_mmu_pt_map_lock(struct psb_mmu_pd
*pd
,
357 uint32_t index
= psb_mmu_pd_index(addr
);
358 struct psb_mmu_pt
*pt
;
359 spinlock_t
*lock
= &pd
->driver
->lock
;
362 pt
= pd
->tables
[index
];
367 pt
->v
= kmap_atomic(pt
->p
);
371 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt
*pt
)
373 struct psb_mmu_pd
*pd
= pt
->pd
;
376 kunmap_atomic(pt
->v
);
377 if (pt
->count
== 0) {
378 v
= kmap_atomic(pd
->p
);
379 v
[pt
->index
] = pd
->invalid_pde
;
380 pd
->tables
[pt
->index
] = NULL
;
382 if (pd
->hw_context
!= -1) {
383 psb_mmu_clflush(pd
->driver
, (void *)&v
[pt
->index
]);
384 atomic_set(&pd
->driver
->needs_tlbflush
, 1);
387 spin_unlock(&pd
->driver
->lock
);
391 spin_unlock(&pd
->driver
->lock
);
394 static inline void psb_mmu_set_pte(struct psb_mmu_pt
*pt
, unsigned long addr
,
397 pt
->v
[psb_mmu_pt_index(addr
)] = pte
;
400 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt
*pt
,
403 pt
->v
[psb_mmu_pt_index(addr
)] = pt
->pd
->invalid_pte
;
406 struct psb_mmu_pd
*psb_mmu_get_default_pd(struct psb_mmu_driver
*driver
)
408 struct psb_mmu_pd
*pd
;
410 down_read(&driver
->sem
);
411 pd
= driver
->default_pd
;
412 up_read(&driver
->sem
);
417 /* Returns the physical address of the PD shared by sgx/msvdx */
418 uint32_t psb_get_default_pd_addr(struct psb_mmu_driver
*driver
)
420 struct psb_mmu_pd
*pd
;
422 pd
= psb_mmu_get_default_pd(driver
);
423 return page_to_pfn(pd
->p
) << PAGE_SHIFT
;
426 void psb_mmu_driver_takedown(struct psb_mmu_driver
*driver
)
428 struct drm_device
*dev
= driver
->dev
;
429 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
431 PSB_WSGX32(driver
->bif_ctrl
, PSB_CR_BIF_CTRL
);
432 psb_mmu_free_pagedir(driver
->default_pd
);
436 struct psb_mmu_driver
*psb_mmu_driver_init(struct drm_device
*dev
,
439 atomic_t
*msvdx_mmu_invaldc
)
441 struct psb_mmu_driver
*driver
;
442 struct drm_psb_private
*dev_priv
= dev
->dev_private
;
444 driver
= kmalloc(sizeof(*driver
), GFP_KERNEL
);
450 driver
->default_pd
= psb_mmu_alloc_pd(driver
, trap_pagefaults
,
452 if (!driver
->default_pd
)
455 spin_lock_init(&driver
->lock
);
456 init_rwsem(&driver
->sem
);
457 down_write(&driver
->sem
);
458 atomic_set(&driver
->needs_tlbflush
, 1);
459 driver
->msvdx_mmu_invaldc
= msvdx_mmu_invaldc
;
461 driver
->bif_ctrl
= PSB_RSGX32(PSB_CR_BIF_CTRL
);
462 PSB_WSGX32(driver
->bif_ctrl
| _PSB_CB_CTRL_CLEAR_FAULT
,
464 PSB_WSGX32(driver
->bif_ctrl
& ~_PSB_CB_CTRL_CLEAR_FAULT
,
467 driver
->has_clflush
= 0;
469 #if defined(CONFIG_X86)
470 if (boot_cpu_has(X86_FEATURE_CLFLUSH
)) {
471 uint32_t tfms
, misc
, cap0
, cap4
, clflush_size
;
474 * clflush size is determined at kernel setup for x86_64 but not
475 * for i386. We have to do it here.
478 cpuid(0x00000001, &tfms
, &misc
, &cap0
, &cap4
);
479 clflush_size
= ((misc
>> 8) & 0xff) * 8;
480 driver
->has_clflush
= 1;
481 driver
->clflush_add
=
482 PAGE_SIZE
* clflush_size
/ sizeof(uint32_t);
483 driver
->clflush_mask
= driver
->clflush_add
- 1;
484 driver
->clflush_mask
= ~driver
->clflush_mask
;
488 up_write(&driver
->sem
);
496 #if defined(CONFIG_X86)
497 static void psb_mmu_flush_ptes(struct psb_mmu_pd
*pd
, unsigned long address
,
498 uint32_t num_pages
, uint32_t desired_tile_stride
,
499 uint32_t hw_tile_stride
)
501 struct psb_mmu_pt
*pt
;
508 unsigned long row_add
;
509 unsigned long clflush_add
= pd
->driver
->clflush_add
;
510 unsigned long clflush_mask
= pd
->driver
->clflush_mask
;
512 if (!pd
->driver
->has_clflush
)
516 rows
= num_pages
/ desired_tile_stride
;
518 desired_tile_stride
= num_pages
;
520 add
= desired_tile_stride
<< PAGE_SHIFT
;
521 row_add
= hw_tile_stride
<< PAGE_SHIFT
;
523 for (i
= 0; i
< rows
; ++i
) {
529 next
= psb_pd_addr_end(addr
, end
);
530 pt
= psb_mmu_pt_map_lock(pd
, addr
);
534 psb_clflush(&pt
->v
[psb_mmu_pt_index(addr
)]);
535 } while (addr
+= clflush_add
,
536 (addr
& clflush_mask
) < next
);
538 psb_mmu_pt_unmap_unlock(pt
);
539 } while (addr
= next
, next
!= end
);
545 static void psb_mmu_flush_ptes(struct psb_mmu_pd
*pd
, unsigned long address
,
546 uint32_t num_pages
, uint32_t desired_tile_stride
,
547 uint32_t hw_tile_stride
)
549 drm_ttm_cache_flush();
553 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd
*pd
,
554 unsigned long address
, uint32_t num_pages
)
556 struct psb_mmu_pt
*pt
;
560 unsigned long f_address
= address
;
562 down_read(&pd
->driver
->sem
);
565 end
= addr
+ (num_pages
<< PAGE_SHIFT
);
568 next
= psb_pd_addr_end(addr
, end
);
569 pt
= psb_mmu_pt_alloc_map_lock(pd
, addr
);
573 psb_mmu_invalidate_pte(pt
, addr
);
575 } while (addr
+= PAGE_SIZE
, addr
< next
);
576 psb_mmu_pt_unmap_unlock(pt
);
578 } while (addr
= next
, next
!= end
);
581 if (pd
->hw_context
!= -1)
582 psb_mmu_flush_ptes(pd
, f_address
, num_pages
, 1, 1);
584 up_read(&pd
->driver
->sem
);
586 if (pd
->hw_context
!= -1)
587 psb_mmu_flush(pd
->driver
);
592 void psb_mmu_remove_pages(struct psb_mmu_pd
*pd
, unsigned long address
,
593 uint32_t num_pages
, uint32_t desired_tile_stride
,
594 uint32_t hw_tile_stride
)
596 struct psb_mmu_pt
*pt
;
603 unsigned long row_add
;
604 unsigned long f_address
= address
;
607 rows
= num_pages
/ desired_tile_stride
;
609 desired_tile_stride
= num_pages
;
611 add
= desired_tile_stride
<< PAGE_SHIFT
;
612 row_add
= hw_tile_stride
<< PAGE_SHIFT
;
614 down_read(&pd
->driver
->sem
);
616 /* Make sure we only need to flush this processor's cache */
618 for (i
= 0; i
< rows
; ++i
) {
624 next
= psb_pd_addr_end(addr
, end
);
625 pt
= psb_mmu_pt_map_lock(pd
, addr
);
629 psb_mmu_invalidate_pte(pt
, addr
);
632 } while (addr
+= PAGE_SIZE
, addr
< next
);
633 psb_mmu_pt_unmap_unlock(pt
);
635 } while (addr
= next
, next
!= end
);
638 if (pd
->hw_context
!= -1)
639 psb_mmu_flush_ptes(pd
, f_address
, num_pages
,
640 desired_tile_stride
, hw_tile_stride
);
642 up_read(&pd
->driver
->sem
);
644 if (pd
->hw_context
!= -1)
645 psb_mmu_flush(pd
->driver
);
648 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd
*pd
, uint32_t start_pfn
,
649 unsigned long address
, uint32_t num_pages
,
652 struct psb_mmu_pt
*pt
;
657 unsigned long f_address
= address
;
660 down_read(&pd
->driver
->sem
);
663 end
= addr
+ (num_pages
<< PAGE_SHIFT
);
666 next
= psb_pd_addr_end(addr
, end
);
667 pt
= psb_mmu_pt_alloc_map_lock(pd
, addr
);
673 pte
= psb_mmu_mask_pte(start_pfn
++, type
);
674 psb_mmu_set_pte(pt
, addr
, pte
);
676 } while (addr
+= PAGE_SIZE
, addr
< next
);
677 psb_mmu_pt_unmap_unlock(pt
);
679 } while (addr
= next
, next
!= end
);
683 if (pd
->hw_context
!= -1)
684 psb_mmu_flush_ptes(pd
, f_address
, num_pages
, 1, 1);
686 up_read(&pd
->driver
->sem
);
688 if (pd
->hw_context
!= -1)
689 psb_mmu_flush(pd
->driver
);
694 int psb_mmu_insert_pages(struct psb_mmu_pd
*pd
, struct page
**pages
,
695 unsigned long address
, uint32_t num_pages
,
696 uint32_t desired_tile_stride
, uint32_t hw_tile_stride
,
699 struct psb_mmu_pt
*pt
;
707 unsigned long row_add
;
708 unsigned long f_address
= address
;
711 if (hw_tile_stride
) {
712 if (num_pages
% desired_tile_stride
!= 0)
714 rows
= num_pages
/ desired_tile_stride
;
716 desired_tile_stride
= num_pages
;
719 add
= desired_tile_stride
<< PAGE_SHIFT
;
720 row_add
= hw_tile_stride
<< PAGE_SHIFT
;
722 down_read(&pd
->driver
->sem
);
724 for (i
= 0; i
< rows
; ++i
) {
730 next
= psb_pd_addr_end(addr
, end
);
731 pt
= psb_mmu_pt_alloc_map_lock(pd
, addr
);
735 pte
= psb_mmu_mask_pte(page_to_pfn(*pages
++),
737 psb_mmu_set_pte(pt
, addr
, pte
);
739 } while (addr
+= PAGE_SIZE
, addr
< next
);
740 psb_mmu_pt_unmap_unlock(pt
);
742 } while (addr
= next
, next
!= end
);
749 if (pd
->hw_context
!= -1)
750 psb_mmu_flush_ptes(pd
, f_address
, num_pages
,
751 desired_tile_stride
, hw_tile_stride
);
753 up_read(&pd
->driver
->sem
);
755 if (pd
->hw_context
!= -1)
756 psb_mmu_flush(pd
->driver
);
761 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd
*pd
, uint32_t virtual,
765 struct psb_mmu_pt
*pt
;
767 spinlock_t
*lock
= &pd
->driver
->lock
;
769 down_read(&pd
->driver
->sem
);
770 pt
= psb_mmu_pt_map_lock(pd
, virtual);
775 v
= kmap_atomic(pd
->p
);
776 tmp
= v
[psb_mmu_pd_index(virtual)];
780 if (tmp
!= pd
->invalid_pde
|| !(tmp
& PSB_PTE_VALID
) ||
781 !(pd
->invalid_pte
& PSB_PTE_VALID
)) {
786 *pfn
= pd
->invalid_pte
>> PAGE_SHIFT
;
789 tmp
= pt
->v
[psb_mmu_pt_index(virtual)];
790 if (!(tmp
& PSB_PTE_VALID
)) {
794 *pfn
= tmp
>> PAGE_SHIFT
;
796 psb_mmu_pt_unmap_unlock(pt
);
798 up_read(&pd
->driver
->sem
);