2 * drivers/gpu/drm/omapdrm/omap_gem.c
4 * Copyright (C) 2011 Texas Instruments
5 * Author: Rob Clark <rob.clark@linaro.org>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published by
9 * the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * You should have received a copy of the GNU General Public License along with
17 * this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/seq_file.h>
21 #include <linux/shmem_fs.h>
22 #include <linux/spinlock.h>
23 #include <linux/pfn_t.h>
25 #include <drm/drm_vma_manager.h>
28 #include "omap_dmm_tiler.h"
31 * GEM buffer object implementation.
34 /* note: we use upper 8 bits of flags for driver-internal flags: */
35 #define OMAP_BO_MEM_DMA_API 0x01000000 /* memory allocated with the dma_alloc_* API */
36 #define OMAP_BO_MEM_SHMEM 0x02000000 /* memory allocated through shmem backing */
37 #define OMAP_BO_MEM_DMABUF 0x08000000 /* memory imported from a dmabuf */
39 struct omap_gem_object
{
40 struct drm_gem_object base
;
42 struct list_head mm_list
;
46 /** width/height for tiled formats (rounded up to slot boundaries) */
47 uint16_t width
, height
;
49 /** roll applied when mapping to DMM */
53 * paddr contains the buffer DMA address. It is valid for
55 * - buffers allocated through the DMA mapping API (with the
56 * OMAP_BO_MEM_DMA_API flag set)
58 * - buffers imported from dmabuf (with the OMAP_BO_MEM_DMABUF flag set)
59 * if they are physically contiguous (when sgt->orig_nents == 1)
61 * - buffers mapped through the TILER when paddr_cnt is not zero, in
62 * which case the DMA address points to the TILER aperture
64 * Physically contiguous buffers have their DMA address equal to the
65 * physical address as we don't remap those buffers through the TILER.
67 * Buffers mapped to the TILER have their DMA address pointing to the
68 * TILER aperture. As TILER mappings are refcounted (through paddr_cnt)
69 * the DMA address must be accessed through omap_get_get_paddr() to
70 * ensure that the mapping won't disappear unexpectedly. References must
71 * be released with omap_gem_put_paddr().
81 * If the buffer has been imported from a dmabuf the OMAP_DB_DMABUF flag
82 * is set and the sgt field is valid.
87 * tiler block used when buffer is remapped in DMM/TILER.
89 struct tiler_block
*block
;
92 * Array of backing pages, if allocated. Note that pages are never
93 * allocated for buffers originally allocated from contiguous memory
97 /** addresses corresponding to pages in above array */
101 * Virtual address, if mapped.
106 * sync-object allocated on demand (if needed)
108 * Per-buffer sync-object for tracking pending and completed hw/dma
109 * read and write operations.
112 uint32_t write_pending
;
113 uint32_t write_complete
;
114 uint32_t read_pending
;
115 uint32_t read_complete
;
119 #define to_omap_bo(x) container_of(x, struct omap_gem_object, base)
121 /* To deal with userspace mmap'ings of 2d tiled buffers, which (a) are
122 * not necessarily pinned in TILER all the time, and (b) when they are
123 * they are not necessarily page aligned, we reserve one or more small
124 * regions in each of the 2d containers to use as a user-GART where we
125 * can create a second page-aligned mapping of parts of the buffer
126 * being accessed from userspace.
128 * Note that we could optimize slightly when we know that multiple
129 * tiler containers are backed by the same PAT.. but I'll leave that
132 #define NUM_USERGART_ENTRIES 2
133 struct omap_drm_usergart_entry
{
134 struct tiler_block
*block
; /* the reserved tiler block */
136 struct drm_gem_object
*obj
; /* the current pinned obj */
137 pgoff_t obj_pgoff
; /* page offset of obj currently
141 struct omap_drm_usergart
{
142 struct omap_drm_usergart_entry entry
[NUM_USERGART_ENTRIES
];
143 int height
; /* height in rows */
144 int height_shift
; /* ilog2(height in rows) */
145 int slot_shift
; /* ilog2(width per slot) */
146 int stride_pfn
; /* stride in pages */
147 int last
; /* index of last used entry */
150 /* -----------------------------------------------------------------------------
154 /** get mmap offset */
155 static uint64_t mmap_offset(struct drm_gem_object
*obj
)
157 struct drm_device
*dev
= obj
->dev
;
161 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
163 /* Make it mmapable */
164 size
= omap_gem_mmap_size(obj
);
165 ret
= drm_gem_create_mmap_offset_size(obj
, size
);
167 dev_err(dev
->dev
, "could not allocate mmap offset\n");
171 return drm_vma_node_offset_addr(&obj
->vma_node
);
174 static bool is_contiguous(struct omap_gem_object
*omap_obj
)
176 if (omap_obj
->flags
& OMAP_BO_MEM_DMA_API
)
179 if ((omap_obj
->flags
& OMAP_BO_MEM_DMABUF
) && omap_obj
->sgt
->nents
== 1)
185 /* -----------------------------------------------------------------------------
189 static void evict_entry(struct drm_gem_object
*obj
,
190 enum tiler_fmt fmt
, struct omap_drm_usergart_entry
*entry
)
192 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
193 struct omap_drm_private
*priv
= obj
->dev
->dev_private
;
194 int n
= priv
->usergart
[fmt
].height
;
195 size_t size
= PAGE_SIZE
* n
;
196 loff_t off
= mmap_offset(obj
) +
197 (entry
->obj_pgoff
<< PAGE_SHIFT
);
198 const int m
= 1 + ((omap_obj
->width
<< fmt
) / PAGE_SIZE
);
202 /* if stride > than PAGE_SIZE then sparse mapping: */
203 for (i
= n
; i
> 0; i
--) {
204 unmap_mapping_range(obj
->dev
->anon_inode
->i_mapping
,
206 off
+= PAGE_SIZE
* m
;
209 unmap_mapping_range(obj
->dev
->anon_inode
->i_mapping
,
216 /* Evict a buffer from usergart, if it is mapped there */
217 static void evict(struct drm_gem_object
*obj
)
219 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
220 struct omap_drm_private
*priv
= obj
->dev
->dev_private
;
222 if (omap_obj
->flags
& OMAP_BO_TILED
) {
223 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
226 for (i
= 0; i
< NUM_USERGART_ENTRIES
; i
++) {
227 struct omap_drm_usergart_entry
*entry
=
228 &priv
->usergart
[fmt
].entry
[i
];
230 if (entry
->obj
== obj
)
231 evict_entry(obj
, fmt
, entry
);
236 /* -----------------------------------------------------------------------------
240 /** ensure backing pages are allocated */
241 static int omap_gem_attach_pages(struct drm_gem_object
*obj
)
243 struct drm_device
*dev
= obj
->dev
;
244 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
246 int npages
= obj
->size
>> PAGE_SHIFT
;
250 WARN_ON(omap_obj
->pages
);
252 pages
= drm_gem_get_pages(obj
);
254 dev_err(obj
->dev
->dev
, "could not get pages: %ld\n", PTR_ERR(pages
));
255 return PTR_ERR(pages
);
258 /* for non-cached buffers, ensure the new pages are clean because
259 * DSS, GPU, etc. are not cache coherent:
261 if (omap_obj
->flags
& (OMAP_BO_WC
|OMAP_BO_UNCACHED
)) {
262 addrs
= kmalloc(npages
* sizeof(*addrs
), GFP_KERNEL
);
268 for (i
= 0; i
< npages
; i
++) {
269 addrs
[i
] = dma_map_page(dev
->dev
, pages
[i
],
270 0, PAGE_SIZE
, DMA_BIDIRECTIONAL
);
272 if (dma_mapping_error(dev
->dev
, addrs
[i
])) {
274 "%s: failed to map page\n", __func__
);
276 for (i
= i
- 1; i
>= 0; --i
) {
277 dma_unmap_page(dev
->dev
, addrs
[i
],
278 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
286 addrs
= kzalloc(npages
* sizeof(*addrs
), GFP_KERNEL
);
293 omap_obj
->addrs
= addrs
;
294 omap_obj
->pages
= pages
;
301 drm_gem_put_pages(obj
, pages
, true, false);
306 /* acquire pages when needed (for example, for DMA where physically
307 * contiguous buffer is not required
309 static int get_pages(struct drm_gem_object
*obj
, struct page
***pages
)
311 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
314 if ((omap_obj
->flags
& OMAP_BO_MEM_SHMEM
) && !omap_obj
->pages
) {
315 ret
= omap_gem_attach_pages(obj
);
317 dev_err(obj
->dev
->dev
, "could not attach pages\n");
322 /* TODO: even phys-contig.. we should have a list of pages? */
323 *pages
= omap_obj
->pages
;
328 /** release backing pages */
329 static void omap_gem_detach_pages(struct drm_gem_object
*obj
)
331 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
333 /* for non-cached buffers, ensure the new pages are clean because
334 * DSS, GPU, etc. are not cache coherent:
336 if (omap_obj
->flags
& (OMAP_BO_WC
|OMAP_BO_UNCACHED
)) {
337 int i
, npages
= obj
->size
>> PAGE_SHIFT
;
338 for (i
= 0; i
< npages
; i
++) {
339 if (omap_obj
->addrs
[i
])
340 dma_unmap_page(obj
->dev
->dev
,
342 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
346 kfree(omap_obj
->addrs
);
347 omap_obj
->addrs
= NULL
;
349 drm_gem_put_pages(obj
, omap_obj
->pages
, true, false);
350 omap_obj
->pages
= NULL
;
353 /* get buffer flags */
354 uint32_t omap_gem_flags(struct drm_gem_object
*obj
)
356 return to_omap_bo(obj
)->flags
;
359 uint64_t omap_gem_mmap_offset(struct drm_gem_object
*obj
)
362 mutex_lock(&obj
->dev
->struct_mutex
);
363 offset
= mmap_offset(obj
);
364 mutex_unlock(&obj
->dev
->struct_mutex
);
369 size_t omap_gem_mmap_size(struct drm_gem_object
*obj
)
371 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
372 size_t size
= obj
->size
;
374 if (omap_obj
->flags
& OMAP_BO_TILED
) {
375 /* for tiled buffers, the virtual size has stride rounded up
376 * to 4kb.. (to hide the fact that row n+1 might start 16kb or
377 * 32kb later!). But we don't back the entire buffer with
378 * pages, only the valid picture part.. so need to adjust for
379 * this in the size used to mmap and generate mmap offset
381 size
= tiler_vsize(gem2fmt(omap_obj
->flags
),
382 omap_obj
->width
, omap_obj
->height
);
388 /* -----------------------------------------------------------------------------
392 /* Normal handling for the case of faulting in non-tiled buffers */
393 static int fault_1d(struct drm_gem_object
*obj
,
394 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
396 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
400 /* We don't use vmf->pgoff since that has the fake offset: */
401 pgoff
= (vmf
->address
- vma
->vm_start
) >> PAGE_SHIFT
;
403 if (omap_obj
->pages
) {
404 omap_gem_cpu_sync(obj
, pgoff
);
405 pfn
= page_to_pfn(omap_obj
->pages
[pgoff
]);
407 BUG_ON(!is_contiguous(omap_obj
));
408 pfn
= (omap_obj
->paddr
>> PAGE_SHIFT
) + pgoff
;
411 VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf
->address
,
412 pfn
, pfn
<< PAGE_SHIFT
);
414 return vm_insert_mixed(vma
, vmf
->address
, __pfn_to_pfn_t(pfn
, PFN_DEV
));
417 /* Special handling for the case of faulting in 2d tiled buffers */
418 static int fault_2d(struct drm_gem_object
*obj
,
419 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
421 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
422 struct omap_drm_private
*priv
= obj
->dev
->dev_private
;
423 struct omap_drm_usergart_entry
*entry
;
424 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
425 struct page
*pages
[64]; /* XXX is this too much to have on stack? */
427 pgoff_t pgoff
, base_pgoff
;
432 * Note the height of the slot is also equal to the number of pages
433 * that need to be mapped in to fill 4kb wide CPU page. If the slot
434 * height is 64, then 64 pages fill a 4kb wide by 64 row region.
436 const int n
= priv
->usergart
[fmt
].height
;
437 const int n_shift
= priv
->usergart
[fmt
].height_shift
;
440 * If buffer width in bytes > PAGE_SIZE then the virtual stride is
441 * rounded up to next multiple of PAGE_SIZE.. this need to be taken
442 * into account in some of the math, so figure out virtual stride
445 const int m
= 1 + ((omap_obj
->width
<< fmt
) / PAGE_SIZE
);
447 /* We don't use vmf->pgoff since that has the fake offset: */
448 pgoff
= (vmf
->address
- vma
->vm_start
) >> PAGE_SHIFT
;
451 * Actual address we start mapping at is rounded down to previous slot
452 * boundary in the y direction:
454 base_pgoff
= round_down(pgoff
, m
<< n_shift
);
456 /* figure out buffer width in slots */
457 slots
= omap_obj
->width
>> priv
->usergart
[fmt
].slot_shift
;
459 vaddr
= vmf
->address
- ((pgoff
- base_pgoff
) << PAGE_SHIFT
);
461 entry
= &priv
->usergart
[fmt
].entry
[priv
->usergart
[fmt
].last
];
463 /* evict previous buffer using this usergart entry, if any: */
465 evict_entry(entry
->obj
, fmt
, entry
);
468 entry
->obj_pgoff
= base_pgoff
;
470 /* now convert base_pgoff to phys offset from virt offset: */
471 base_pgoff
= (base_pgoff
>> n_shift
) * slots
;
473 /* for wider-than 4k.. figure out which part of the slot-row we want: */
476 entry
->obj_pgoff
+= off
;
478 slots
= min(slots
- (off
<< n_shift
), n
);
479 base_pgoff
+= off
<< n_shift
;
480 vaddr
+= off
<< PAGE_SHIFT
;
484 * Map in pages. Beyond the valid pixel part of the buffer, we set
485 * pages[i] to NULL to get a dummy page mapped in.. if someone
486 * reads/writes it they will get random/undefined content, but at
487 * least it won't be corrupting whatever other random page used to
488 * be mapped in, or other undefined behavior.
490 memcpy(pages
, &omap_obj
->pages
[base_pgoff
],
491 sizeof(struct page
*) * slots
);
492 memset(pages
+ slots
, 0,
493 sizeof(struct page
*) * (n
- slots
));
495 ret
= tiler_pin(entry
->block
, pages
, ARRAY_SIZE(pages
), 0, true);
497 dev_err(obj
->dev
->dev
, "failed to pin: %d\n", ret
);
501 pfn
= entry
->paddr
>> PAGE_SHIFT
;
503 VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf
->address
,
504 pfn
, pfn
<< PAGE_SHIFT
);
506 for (i
= n
; i
> 0; i
--) {
507 vm_insert_mixed(vma
, vaddr
, __pfn_to_pfn_t(pfn
, PFN_DEV
));
508 pfn
+= priv
->usergart
[fmt
].stride_pfn
;
509 vaddr
+= PAGE_SIZE
* m
;
512 /* simple round-robin: */
513 priv
->usergart
[fmt
].last
= (priv
->usergart
[fmt
].last
+ 1)
514 % NUM_USERGART_ENTRIES
;
520 * omap_gem_fault - pagefault handler for GEM objects
521 * @vma: the VMA of the GEM object
524 * Invoked when a fault occurs on an mmap of a GEM managed area. GEM
525 * does most of the work for us including the actual map/unmap calls
526 * but we need to do the actual page work.
528 * The VMA was set up by GEM. In doing so it also ensured that the
529 * vma->vm_private_data points to the GEM object that is backing this
532 int omap_gem_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
534 struct drm_gem_object
*obj
= vma
->vm_private_data
;
535 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
536 struct drm_device
*dev
= obj
->dev
;
540 /* Make sure we don't parallel update on a fault, nor move or remove
541 * something from beneath our feet
543 mutex_lock(&dev
->struct_mutex
);
545 /* if a shmem backed object, make sure we have pages attached now */
546 ret
= get_pages(obj
, &pages
);
550 /* where should we do corresponding put_pages().. we are mapping
551 * the original page, rather than thru a GART, so we can't rely
552 * on eviction to trigger this. But munmap() or all mappings should
553 * probably trigger put_pages()?
556 if (omap_obj
->flags
& OMAP_BO_TILED
)
557 ret
= fault_2d(obj
, vma
, vmf
);
559 ret
= fault_1d(obj
, vma
, vmf
);
563 mutex_unlock(&dev
->struct_mutex
);
570 * EBUSY is ok: this just means that another thread
571 * already did the job.
573 return VM_FAULT_NOPAGE
;
577 return VM_FAULT_SIGBUS
;
581 /** We override mainly to fix up some of the vm mapping flags.. */
582 int omap_gem_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
586 ret
= drm_gem_mmap(filp
, vma
);
588 DBG("mmap failed: %d", ret
);
592 return omap_gem_mmap_obj(vma
->vm_private_data
, vma
);
595 int omap_gem_mmap_obj(struct drm_gem_object
*obj
,
596 struct vm_area_struct
*vma
)
598 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
600 vma
->vm_flags
&= ~VM_PFNMAP
;
601 vma
->vm_flags
|= VM_MIXEDMAP
;
603 if (omap_obj
->flags
& OMAP_BO_WC
) {
604 vma
->vm_page_prot
= pgprot_writecombine(vm_get_page_prot(vma
->vm_flags
));
605 } else if (omap_obj
->flags
& OMAP_BO_UNCACHED
) {
606 vma
->vm_page_prot
= pgprot_noncached(vm_get_page_prot(vma
->vm_flags
));
609 * We do have some private objects, at least for scanout buffers
610 * on hardware without DMM/TILER. But these are allocated write-
613 if (WARN_ON(!obj
->filp
))
617 * Shunt off cached objs to shmem file so they have their own
618 * address_space (so unmap_mapping_range does what we want,
619 * in particular in the case of mmap'd dmabufs)
623 vma
->vm_file
= get_file(obj
->filp
);
625 vma
->vm_page_prot
= vm_get_page_prot(vma
->vm_flags
);
631 /* -----------------------------------------------------------------------------
636 * omap_gem_dumb_create - create a dumb buffer
637 * @drm_file: our client file
639 * @args: the requested arguments copied from userspace
641 * Allocate a buffer suitable for use for a frame buffer of the
642 * form described by user space. Give userspace a handle by which
645 int omap_gem_dumb_create(struct drm_file
*file
, struct drm_device
*dev
,
646 struct drm_mode_create_dumb
*args
)
648 union omap_gem_size gsize
;
650 args
->pitch
= DIV_ROUND_UP(args
->width
* args
->bpp
, 8);
652 args
->size
= PAGE_ALIGN(args
->pitch
* args
->height
);
654 gsize
= (union omap_gem_size
){
658 return omap_gem_new_handle(dev
, file
, gsize
,
659 OMAP_BO_SCANOUT
| OMAP_BO_WC
, &args
->handle
);
663 * omap_gem_dumb_map - buffer mapping for dumb interface
664 * @file: our drm client file
666 * @handle: GEM handle to the object (from dumb_create)
668 * Do the necessary setup to allow the mapping of the frame buffer
669 * into user memory. We don't have to do much here at the moment.
671 int omap_gem_dumb_map_offset(struct drm_file
*file
, struct drm_device
*dev
,
672 uint32_t handle
, uint64_t *offset
)
674 struct drm_gem_object
*obj
;
677 /* GEM does all our handle to object mapping */
678 obj
= drm_gem_object_lookup(file
, handle
);
684 *offset
= omap_gem_mmap_offset(obj
);
686 drm_gem_object_unreference_unlocked(obj
);
692 #ifdef CONFIG_DRM_FBDEV_EMULATION
693 /* Set scrolling position. This allows us to implement fast scrolling
696 * Call only from non-atomic contexts.
698 int omap_gem_roll(struct drm_gem_object
*obj
, uint32_t roll
)
700 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
701 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
705 dev_err(obj
->dev
->dev
, "invalid roll: %d\n", roll
);
709 omap_obj
->roll
= roll
;
711 mutex_lock(&obj
->dev
->struct_mutex
);
713 /* if we aren't mapped yet, we don't need to do anything */
714 if (omap_obj
->block
) {
716 ret
= get_pages(obj
, &pages
);
719 ret
= tiler_pin(omap_obj
->block
, pages
, npages
, roll
, true);
721 dev_err(obj
->dev
->dev
, "could not repin: %d\n", ret
);
725 mutex_unlock(&obj
->dev
->struct_mutex
);
731 /* -----------------------------------------------------------------------------
732 * Memory Management & DMA Sync
736 * shmem buffers that are mapped cached can simulate coherency via using
737 * page faulting to keep track of dirty pages
739 static inline bool is_cached_coherent(struct drm_gem_object
*obj
)
741 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
743 return (omap_obj
->flags
& OMAP_BO_MEM_SHMEM
) &&
744 ((omap_obj
->flags
& OMAP_BO_CACHE_MASK
) == OMAP_BO_CACHED
);
747 /* Sync the buffer for CPU access.. note pages should already be
748 * attached, ie. omap_gem_get_pages()
750 void omap_gem_cpu_sync(struct drm_gem_object
*obj
, int pgoff
)
752 struct drm_device
*dev
= obj
->dev
;
753 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
755 if (is_cached_coherent(obj
) && omap_obj
->addrs
[pgoff
]) {
756 dma_unmap_page(dev
->dev
, omap_obj
->addrs
[pgoff
],
757 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
758 omap_obj
->addrs
[pgoff
] = 0;
762 /* sync the buffer for DMA access */
763 void omap_gem_dma_sync(struct drm_gem_object
*obj
,
764 enum dma_data_direction dir
)
766 struct drm_device
*dev
= obj
->dev
;
767 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
769 if (is_cached_coherent(obj
)) {
770 int i
, npages
= obj
->size
>> PAGE_SHIFT
;
771 struct page
**pages
= omap_obj
->pages
;
774 for (i
= 0; i
< npages
; i
++) {
775 if (!omap_obj
->addrs
[i
]) {
778 addr
= dma_map_page(dev
->dev
, pages
[i
], 0,
779 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
781 if (dma_mapping_error(dev
->dev
, addr
)) {
783 "%s: failed to map page\n",
789 omap_obj
->addrs
[i
] = addr
;
794 unmap_mapping_range(obj
->filp
->f_mapping
, 0,
795 omap_gem_mmap_size(obj
), 1);
800 /* Get physical address for DMA.. if 'remap' is true, and the buffer is not
801 * already contiguous, remap it to pin in physically contiguous memory.. (ie.
804 int omap_gem_get_paddr(struct drm_gem_object
*obj
,
805 dma_addr_t
*paddr
, bool remap
)
807 struct omap_drm_private
*priv
= obj
->dev
->dev_private
;
808 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
811 mutex_lock(&obj
->dev
->struct_mutex
);
813 if (!is_contiguous(omap_obj
) && remap
&& priv
->has_dmm
) {
814 if (omap_obj
->paddr_cnt
== 0) {
816 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
817 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
818 struct tiler_block
*block
;
820 BUG_ON(omap_obj
->block
);
822 ret
= get_pages(obj
, &pages
);
826 if (omap_obj
->flags
& OMAP_BO_TILED
) {
827 block
= tiler_reserve_2d(fmt
,
829 omap_obj
->height
, 0);
831 block
= tiler_reserve_1d(obj
->size
);
835 ret
= PTR_ERR(block
);
836 dev_err(obj
->dev
->dev
,
837 "could not remap: %d (%d)\n", ret
, fmt
);
841 /* TODO: enable async refill.. */
842 ret
= tiler_pin(block
, pages
, npages
,
843 omap_obj
->roll
, true);
845 tiler_release(block
);
846 dev_err(obj
->dev
->dev
,
847 "could not pin: %d\n", ret
);
851 omap_obj
->paddr
= tiler_ssptr(block
);
852 omap_obj
->block
= block
;
854 DBG("got paddr: %pad", &omap_obj
->paddr
);
857 omap_obj
->paddr_cnt
++;
859 *paddr
= omap_obj
->paddr
;
860 } else if (is_contiguous(omap_obj
)) {
861 *paddr
= omap_obj
->paddr
;
868 mutex_unlock(&obj
->dev
->struct_mutex
);
873 /* Release physical address, when DMA is no longer being performed.. this
874 * could potentially unpin and unmap buffers from TILER
876 void omap_gem_put_paddr(struct drm_gem_object
*obj
)
878 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
881 mutex_lock(&obj
->dev
->struct_mutex
);
882 if (omap_obj
->paddr_cnt
> 0) {
883 omap_obj
->paddr_cnt
--;
884 if (omap_obj
->paddr_cnt
== 0) {
885 ret
= tiler_unpin(omap_obj
->block
);
887 dev_err(obj
->dev
->dev
,
888 "could not unpin pages: %d\n", ret
);
890 ret
= tiler_release(omap_obj
->block
);
892 dev_err(obj
->dev
->dev
,
893 "could not release unmap: %d\n", ret
);
896 omap_obj
->block
= NULL
;
900 mutex_unlock(&obj
->dev
->struct_mutex
);
903 /* Get rotated scanout address (only valid if already pinned), at the
904 * specified orientation and x,y offset from top-left corner of buffer
905 * (only valid for tiled 2d buffers)
907 int omap_gem_rotated_paddr(struct drm_gem_object
*obj
, uint32_t orient
,
908 int x
, int y
, dma_addr_t
*paddr
)
910 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
913 mutex_lock(&obj
->dev
->struct_mutex
);
914 if ((omap_obj
->paddr_cnt
> 0) && omap_obj
->block
&&
915 (omap_obj
->flags
& OMAP_BO_TILED
)) {
916 *paddr
= tiler_tsptr(omap_obj
->block
, orient
, x
, y
);
919 mutex_unlock(&obj
->dev
->struct_mutex
);
923 /* Get tiler stride for the buffer (only valid for 2d tiled buffers) */
924 int omap_gem_tiled_stride(struct drm_gem_object
*obj
, uint32_t orient
)
926 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
928 if (omap_obj
->flags
& OMAP_BO_TILED
)
929 ret
= tiler_stride(gem2fmt(omap_obj
->flags
), orient
);
933 /* if !remap, and we don't have pages backing, then fail, rather than
934 * increasing the pin count (which we don't really do yet anyways,
935 * because we don't support swapping pages back out). And 'remap'
936 * might not be quite the right name, but I wanted to keep it working
937 * similarly to omap_gem_get_paddr(). Note though that mutex is not
938 * aquired if !remap (because this can be called in atomic ctxt),
939 * but probably omap_gem_get_paddr() should be changed to work in the
940 * same way. If !remap, a matching omap_gem_put_pages() call is not
941 * required (and should not be made).
943 int omap_gem_get_pages(struct drm_gem_object
*obj
, struct page
***pages
,
948 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
949 if (!omap_obj
->pages
)
951 *pages
= omap_obj
->pages
;
954 mutex_lock(&obj
->dev
->struct_mutex
);
955 ret
= get_pages(obj
, pages
);
956 mutex_unlock(&obj
->dev
->struct_mutex
);
960 /* release pages when DMA no longer being performed */
961 int omap_gem_put_pages(struct drm_gem_object
*obj
)
963 /* do something here if we dynamically attach/detach pages.. at
964 * least they would no longer need to be pinned if everyone has
965 * released the pages..
970 #ifdef CONFIG_DRM_FBDEV_EMULATION
971 /* Get kernel virtual address for CPU access.. this more or less only
972 * exists for omap_fbdev. This should be called with struct_mutex
975 void *omap_gem_vaddr(struct drm_gem_object
*obj
)
977 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
978 WARN_ON(!mutex_is_locked(&obj
->dev
->struct_mutex
));
979 if (!omap_obj
->vaddr
) {
981 int ret
= get_pages(obj
, &pages
);
984 omap_obj
->vaddr
= vmap(pages
, obj
->size
>> PAGE_SHIFT
,
985 VM_MAP
, pgprot_writecombine(PAGE_KERNEL
));
987 return omap_obj
->vaddr
;
991 /* -----------------------------------------------------------------------------
996 /* re-pin objects in DMM in resume path: */
997 int omap_gem_resume(struct device
*dev
)
999 struct drm_device
*drm_dev
= dev_get_drvdata(dev
);
1000 struct omap_drm_private
*priv
= drm_dev
->dev_private
;
1001 struct omap_gem_object
*omap_obj
;
1004 list_for_each_entry(omap_obj
, &priv
->obj_list
, mm_list
) {
1005 if (omap_obj
->block
) {
1006 struct drm_gem_object
*obj
= &omap_obj
->base
;
1007 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
1008 WARN_ON(!omap_obj
->pages
); /* this can't happen */
1009 ret
= tiler_pin(omap_obj
->block
,
1010 omap_obj
->pages
, npages
,
1011 omap_obj
->roll
, true);
1013 dev_err(dev
, "could not repin: %d\n", ret
);
1023 /* -----------------------------------------------------------------------------
1027 #ifdef CONFIG_DEBUG_FS
1028 void omap_gem_describe(struct drm_gem_object
*obj
, struct seq_file
*m
)
1030 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1033 off
= drm_vma_node_start(&obj
->vma_node
);
1035 seq_printf(m
, "%08x: %2d (%2d) %08llx %pad (%2d) %p %4d",
1036 omap_obj
->flags
, obj
->name
, kref_read(&obj
->refcount
),
1037 off
, &omap_obj
->paddr
, omap_obj
->paddr_cnt
,
1038 omap_obj
->vaddr
, omap_obj
->roll
);
1040 if (omap_obj
->flags
& OMAP_BO_TILED
) {
1041 seq_printf(m
, " %dx%d", omap_obj
->width
, omap_obj
->height
);
1042 if (omap_obj
->block
) {
1043 struct tcm_area
*area
= &omap_obj
->block
->area
;
1044 seq_printf(m
, " (%dx%d, %dx%d)",
1045 area
->p0
.x
, area
->p0
.y
,
1046 area
->p1
.x
, area
->p1
.y
);
1049 seq_printf(m
, " %d", obj
->size
);
1052 seq_printf(m
, "\n");
1055 void omap_gem_describe_objects(struct list_head
*list
, struct seq_file
*m
)
1057 struct omap_gem_object
*omap_obj
;
1061 list_for_each_entry(omap_obj
, list
, mm_list
) {
1062 struct drm_gem_object
*obj
= &omap_obj
->base
;
1064 omap_gem_describe(obj
, m
);
1069 seq_printf(m
, "Total %d objects, %zu bytes\n", count
, size
);
1073 /* -----------------------------------------------------------------------------
1074 * Buffer Synchronization
1077 static DEFINE_SPINLOCK(sync_lock
);
1079 struct omap_gem_sync_waiter
{
1080 struct list_head list
;
1081 struct omap_gem_object
*omap_obj
;
1082 enum omap_gem_op op
;
1083 uint32_t read_target
, write_target
;
1084 /* notify called w/ sync_lock held */
1085 void (*notify
)(void *arg
);
1089 /* list of omap_gem_sync_waiter.. the notify fxn gets called back when
1090 * the read and/or write target count is achieved which can call a user
1091 * callback (ex. to kick 3d and/or 2d), wakeup blocked task (prep for
1094 static LIST_HEAD(waiters
);
1096 static inline bool is_waiting(struct omap_gem_sync_waiter
*waiter
)
1098 struct omap_gem_object
*omap_obj
= waiter
->omap_obj
;
1099 if ((waiter
->op
& OMAP_GEM_READ
) &&
1100 (omap_obj
->sync
->write_complete
< waiter
->write_target
))
1102 if ((waiter
->op
& OMAP_GEM_WRITE
) &&
1103 (omap_obj
->sync
->read_complete
< waiter
->read_target
))
1108 /* macro for sync debug.. */
1110 #define SYNC(fmt, ...) do { if (SYNCDBG) \
1111 printk(KERN_ERR "%s:%d: "fmt"\n", \
1112 __func__, __LINE__, ##__VA_ARGS__); \
1116 static void sync_op_update(void)
1118 struct omap_gem_sync_waiter
*waiter
, *n
;
1119 list_for_each_entry_safe(waiter
, n
, &waiters
, list
) {
1120 if (!is_waiting(waiter
)) {
1121 list_del(&waiter
->list
);
1122 SYNC("notify: %p", waiter
);
1123 waiter
->notify(waiter
->arg
);
1129 static inline int sync_op(struct drm_gem_object
*obj
,
1130 enum omap_gem_op op
, bool start
)
1132 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1135 spin_lock(&sync_lock
);
1137 if (!omap_obj
->sync
) {
1138 omap_obj
->sync
= kzalloc(sizeof(*omap_obj
->sync
), GFP_ATOMIC
);
1139 if (!omap_obj
->sync
) {
1146 if (op
& OMAP_GEM_READ
)
1147 omap_obj
->sync
->read_pending
++;
1148 if (op
& OMAP_GEM_WRITE
)
1149 omap_obj
->sync
->write_pending
++;
1151 if (op
& OMAP_GEM_READ
)
1152 omap_obj
->sync
->read_complete
++;
1153 if (op
& OMAP_GEM_WRITE
)
1154 omap_obj
->sync
->write_complete
++;
1159 spin_unlock(&sync_lock
);
1164 /* mark the start of read and/or write operation */
1165 int omap_gem_op_start(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1167 return sync_op(obj
, op
, true);
1170 int omap_gem_op_finish(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1172 return sync_op(obj
, op
, false);
1175 static DECLARE_WAIT_QUEUE_HEAD(sync_event
);
1177 static void sync_notify(void *arg
)
1179 struct task_struct
**waiter_task
= arg
;
1180 *waiter_task
= NULL
;
1181 wake_up_all(&sync_event
);
1184 int omap_gem_op_sync(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1186 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1188 if (omap_obj
->sync
) {
1189 struct task_struct
*waiter_task
= current
;
1190 struct omap_gem_sync_waiter
*waiter
=
1191 kzalloc(sizeof(*waiter
), GFP_KERNEL
);
1196 waiter
->omap_obj
= omap_obj
;
1198 waiter
->read_target
= omap_obj
->sync
->read_pending
;
1199 waiter
->write_target
= omap_obj
->sync
->write_pending
;
1200 waiter
->notify
= sync_notify
;
1201 waiter
->arg
= &waiter_task
;
1203 spin_lock(&sync_lock
);
1204 if (is_waiting(waiter
)) {
1205 SYNC("waited: %p", waiter
);
1206 list_add_tail(&waiter
->list
, &waiters
);
1207 spin_unlock(&sync_lock
);
1208 ret
= wait_event_interruptible(sync_event
,
1209 (waiter_task
== NULL
));
1210 spin_lock(&sync_lock
);
1212 SYNC("interrupted: %p", waiter
);
1213 /* we were interrupted */
1214 list_del(&waiter
->list
);
1217 /* freed in sync_op_update() */
1221 spin_unlock(&sync_lock
);
1227 /* call fxn(arg), either synchronously or asynchronously if the op
1228 * is currently blocked.. fxn() can be called from any context
1230 * (TODO for now fxn is called back from whichever context calls
1231 * omap_gem_op_finish().. but this could be better defined later
1234 * TODO more code in common w/ _sync()..
1236 int omap_gem_op_async(struct drm_gem_object
*obj
, enum omap_gem_op op
,
1237 void (*fxn
)(void *arg
), void *arg
)
1239 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1240 if (omap_obj
->sync
) {
1241 struct omap_gem_sync_waiter
*waiter
=
1242 kzalloc(sizeof(*waiter
), GFP_ATOMIC
);
1247 waiter
->omap_obj
= omap_obj
;
1249 waiter
->read_target
= omap_obj
->sync
->read_pending
;
1250 waiter
->write_target
= omap_obj
->sync
->write_pending
;
1251 waiter
->notify
= fxn
;
1254 spin_lock(&sync_lock
);
1255 if (is_waiting(waiter
)) {
1256 SYNC("waited: %p", waiter
);
1257 list_add_tail(&waiter
->list
, &waiters
);
1258 spin_unlock(&sync_lock
);
1262 spin_unlock(&sync_lock
);
1273 /* -----------------------------------------------------------------------------
1274 * Constructor & Destructor
1277 void omap_gem_free_object(struct drm_gem_object
*obj
)
1279 struct drm_device
*dev
= obj
->dev
;
1280 struct omap_drm_private
*priv
= dev
->dev_private
;
1281 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1285 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
1287 spin_lock(&priv
->list_lock
);
1288 list_del(&omap_obj
->mm_list
);
1289 spin_unlock(&priv
->list_lock
);
1291 /* this means the object is still pinned.. which really should
1292 * not happen. I think..
1294 WARN_ON(omap_obj
->paddr_cnt
> 0);
1296 if (omap_obj
->pages
) {
1297 if (omap_obj
->flags
& OMAP_BO_MEM_DMABUF
)
1298 kfree(omap_obj
->pages
);
1300 omap_gem_detach_pages(obj
);
1303 if (omap_obj
->flags
& OMAP_BO_MEM_DMA_API
) {
1304 dma_free_wc(dev
->dev
, obj
->size
, omap_obj
->vaddr
,
1306 } else if (omap_obj
->vaddr
) {
1307 vunmap(omap_obj
->vaddr
);
1308 } else if (obj
->import_attach
) {
1309 drm_prime_gem_destroy(obj
, omap_obj
->sgt
);
1312 kfree(omap_obj
->sync
);
1314 drm_gem_object_release(obj
);
1319 /* GEM buffer object constructor */
1320 struct drm_gem_object
*omap_gem_new(struct drm_device
*dev
,
1321 union omap_gem_size gsize
, uint32_t flags
)
1323 struct omap_drm_private
*priv
= dev
->dev_private
;
1324 struct omap_gem_object
*omap_obj
;
1325 struct drm_gem_object
*obj
;
1326 struct address_space
*mapping
;
1330 /* Validate the flags and compute the memory and cache flags. */
1331 if (flags
& OMAP_BO_TILED
) {
1332 if (!priv
->usergart
) {
1333 dev_err(dev
->dev
, "Tiled buffers require DMM\n");
1338 * Tiled buffers are always shmem paged backed. When they are
1339 * scanned out, they are remapped into DMM/TILER.
1341 flags
&= ~OMAP_BO_SCANOUT
;
1342 flags
|= OMAP_BO_MEM_SHMEM
;
1345 * Currently don't allow cached buffers. There is some caching
1346 * stuff that needs to be handled better.
1348 flags
&= ~(OMAP_BO_CACHED
|OMAP_BO_WC
|OMAP_BO_UNCACHED
);
1349 flags
|= tiler_get_cpu_cache_flags();
1350 } else if ((flags
& OMAP_BO_SCANOUT
) && !priv
->has_dmm
) {
1352 * OMAP_BO_SCANOUT hints that the buffer doesn't need to be
1353 * tiled. However, to lower the pressure on memory allocation,
1354 * use contiguous memory only if no TILER is available.
1356 flags
|= OMAP_BO_MEM_DMA_API
;
1357 } else if (!(flags
& OMAP_BO_MEM_DMABUF
)) {
1359 * All other buffers not backed by dma_buf are shmem-backed.
1361 flags
|= OMAP_BO_MEM_SHMEM
;
1364 /* Allocate the initialize the OMAP GEM object. */
1365 omap_obj
= kzalloc(sizeof(*omap_obj
), GFP_KERNEL
);
1369 obj
= &omap_obj
->base
;
1370 omap_obj
->flags
= flags
;
1372 if (flags
& OMAP_BO_TILED
) {
1374 * For tiled buffers align dimensions to slot boundaries and
1375 * calculate size based on aligned dimensions.
1377 tiler_align(gem2fmt(flags
), &gsize
.tiled
.width
,
1378 &gsize
.tiled
.height
);
1380 size
= tiler_size(gem2fmt(flags
), gsize
.tiled
.width
,
1381 gsize
.tiled
.height
);
1383 omap_obj
->width
= gsize
.tiled
.width
;
1384 omap_obj
->height
= gsize
.tiled
.height
;
1386 size
= PAGE_ALIGN(gsize
.bytes
);
1389 /* Initialize the GEM object. */
1390 if (!(flags
& OMAP_BO_MEM_SHMEM
)) {
1391 drm_gem_private_object_init(dev
, obj
, size
);
1393 ret
= drm_gem_object_init(dev
, obj
, size
);
1397 mapping
= obj
->filp
->f_mapping
;
1398 mapping_set_gfp_mask(mapping
, GFP_USER
| __GFP_DMA32
);
1401 /* Allocate memory if needed. */
1402 if (flags
& OMAP_BO_MEM_DMA_API
) {
1403 omap_obj
->vaddr
= dma_alloc_wc(dev
->dev
, size
,
1406 if (!omap_obj
->vaddr
)
1410 spin_lock(&priv
->list_lock
);
1411 list_add(&omap_obj
->mm_list
, &priv
->obj_list
);
1412 spin_unlock(&priv
->list_lock
);
1417 drm_gem_object_release(obj
);
1423 struct drm_gem_object
*omap_gem_new_dmabuf(struct drm_device
*dev
, size_t size
,
1424 struct sg_table
*sgt
)
1426 struct omap_drm_private
*priv
= dev
->dev_private
;
1427 struct omap_gem_object
*omap_obj
;
1428 struct drm_gem_object
*obj
;
1429 union omap_gem_size gsize
;
1431 /* Without a DMM only physically contiguous buffers can be supported. */
1432 if (sgt
->orig_nents
!= 1 && !priv
->has_dmm
)
1433 return ERR_PTR(-EINVAL
);
1435 mutex_lock(&dev
->struct_mutex
);
1437 gsize
.bytes
= PAGE_ALIGN(size
);
1438 obj
= omap_gem_new(dev
, gsize
, OMAP_BO_MEM_DMABUF
| OMAP_BO_WC
);
1440 obj
= ERR_PTR(-ENOMEM
);
1444 omap_obj
= to_omap_bo(obj
);
1445 omap_obj
->sgt
= sgt
;
1447 if (sgt
->orig_nents
== 1) {
1448 omap_obj
->paddr
= sg_dma_address(sgt
->sgl
);
1450 /* Create pages list from sgt */
1451 struct sg_page_iter iter
;
1452 struct page
**pages
;
1453 unsigned int npages
;
1456 npages
= DIV_ROUND_UP(size
, PAGE_SIZE
);
1457 pages
= kcalloc(npages
, sizeof(*pages
), GFP_KERNEL
);
1459 omap_gem_free_object(obj
);
1460 obj
= ERR_PTR(-ENOMEM
);
1464 omap_obj
->pages
= pages
;
1466 for_each_sg_page(sgt
->sgl
, &iter
, sgt
->orig_nents
, 0) {
1467 pages
[i
++] = sg_page_iter_page(&iter
);
1472 if (WARN_ON(i
!= npages
)) {
1473 omap_gem_free_object(obj
);
1474 obj
= ERR_PTR(-ENOMEM
);
1480 mutex_unlock(&dev
->struct_mutex
);
1484 /* convenience method to construct a GEM buffer object, and userspace handle */
1485 int omap_gem_new_handle(struct drm_device
*dev
, struct drm_file
*file
,
1486 union omap_gem_size gsize
, uint32_t flags
, uint32_t *handle
)
1488 struct drm_gem_object
*obj
;
1491 obj
= omap_gem_new(dev
, gsize
, flags
);
1495 ret
= drm_gem_handle_create(file
, obj
, handle
);
1497 omap_gem_free_object(obj
);
1501 /* drop reference from allocate - handle holds it now */
1502 drm_gem_object_unreference_unlocked(obj
);
1507 /* -----------------------------------------------------------------------------
1511 /* If DMM is used, we need to set some stuff up.. */
1512 void omap_gem_init(struct drm_device
*dev
)
1514 struct omap_drm_private
*priv
= dev
->dev_private
;
1515 struct omap_drm_usergart
*usergart
;
1516 const enum tiler_fmt fmts
[] = {
1517 TILFMT_8BIT
, TILFMT_16BIT
, TILFMT_32BIT
1521 if (!dmm_is_available()) {
1522 /* DMM only supported on OMAP4 and later, so this isn't fatal */
1523 dev_warn(dev
->dev
, "DMM not available, disable DMM support\n");
1527 usergart
= kcalloc(3, sizeof(*usergart
), GFP_KERNEL
);
1531 /* reserve 4k aligned/wide regions for userspace mappings: */
1532 for (i
= 0; i
< ARRAY_SIZE(fmts
); i
++) {
1533 uint16_t h
= 1, w
= PAGE_SIZE
>> i
;
1534 tiler_align(fmts
[i
], &w
, &h
);
1535 /* note: since each region is 1 4kb page wide, and minimum
1536 * number of rows, the height ends up being the same as the
1537 * # of pages in the region
1539 usergart
[i
].height
= h
;
1540 usergart
[i
].height_shift
= ilog2(h
);
1541 usergart
[i
].stride_pfn
= tiler_stride(fmts
[i
], 0) >> PAGE_SHIFT
;
1542 usergart
[i
].slot_shift
= ilog2((PAGE_SIZE
/ h
) >> i
);
1543 for (j
= 0; j
< NUM_USERGART_ENTRIES
; j
++) {
1544 struct omap_drm_usergart_entry
*entry
;
1545 struct tiler_block
*block
;
1547 entry
= &usergart
[i
].entry
[j
];
1548 block
= tiler_reserve_2d(fmts
[i
], w
, h
, PAGE_SIZE
);
1549 if (IS_ERR(block
)) {
1551 "reserve failed: %d, %d, %ld\n",
1552 i
, j
, PTR_ERR(block
));
1555 entry
->paddr
= tiler_ssptr(block
);
1556 entry
->block
= block
;
1558 DBG("%d:%d: %dx%d: paddr=%pad stride=%d", i
, j
, w
, h
,
1560 usergart
[i
].stride_pfn
<< PAGE_SHIFT
);
1564 priv
->usergart
= usergart
;
1565 priv
->has_dmm
= true;
1568 void omap_gem_deinit(struct drm_device
*dev
)
1570 struct omap_drm_private
*priv
= dev
->dev_private
;
1572 /* I believe we can rely on there being no more outstanding GEM
1573 * objects which could depend on usergart/dmm at this point.
1575 kfree(priv
->usergart
);