2 * Copyright 2015 Advanced Micro Devices, Inc.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Christian König <deathsimple@vodafone.de>
33 #include "amdgpu_trace.h"
35 #define AMDGPU_BO_LIST_MAX_PRIORITY 32u
36 #define AMDGPU_BO_LIST_NUM_BUCKETS (AMDGPU_BO_LIST_MAX_PRIORITY + 1)
38 static void amdgpu_bo_list_free_rcu(struct rcu_head
*rcu
)
40 struct amdgpu_bo_list
*list
= container_of(rcu
, struct amdgpu_bo_list
,
46 static void amdgpu_bo_list_free(struct kref
*ref
)
48 struct amdgpu_bo_list
*list
= container_of(ref
, struct amdgpu_bo_list
,
50 struct amdgpu_bo_list_entry
*e
;
52 amdgpu_bo_list_for_each_entry(e
, list
) {
53 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
58 call_rcu(&list
->rhead
, amdgpu_bo_list_free_rcu
);
61 int amdgpu_bo_list_create(struct amdgpu_device
*adev
, struct drm_file
*filp
,
62 struct drm_amdgpu_bo_list_entry
*info
,
63 unsigned num_entries
, struct amdgpu_bo_list
**result
)
65 unsigned last_entry
= 0, first_userptr
= num_entries
;
66 struct amdgpu_bo_list_entry
*array
;
67 struct amdgpu_bo_list
*list
;
68 uint64_t total_size
= 0;
73 if (num_entries
> (SIZE_MAX
- sizeof(struct amdgpu_bo_list
))
74 / sizeof(struct amdgpu_bo_list_entry
))
77 size
= sizeof(struct amdgpu_bo_list
);
78 size
+= num_entries
* sizeof(struct amdgpu_bo_list_entry
);
79 list
= kvmalloc(size
, GFP_KERNEL
);
83 kref_init(&list
->refcount
);
84 list
->gds_obj
= adev
->gds
.gds_gfx_bo
;
85 list
->gws_obj
= adev
->gds
.gws_gfx_bo
;
86 list
->oa_obj
= adev
->gds
.oa_gfx_bo
;
88 array
= amdgpu_bo_list_array_entry(list
, 0);
89 memset(array
, 0, num_entries
* sizeof(struct amdgpu_bo_list_entry
));
91 for (i
= 0; i
< num_entries
; ++i
) {
92 struct amdgpu_bo_list_entry
*entry
;
93 struct drm_gem_object
*gobj
;
95 struct mm_struct
*usermm
;
97 gobj
= drm_gem_object_lookup(filp
, info
[i
].bo_handle
);
103 bo
= amdgpu_bo_ref(gem_to_amdgpu_bo(gobj
));
104 drm_gem_object_put_unlocked(gobj
);
106 usermm
= amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
);
108 if (usermm
!= current
->mm
) {
109 amdgpu_bo_unref(&bo
);
113 entry
= &array
[--first_userptr
];
115 entry
= &array
[last_entry
++];
118 entry
->priority
= min(info
[i
].bo_priority
,
119 AMDGPU_BO_LIST_MAX_PRIORITY
);
120 entry
->tv
.bo
= &bo
->tbo
;
121 entry
->tv
.shared
= !bo
->prime_shared_count
;
123 if (bo
->preferred_domains
== AMDGPU_GEM_DOMAIN_GDS
)
125 if (bo
->preferred_domains
== AMDGPU_GEM_DOMAIN_GWS
)
127 if (bo
->preferred_domains
== AMDGPU_GEM_DOMAIN_OA
)
130 total_size
+= amdgpu_bo_size(bo
);
131 trace_amdgpu_bo_list_set(list
, bo
);
134 list
->first_userptr
= first_userptr
;
135 list
->num_entries
= num_entries
;
137 trace_amdgpu_cs_bo_status(list
->num_entries
, total_size
);
144 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(array
[i
].tv
.bo
);
146 amdgpu_bo_unref(&bo
);
153 static void amdgpu_bo_list_destroy(struct amdgpu_fpriv
*fpriv
, int id
)
155 struct amdgpu_bo_list
*list
;
157 mutex_lock(&fpriv
->bo_list_lock
);
158 list
= idr_remove(&fpriv
->bo_list_handles
, id
);
159 mutex_unlock(&fpriv
->bo_list_lock
);
161 kref_put(&list
->refcount
, amdgpu_bo_list_free
);
164 int amdgpu_bo_list_get(struct amdgpu_fpriv
*fpriv
, int id
,
165 struct amdgpu_bo_list
**result
)
168 *result
= idr_find(&fpriv
->bo_list_handles
, id
);
170 if (*result
&& kref_get_unless_zero(&(*result
)->refcount
)) {
179 void amdgpu_bo_list_get_list(struct amdgpu_bo_list
*list
,
180 struct list_head
*validated
)
182 /* This is based on the bucket sort with O(n) time complexity.
183 * An item with priority "i" is added to bucket[i]. The lists are then
184 * concatenated in descending order.
186 struct list_head bucket
[AMDGPU_BO_LIST_NUM_BUCKETS
];
187 struct amdgpu_bo_list_entry
*e
;
190 for (i
= 0; i
< AMDGPU_BO_LIST_NUM_BUCKETS
; i
++)
191 INIT_LIST_HEAD(&bucket
[i
]);
193 /* Since buffers which appear sooner in the relocation list are
194 * likely to be used more often than buffers which appear later
195 * in the list, the sort mustn't change the ordering of buffers
196 * with the same priority, i.e. it must be stable.
198 amdgpu_bo_list_for_each_entry(e
, list
) {
199 struct amdgpu_bo
*bo
= ttm_to_amdgpu_bo(e
->tv
.bo
);
200 unsigned priority
= e
->priority
;
203 list_add_tail(&e
->tv
.head
, &bucket
[priority
]);
205 e
->user_pages
= NULL
;
208 /* Connect the sorted buckets in the output list. */
209 for (i
= 0; i
< AMDGPU_BO_LIST_NUM_BUCKETS
; i
++)
210 list_splice(&bucket
[i
], validated
);
213 void amdgpu_bo_list_put(struct amdgpu_bo_list
*list
)
215 kref_put(&list
->refcount
, amdgpu_bo_list_free
);
218 int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in
*in
,
219 struct drm_amdgpu_bo_list_entry
**info_param
)
221 const void __user
*uptr
= u64_to_user_ptr(in
->bo_info_ptr
);
222 const uint32_t info_size
= sizeof(struct drm_amdgpu_bo_list_entry
);
223 struct drm_amdgpu_bo_list_entry
*info
;
226 info
= kvmalloc_array(in
->bo_number
, info_size
, GFP_KERNEL
);
230 /* copy the handle array from userspace to a kernel buffer */
232 if (likely(info_size
== in
->bo_info_size
)) {
233 unsigned long bytes
= in
->bo_number
*
236 if (copy_from_user(info
, uptr
, bytes
))
240 unsigned long bytes
= min(in
->bo_info_size
, info_size
);
243 memset(info
, 0, in
->bo_number
* info_size
);
244 for (i
= 0; i
< in
->bo_number
; ++i
) {
245 if (copy_from_user(&info
[i
], uptr
, bytes
))
248 uptr
+= in
->bo_info_size
;
260 int amdgpu_bo_list_ioctl(struct drm_device
*dev
, void *data
,
261 struct drm_file
*filp
)
263 struct amdgpu_device
*adev
= dev
->dev_private
;
264 struct amdgpu_fpriv
*fpriv
= filp
->driver_priv
;
265 union drm_amdgpu_bo_list
*args
= data
;
266 uint32_t handle
= args
->in
.list_handle
;
267 struct drm_amdgpu_bo_list_entry
*info
= NULL
;
268 struct amdgpu_bo_list
*list
, *old
;
271 r
= amdgpu_bo_create_list_entry_array(&args
->in
, &info
);
275 switch (args
->in
.operation
) {
276 case AMDGPU_BO_LIST_OP_CREATE
:
277 r
= amdgpu_bo_list_create(adev
, filp
, info
, args
->in
.bo_number
,
282 mutex_lock(&fpriv
->bo_list_lock
);
283 r
= idr_alloc(&fpriv
->bo_list_handles
, list
, 1, 0, GFP_KERNEL
);
284 mutex_unlock(&fpriv
->bo_list_lock
);
286 amdgpu_bo_list_put(list
);
293 case AMDGPU_BO_LIST_OP_DESTROY
:
294 amdgpu_bo_list_destroy(fpriv
, handle
);
298 case AMDGPU_BO_LIST_OP_UPDATE
:
299 r
= amdgpu_bo_list_create(adev
, filp
, info
, args
->in
.bo_number
,
304 mutex_lock(&fpriv
->bo_list_lock
);
305 old
= idr_replace(&fpriv
->bo_list_handles
, list
, handle
);
306 mutex_unlock(&fpriv
->bo_list_lock
);
309 amdgpu_bo_list_put(list
);
314 amdgpu_bo_list_put(old
);
322 memset(args
, 0, sizeof(*args
));
323 args
->out
.list_handle
= handle
;