2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include "intel_frontbuffer.h"
27 #include "i915_gem_clflush.h"
29 static DEFINE_SPINLOCK(clflush_lock
);
32 struct dma_fence dma
; /* Must be first for dma_fence_free() */
33 struct i915_sw_fence wait
;
34 struct work_struct work
;
35 struct drm_i915_gem_object
*obj
;
38 static const char *i915_clflush_get_driver_name(struct dma_fence
*fence
)
43 static const char *i915_clflush_get_timeline_name(struct dma_fence
*fence
)
48 static bool i915_clflush_enable_signaling(struct dma_fence
*fence
)
53 static void i915_clflush_release(struct dma_fence
*fence
)
55 struct clflush
*clflush
= container_of(fence
, typeof(*clflush
), dma
);
57 i915_sw_fence_fini(&clflush
->wait
);
59 BUILD_BUG_ON(offsetof(typeof(*clflush
), dma
));
60 dma_fence_free(&clflush
->dma
);
63 static const struct dma_fence_ops i915_clflush_ops
= {
64 .get_driver_name
= i915_clflush_get_driver_name
,
65 .get_timeline_name
= i915_clflush_get_timeline_name
,
66 .enable_signaling
= i915_clflush_enable_signaling
,
67 .wait
= dma_fence_default_wait
,
68 .release
= i915_clflush_release
,
71 static void __i915_do_clflush(struct drm_i915_gem_object
*obj
)
73 drm_clflush_sg(obj
->mm
.pages
);
74 intel_fb_obj_flush(obj
, ORIGIN_CPU
);
77 static void i915_clflush_work(struct work_struct
*work
)
79 struct clflush
*clflush
= container_of(work
, typeof(*clflush
), work
);
80 struct drm_i915_gem_object
*obj
= clflush
->obj
;
82 if (i915_gem_object_pin_pages(obj
)) {
83 DRM_ERROR("Failed to acquire obj->pages for clflushing\n");
87 __i915_do_clflush(obj
);
89 i915_gem_object_unpin_pages(obj
);
92 i915_gem_object_put(obj
);
94 dma_fence_signal(&clflush
->dma
);
95 dma_fence_put(&clflush
->dma
);
98 static int __i915_sw_fence_call
99 i915_clflush_notify(struct i915_sw_fence
*fence
,
100 enum i915_sw_fence_notify state
)
102 struct clflush
*clflush
= container_of(fence
, typeof(*clflush
), wait
);
106 schedule_work(&clflush
->work
);
110 dma_fence_put(&clflush
->dma
);
117 bool i915_gem_clflush_object(struct drm_i915_gem_object
*obj
,
120 struct clflush
*clflush
;
123 * Stolen memory is always coherent with the GPU as it is explicitly
124 * marked as wc by the system, or the system is cache-coherent.
125 * Similarly, we only access struct pages through the CPU cache, so
126 * anything not backed by physical memory we consider to be always
127 * coherent and not need clflushing.
129 if (!i915_gem_object_has_struct_page(obj
)) {
130 obj
->cache_dirty
= false;
134 /* If the GPU is snooping the contents of the CPU cache,
135 * we do not need to manually clear the CPU cache lines. However,
136 * the caches are only snooped when the render cache is
137 * flushed/invalidated. As we always have to emit invalidations
138 * and flushes when moving into and out of the RENDER domain, correct
139 * snooping behaviour occurs naturally as the result of our domain
142 if (!(flags
& I915_CLFLUSH_FORCE
) && obj
->cache_coherent
)
145 trace_i915_gem_object_clflush(obj
);
148 if (!(flags
& I915_CLFLUSH_SYNC
))
149 clflush
= kmalloc(sizeof(*clflush
), GFP_KERNEL
);
151 GEM_BUG_ON(!obj
->cache_dirty
);
153 dma_fence_init(&clflush
->dma
,
156 to_i915(obj
->base
.dev
)->mm
.unordered_timeline
,
158 i915_sw_fence_init(&clflush
->wait
, i915_clflush_notify
);
160 clflush
->obj
= i915_gem_object_get(obj
);
161 INIT_WORK(&clflush
->work
, i915_clflush_work
);
163 dma_fence_get(&clflush
->dma
);
165 i915_sw_fence_await_reservation(&clflush
->wait
,
167 true, I915_FENCE_TIMEOUT
,
170 reservation_object_lock(obj
->resv
, NULL
);
171 reservation_object_add_excl_fence(obj
->resv
, &clflush
->dma
);
172 reservation_object_unlock(obj
->resv
);
174 i915_sw_fence_commit(&clflush
->wait
);
175 } else if (obj
->mm
.pages
) {
176 __i915_do_clflush(obj
);
178 GEM_BUG_ON(obj
->base
.write_domain
!= I915_GEM_DOMAIN_CPU
);
181 obj
->cache_dirty
= false;