2 * Copyright 2018 Advanced Micro Devices, Inc.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
27 #include <linux/io-64-nonatomic-lo-hi.h>
32 * amdgpu_gmc_get_pde_for_bo - get the PDE for a BO
34 * @bo: the BO to get the PDE for
35 * @level: the level in the PD hirarchy
36 * @addr: resulting addr
37 * @flags: resulting flags
39 * Get the address and flags to be used for a PDE (Page Directory Entry).
41 void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo
*bo
, int level
,
42 uint64_t *addr
, uint64_t *flags
)
44 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
45 struct ttm_dma_tt
*ttm
;
47 switch (bo
->tbo
.mem
.mem_type
) {
49 ttm
= container_of(bo
->tbo
.ttm
, struct ttm_dma_tt
, ttm
);
50 *addr
= ttm
->dma_address
[0];
53 *addr
= amdgpu_bo_gpu_offset(bo
);
59 *flags
= amdgpu_ttm_tt_pde_flags(bo
->tbo
.ttm
, &bo
->tbo
.mem
);
60 amdgpu_gmc_get_vm_pde(adev
, level
, addr
, flags
);
64 * amdgpu_gmc_pd_addr - return the address of the root directory
67 uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo
*bo
)
69 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
72 /* TODO: move that into ASIC specific code */
73 if (adev
->asic_type
>= CHIP_VEGA10
) {
74 uint64_t flags
= AMDGPU_PTE_VALID
;
76 amdgpu_gmc_get_pde_for_bo(bo
, -1, &pd_addr
, &flags
);
79 pd_addr
= amdgpu_bo_gpu_offset(bo
);
85 * amdgpu_gmc_set_pte_pde - update the page tables using CPU
87 * @adev: amdgpu_device pointer
88 * @cpu_pt_addr: cpu address of the page table
89 * @gpu_page_idx: entry in the page table to update
90 * @addr: dst addr to write into pte/pde
91 * @flags: access flags
93 * Update the page tables using CPU.
95 int amdgpu_gmc_set_pte_pde(struct amdgpu_device
*adev
, void *cpu_pt_addr
,
96 uint32_t gpu_page_idx
, uint64_t addr
,
99 void __iomem
*ptr
= (void *)cpu_pt_addr
;
103 * The following is for PTE only. GART does not have PDEs.
105 value
= addr
& 0x0000FFFFFFFFF000ULL
;
107 writeq(value
, ptr
+ (gpu_page_idx
* 8));
112 * amdgpu_gmc_agp_addr - return the address in the AGP address space
114 * @tbo: TTM BO which needs the address, must be in GTT domain
116 * Tries to figure out how to access the BO through the AGP aperture. Returns
117 * AMDGPU_BO_INVALID_OFFSET if that is not possible.
119 uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object
*bo
)
121 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->bdev
);
122 struct ttm_dma_tt
*ttm
;
124 if (bo
->num_pages
!= 1 || bo
->ttm
->caching_state
== tt_cached
)
125 return AMDGPU_BO_INVALID_OFFSET
;
127 ttm
= container_of(bo
->ttm
, struct ttm_dma_tt
, ttm
);
128 if (ttm
->dma_address
[0] + PAGE_SIZE
>= adev
->gmc
.agp_size
)
129 return AMDGPU_BO_INVALID_OFFSET
;
131 return adev
->gmc
.agp_start
+ ttm
->dma_address
[0];
135 * amdgpu_gmc_vram_location - try to find VRAM location
137 * @adev: amdgpu device structure holding all necessary informations
138 * @mc: memory controller structure holding memory informations
139 * @base: base address at which to put VRAM
141 * Function will try to place VRAM at base address provided
144 void amdgpu_gmc_vram_location(struct amdgpu_device
*adev
, struct amdgpu_gmc
*mc
,
147 uint64_t limit
= (uint64_t)amdgpu_vram_limit
<< 20;
149 mc
->vram_start
= base
;
150 mc
->vram_end
= mc
->vram_start
+ mc
->mc_vram_size
- 1;
151 if (limit
&& limit
< mc
->real_vram_size
)
152 mc
->real_vram_size
= limit
;
154 if (mc
->xgmi
.num_physical_nodes
== 0) {
155 mc
->fb_start
= mc
->vram_start
;
156 mc
->fb_end
= mc
->vram_end
;
158 dev_info(adev
->dev
, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
159 mc
->mc_vram_size
>> 20, mc
->vram_start
,
160 mc
->vram_end
, mc
->real_vram_size
>> 20);
164 * amdgpu_gmc_gart_location - try to find GART location
166 * @adev: amdgpu device structure holding all necessary informations
167 * @mc: memory controller structure holding memory informations
169 * Function will place try to place GART before or after VRAM.
171 * If GART size is bigger than space left then we ajust GART size.
172 * Thus function will never fails.
174 void amdgpu_gmc_gart_location(struct amdgpu_device
*adev
, struct amdgpu_gmc
*mc
)
176 const uint64_t four_gb
= 0x100000000ULL
;
177 u64 size_af
, size_bf
;
178 /*To avoid the hole, limit the max mc address to AMDGPU_GMC_HOLE_START*/
179 u64 max_mc_address
= min(adev
->gmc
.mc_mask
, AMDGPU_GMC_HOLE_START
- 1);
181 mc
->gart_size
+= adev
->pm
.smu_prv_buffer_size
;
183 /* VCE doesn't like it when BOs cross a 4GB segment, so align
184 * the GART base on a 4GB boundary as well.
186 size_bf
= mc
->fb_start
;
187 size_af
= max_mc_address
+ 1 - ALIGN(mc
->fb_end
+ 1, four_gb
);
189 if (mc
->gart_size
> max(size_bf
, size_af
)) {
190 dev_warn(adev
->dev
, "limiting GART\n");
191 mc
->gart_size
= max(size_bf
, size_af
);
194 if ((size_bf
>= mc
->gart_size
&& size_bf
< size_af
) ||
195 (size_af
< mc
->gart_size
))
198 mc
->gart_start
= max_mc_address
- mc
->gart_size
+ 1;
200 mc
->gart_start
&= ~(four_gb
- 1);
201 mc
->gart_end
= mc
->gart_start
+ mc
->gart_size
- 1;
202 dev_info(adev
->dev
, "GART: %lluM 0x%016llX - 0x%016llX\n",
203 mc
->gart_size
>> 20, mc
->gart_start
, mc
->gart_end
);
207 * amdgpu_gmc_agp_location - try to find AGP location
208 * @adev: amdgpu device structure holding all necessary informations
209 * @mc: memory controller structure holding memory informations
211 * Function will place try to find a place for the AGP BAR in the MC address
214 * AGP BAR will be assigned the largest available hole in the address space.
215 * Should be called after VRAM and GART locations are setup.
217 void amdgpu_gmc_agp_location(struct amdgpu_device
*adev
, struct amdgpu_gmc
*mc
)
219 const uint64_t sixteen_gb
= 1ULL << 34;
220 const uint64_t sixteen_gb_mask
= ~(sixteen_gb
- 1);
221 u64 size_af
, size_bf
;
223 if (mc
->fb_start
> mc
->gart_start
) {
224 size_bf
= (mc
->fb_start
& sixteen_gb_mask
) -
225 ALIGN(mc
->gart_end
+ 1, sixteen_gb
);
226 size_af
= mc
->mc_mask
+ 1 - ALIGN(mc
->fb_end
+ 1, sixteen_gb
);
228 size_bf
= mc
->fb_start
& sixteen_gb_mask
;
229 size_af
= (mc
->gart_start
& sixteen_gb_mask
) -
230 ALIGN(mc
->fb_end
+ 1, sixteen_gb
);
233 if (size_bf
> size_af
) {
234 mc
->agp_start
= (mc
->fb_start
- size_bf
) & sixteen_gb_mask
;
235 mc
->agp_size
= size_bf
;
237 mc
->agp_start
= ALIGN(mc
->fb_end
+ 1, sixteen_gb
);
238 mc
->agp_size
= size_af
;
241 mc
->agp_end
= mc
->agp_start
+ mc
->agp_size
- 1;
242 dev_info(adev
->dev
, "AGP: %lluM 0x%016llX - 0x%016llX\n",
243 mc
->agp_size
>> 20, mc
->agp_start
, mc
->agp_end
);
247 * amdgpu_gmc_filter_faults - filter VM faults
249 * @adev: amdgpu device structure
250 * @addr: address of the VM fault
251 * @pasid: PASID of the process causing the fault
252 * @timestamp: timestamp of the fault
255 * True if the fault was filtered and should not be processed further.
256 * False if the fault is a new one and needs to be handled.
258 bool amdgpu_gmc_filter_faults(struct amdgpu_device
*adev
, uint64_t addr
,
259 uint16_t pasid
, uint64_t timestamp
)
261 struct amdgpu_gmc
*gmc
= &adev
->gmc
;
263 uint64_t stamp
, key
= addr
<< 4 | pasid
;
264 struct amdgpu_gmc_fault
*fault
;
267 /* If we don't have space left in the ring buffer return immediately */
268 stamp
= max(timestamp
, AMDGPU_GMC_FAULT_TIMEOUT
+ 1) -
269 AMDGPU_GMC_FAULT_TIMEOUT
;
270 if (gmc
->fault_ring
[gmc
->last_fault
].timestamp
>= stamp
)
273 /* Try to find the fault in the hash */
274 hash
= hash_64(key
, AMDGPU_GMC_FAULT_HASH_ORDER
);
275 fault
= &gmc
->fault_ring
[gmc
->fault_hash
[hash
].idx
];
276 while (fault
->timestamp
>= stamp
) {
279 if (fault
->key
== key
)
282 tmp
= fault
->timestamp
;
283 fault
= &gmc
->fault_ring
[fault
->next
];
285 /* Check if the entry was reused */
286 if (fault
->timestamp
>= tmp
)
290 /* Add the fault to the ring */
291 fault
= &gmc
->fault_ring
[gmc
->last_fault
];
293 fault
->timestamp
= timestamp
;
295 /* And update the hash */
296 fault
->next
= gmc
->fault_hash
[hash
].idx
;
297 gmc
->fault_hash
[hash
].idx
= gmc
->last_fault
++;