[mirror_ubuntu-artful-kernel.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_amdkfd.c

/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#include "amdgpu_amdkfd.h"
#include "amd_shared.h"
#include <drm/drmP.h>
#include "amdgpu.h"
#include "amdgpu_gfx.h"
#include <linux/module.h>

const struct kfd2kgd_calls *kfd2kgd;
const struct kgd2kfd_calls *kgd2kfd;
bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);

int amdgpu_amdkfd_init(void)
{
	int ret;

#if defined(CONFIG_HSA_AMD_MODULE)
	int (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);

	kgd2kfd_init_p = symbol_request(kgd2kfd_init);

	if (kgd2kfd_init_p == NULL)
		return -ENOENT;

	ret = kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kgd2kfd);
	if (ret) {
		symbol_put(kgd2kfd_init);
		kgd2kfd = NULL;
	}

#elif defined(CONFIG_HSA_AMD)
	ret = kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd);
	if (ret)
		kgd2kfd = NULL;

#else
	ret = -ENOENT;
#endif

	return ret;
}

bool amdgpu_amdkfd_load_interface(struct amdgpu_device *adev)
{
	switch (adev->asic_type) {
#ifdef CONFIG_DRM_AMDGPU_CIK
	case CHIP_KAVERI:
		kfd2kgd = amdgpu_amdkfd_gfx_7_get_functions();
		break;
#endif
	case CHIP_CARRIZO:
		kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
		break;
	default:
		return false;
	}

	return true;
}

void amdgpu_amdkfd_fini(void)
{
	if (kgd2kfd) {
		kgd2kfd->exit();
		symbol_put(kgd2kfd_init);
	}
}

void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
{
	if (kgd2kfd)
		adev->kfd = kgd2kfd->probe((struct kgd_dev *)adev,
					adev->pdev, kfd2kgd);
}

void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
{
	int i;
	int last_valid_bit;
	if (adev->kfd) {
		struct kgd2kfd_shared_resources gpu_resources = {
			.compute_vmid_bitmap = 0xFF00,
			.num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec,
			.num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe
		};

		/* this is going to have a few of the MSBs set that we need to
		 * clear */
		bitmap_complement(gpu_resources.queue_bitmap,
				  adev->gfx.mec.queue_bitmap,
				  KGD_MAX_QUEUES);

		/* remove the KIQ bit as well */
		if (adev->gfx.kiq.ring.ready)
			clear_bit(amdgpu_gfx_queue_to_bit(adev,
							  adev->gfx.kiq.ring.me - 1,
							  adev->gfx.kiq.ring.pipe,
							  adev->gfx.kiq.ring.queue),
				  gpu_resources.queue_bitmap);

		/* According to linux/bitmap.h we shouldn't use bitmap_clear if
		 * nbits is not compile time constant */
		last_valid_bit = 1 /* only first MEC can have compute queues */
				* adev->gfx.mec.num_pipe_per_mec
				* adev->gfx.mec.num_queue_per_pipe;
		for (i = last_valid_bit; i < KGD_MAX_QUEUES; ++i)
			clear_bit(i, gpu_resources.queue_bitmap);

		amdgpu_doorbell_get_kfd_info(adev,
				&gpu_resources.doorbell_physical_address,
				&gpu_resources.doorbell_aperture_size,
				&gpu_resources.doorbell_start_offset);

		kgd2kfd->device_init(adev->kfd, &gpu_resources);
	}
}

void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev)
{
	if (adev->kfd) {
		kgd2kfd->device_exit(adev->kfd);
		adev->kfd = NULL;
	}
}

void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
		const void *ih_ring_entry)
{
	if (adev->kfd)
		kgd2kfd->interrupt(adev->kfd, ih_ring_entry);
}

void amdgpu_amdkfd_suspend(struct amdgpu_device *adev)
{
	if (adev->kfd)
		kgd2kfd->suspend(adev->kfd);
}

int amdgpu_amdkfd_resume(struct amdgpu_device *adev)
{
	int r = 0;

	if (adev->kfd)
		r = kgd2kfd->resume(adev->kfd);

	return r;
}

int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
			void **mem_obj, uint64_t *gpu_addr,
			void **cpu_ptr)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
	struct kgd_mem **mem = (struct kgd_mem **) mem_obj;
	int r;

	BUG_ON(kgd == NULL);
	BUG_ON(gpu_addr == NULL);
	BUG_ON(cpu_ptr == NULL);

	*mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL);
	if ((*mem) == NULL)
		return -ENOMEM;

	r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT,
			     AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, &(*mem)->bo);
	if (r) {
		dev_err(adev->dev,
			"failed to allocate BO for amdkfd (%d)\n", r);
		return r;
	}

	/* map the buffer */
	r = amdgpu_bo_reserve((*mem)->bo, true);
	if (r) {
		dev_err(adev->dev, "(%d) failed to reserve bo for amdkfd\n", r);
		goto allocate_mem_reserve_bo_failed;
	}

	r = amdgpu_bo_pin((*mem)->bo, AMDGPU_GEM_DOMAIN_GTT,
				&(*mem)->gpu_addr);
	if (r) {
		dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r);
		goto allocate_mem_pin_bo_failed;
	}
	*gpu_addr = (*mem)->gpu_addr;

	r = amdgpu_bo_kmap((*mem)->bo, &(*mem)->cpu_ptr);
	if (r) {
		dev_err(adev->dev,
			"(%d) failed to map bo to kernel for amdkfd\n", r);
		goto allocate_mem_kmap_bo_failed;
	}
	*cpu_ptr = (*mem)->cpu_ptr;

	amdgpu_bo_unreserve((*mem)->bo);

	return 0;

allocate_mem_kmap_bo_failed:
	amdgpu_bo_unpin((*mem)->bo);
allocate_mem_pin_bo_failed:
	amdgpu_bo_unreserve((*mem)->bo);
allocate_mem_reserve_bo_failed:
	amdgpu_bo_unref(&(*mem)->bo);

	return r;
}

void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
{
	struct kgd_mem *mem = (struct kgd_mem *) mem_obj;

	BUG_ON(mem == NULL);

	amdgpu_bo_reserve(mem->bo, true);
	amdgpu_bo_kunmap(mem->bo);
	amdgpu_bo_unpin(mem->bo);
	amdgpu_bo_unreserve(mem->bo);
	amdgpu_bo_unref(&(mem->bo));
	kfree(mem);
}

uint64_t get_vmem_size(struct kgd_dev *kgd)
{
	struct amdgpu_device *adev =
		(struct amdgpu_device *)kgd;

	BUG_ON(kgd == NULL);

	return adev->mc.real_vram_size;
}

uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;

	if (adev->gfx.funcs->get_gpu_clock_counter)
		return adev->gfx.funcs->get_gpu_clock_counter(adev);
	return 0;
}

uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;

	/* The sclk is in quantas of 10kHz */
	return adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
}
Commit	Line	Data
130e0371 OG	1	/*
	2	* Copyright 2014 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*/
	22
	23	#include "amdgpu_amdkfd.h"
2f7d10b3	24	#include "amd_shared.h"
130e0371 OG	25	#include <drm/drmP.h>
130e0371 OG	26	#include "amdgpu.h"
2db0cdbe	27	#include "amdgpu_gfx.h"
130e0371 OG	28	#include <linux/module.h>
	29
	30	const struct kfd2kgd_calls *kfd2kgd;
	31	const struct kgd2kfd_calls *kgd2kfd;
	32	bool (kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls*);
	33
efb1c658	34	int amdgpu_amdkfd_init(void)
130e0371	35	{
efb1c658 OG	36	int ret;
efb1c658 OG	37
130e0371	38	#if defined(CONFIG_HSA_AMD_MODULE)
efb1c658	39	int (kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls*);
130e0371 OG	40
	41	kgd2kfd_init_p = symbol_request(kgd2kfd_init);
	42
	43	if (kgd2kfd_init_p == NULL)
efb1c658 OG	44	return -ENOENT;
	45
	46	ret = kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kgd2kfd);
	47	if (ret) {
	48	symbol_put(kgd2kfd_init);
	49	kgd2kfd = NULL;
	50	}
	51
	52	#elif defined(CONFIG_HSA_AMD)
	53	ret = kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd);
	54	if (ret)
	55	kgd2kfd = NULL;
	56
	57	#else
	58	ret = -ENOENT;
130e0371	59	#endif
efb1c658 OG	60
efb1c658 OG	61	return ret;
130e0371 OG	62	}
130e0371 OG	63
dc102c43	64	bool amdgpu_amdkfd_load_interface(struct amdgpu_device *adev)
130e0371	65	{
dc102c43	66	switch (adev->asic_type) {
41548ef7	67	#ifdef CONFIG_DRM_AMDGPU_CIK
130e0371	68	case CHIP_KAVERI:
32c22e99 OG	69	kfd2kgd = amdgpu_amdkfd_gfx_7_get_functions();
32c22e99 OG	70	break;
41548ef7	71	#endif
ff758a12 BG	72	case CHIP_CARRIZO:
	73	kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
	74	break;
130e0371 OG	75	default:
	76	return false;
	77	}
	78
130e0371	79	return true;
130e0371 OG	80	}
	81
	82	void amdgpu_amdkfd_fini(void)
	83	{
	84	if (kgd2kfd) {
	85	kgd2kfd->exit();
	86	symbol_put(kgd2kfd_init);
	87	}
	88	}
	89
dc102c43	90	void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
130e0371 OG	91	{
130e0371 OG	92	if (kgd2kfd)
dc102c43 AR	93	adev->kfd = kgd2kfd->probe((struct kgd_dev *)adev,
dc102c43 AR	94	adev->pdev, kfd2kgd);
130e0371 OG	95	}
130e0371 OG	96
dc102c43	97	void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
130e0371	98	{
d0b63bb3 AR	99	int i;
d0b63bb3 AR	100	int last_valid_bit;
dc102c43	101	if (adev->kfd) {
130e0371 OG	102	struct kgd2kfd_shared_resources gpu_resources = {
130e0371 OG	103	.compute_vmid_bitmap = 0xFF00,
d0b63bb3 AR	104	.num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec,
d0b63bb3 AR	105	.num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe
130e0371 OG	106	};
130e0371 OG	107
d0b63bb3 AR	108	/* this is going to have a few of the MSBs set that we need to
	109	* clear */
	110	bitmap_complement(gpu_resources.queue_bitmap,
	111	adev->gfx.mec.queue_bitmap,
	112	KGD_MAX_QUEUES);
	113
7b2124a5 AR	114	/* remove the KIQ bit as well */
7b2124a5 AR	115	if (adev->gfx.kiq.ring.ready)
2db0cdbe AD	116	clear_bit(amdgpu_gfx_queue_to_bit(adev,
	117	adev->gfx.kiq.ring.me - 1,
	118	adev->gfx.kiq.ring.pipe,
	119	adev->gfx.kiq.ring.queue),
7b2124a5 AR	120	gpu_resources.queue_bitmap);
7b2124a5 AR	121
d0b63bb3 AR	122	/* According to linux/bitmap.h we shouldn't use bitmap_clear if
d0b63bb3 AR	123	* nbits is not compile time constant */
3447d220	124	last_valid_bit = 1 /* only first MEC can have compute queues */
d0b63bb3 AR	125	* adev->gfx.mec.num_pipe_per_mec
	126	* adev->gfx.mec.num_queue_per_pipe;
	127	for (i = last_valid_bit; i < KGD_MAX_QUEUES; ++i)
	128	clear_bit(i, gpu_resources.queue_bitmap);
	129
dc102c43	130	amdgpu_doorbell_get_kfd_info(adev,
130e0371 OG	131	&gpu_resources.doorbell_physical_address,
	132	&gpu_resources.doorbell_aperture_size,
	133	&gpu_resources.doorbell_start_offset);
	134
dc102c43	135	kgd2kfd->device_init(adev->kfd, &gpu_resources);
130e0371 OG	136	}
	137	}
	138
dc102c43	139	void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev)
130e0371	140	{
dc102c43 AR	141	if (adev->kfd) {
	142	kgd2kfd->device_exit(adev->kfd);
	143	adev->kfd = NULL;
130e0371 OG	144	}
	145	}
	146
dc102c43	147	void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
130e0371 OG	148	const void *ih_ring_entry)
130e0371 OG	149	{
dc102c43 AR	150	if (adev->kfd)
dc102c43 AR	151	kgd2kfd->interrupt(adev->kfd, ih_ring_entry);
130e0371 OG	152	}
130e0371 OG	153
dc102c43	154	void amdgpu_amdkfd_suspend(struct amdgpu_device *adev)
130e0371	155	{
dc102c43 AR	156	if (adev->kfd)
dc102c43 AR	157	kgd2kfd->suspend(adev->kfd);
130e0371 OG	158	}
130e0371 OG	159
dc102c43	160	int amdgpu_amdkfd_resume(struct amdgpu_device *adev)
130e0371 OG	161	{
	162	int r = 0;
	163
dc102c43 AR	164	if (adev->kfd)
dc102c43 AR	165	r = kgd2kfd->resume(adev->kfd);
130e0371 OG	166
	167	return r;
	168	}
	169
130e0371 OG	170	int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
	171	void *mem_obj, uint64_t gpu_addr,
	172	void **cpu_ptr)
	173	{
dc102c43	174	struct amdgpu_device adev = (struct amdgpu_device )kgd;
130e0371 OG	175	struct kgd_mem mem = (struct kgd_mem ) mem_obj;
	176	int r;
	177
	178	BUG_ON(kgd == NULL);
	179	BUG_ON(gpu_addr == NULL);
	180	BUG_ON(cpu_ptr == NULL);
	181
	182	*mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL);
	183	if ((*mem) == NULL)
	184	return -ENOMEM;
	185
dc102c43	186	r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT,
72d7668b	187	AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, &(*mem)->bo);
130e0371	188	if (r) {
dc102c43	189	dev_err(adev->dev,
130e0371 OG	190	"failed to allocate BO for amdkfd (%d)\n", r);
	191	return r;
	192	}
	193
	194	/* map the buffer */
	195	r = amdgpu_bo_reserve((*mem)->bo, true);
	196	if (r) {
dc102c43	197	dev_err(adev->dev, "(%d) failed to reserve bo for amdkfd\n", r);
130e0371 OG	198	goto allocate_mem_reserve_bo_failed;
	199	}
	200
	201	r = amdgpu_bo_pin((*mem)->bo, AMDGPU_GEM_DOMAIN_GTT,
	202	&(*mem)->gpu_addr);
	203	if (r) {
dc102c43	204	dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r);
130e0371 OG	205	goto allocate_mem_pin_bo_failed;
	206	}
	207	gpu_addr = (mem)->gpu_addr;
	208
	209	r = amdgpu_bo_kmap((mem)->bo, &(mem)->cpu_ptr);
	210	if (r) {
dc102c43	211	dev_err(adev->dev,
130e0371 OG	212	"(%d) failed to map bo to kernel for amdkfd\n", r);
	213	goto allocate_mem_kmap_bo_failed;
	214	}
	215	cpu_ptr = (mem)->cpu_ptr;
	216
	217	amdgpu_bo_unreserve((*mem)->bo);
	218
	219	return 0;
	220
	221	allocate_mem_kmap_bo_failed:
	222	amdgpu_bo_unpin((*mem)->bo);
	223	allocate_mem_pin_bo_failed:
	224	amdgpu_bo_unreserve((*mem)->bo);
	225	allocate_mem_reserve_bo_failed:
	226	amdgpu_bo_unref(&(*mem)->bo);
	227
	228	return r;
	229	}
	230
	231	void free_gtt_mem(struct kgd_dev kgd, void mem_obj)
	232	{
	233	struct kgd_mem mem = (struct kgd_mem ) mem_obj;
	234
	235	BUG_ON(mem == NULL);
	236
	237	amdgpu_bo_reserve(mem->bo, true);
	238	amdgpu_bo_kunmap(mem->bo);
	239	amdgpu_bo_unpin(mem->bo);
	240	amdgpu_bo_unreserve(mem->bo);
	241	amdgpu_bo_unref(&(mem->bo));
	242	kfree(mem);
	243	}
	244
	245	uint64_t get_vmem_size(struct kgd_dev *kgd)
	246	{
dc102c43	247	struct amdgpu_device *adev =
130e0371 OG	248	(struct amdgpu_device *)kgd;
	249
	250	BUG_ON(kgd == NULL);
	251
dc102c43	252	return adev->mc.real_vram_size;
130e0371 OG	253	}
	254
	255	uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
	256	{
dc102c43	257	struct amdgpu_device adev = (struct amdgpu_device )kgd;
130e0371	258
dc102c43 AR	259	if (adev->gfx.funcs->get_gpu_clock_counter)
dc102c43 AR	260	return adev->gfx.funcs->get_gpu_clock_counter(adev);
130e0371 OG	261	return 0;
	262	}
	263
	264	uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
	265	{
dc102c43	266	struct amdgpu_device adev = (struct amdgpu_device )kgd;
130e0371 OG	267
130e0371 OG	268	/* The sclk is in quantas of 10kHz */
dc102c43	269	return adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
130e0371	270	}