2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
28 #include <linux/irq.h>
30 #include <drm/drm_crtc_helper.h>
31 #include <drm/amdgpu_drm.h>
33 #include "amdgpu_ih.h"
35 #include "amdgpu_connectors.h"
36 #include "amdgpu_trace.h"
38 #include <linux/pm_runtime.h>
40 #ifdef CONFIG_DRM_AMD_DC
41 #include "amdgpu_dm_irq.h"
44 #define AMDGPU_WAIT_IDLE_TIMEOUT 200
47 * Handle hotplug events outside the interrupt handler proper.
50 * amdgpu_hotplug_work_func - display hotplug work handler
54 * This is the hot plug event work handler (all asics).
55 * The work gets scheduled from the irq handler if there
56 * was a hot plug interrupt. It walks the connector table
57 * and calls the hotplug handler for each one, then sends
58 * a drm hotplug event to alert userspace.
60 static void amdgpu_hotplug_work_func(struct work_struct
*work
)
62 struct amdgpu_device
*adev
= container_of(work
, struct amdgpu_device
,
64 struct drm_device
*dev
= adev
->ddev
;
65 struct drm_mode_config
*mode_config
= &dev
->mode_config
;
66 struct drm_connector
*connector
;
68 mutex_lock(&mode_config
->mutex
);
69 list_for_each_entry(connector
, &mode_config
->connector_list
, head
)
70 amdgpu_connector_hotplug(connector
);
71 mutex_unlock(&mode_config
->mutex
);
72 /* Just fire off a uevent and let userspace tell us what to do */
73 drm_helper_hpd_irq_event(dev
);
77 * amdgpu_irq_reset_work_func - execute gpu reset
81 * Execute scheduled gpu reset (cayman+).
82 * This function is called when the irq handler
83 * thinks we need a gpu reset.
85 static void amdgpu_irq_reset_work_func(struct work_struct
*work
)
87 struct amdgpu_device
*adev
= container_of(work
, struct amdgpu_device
,
90 if (!amdgpu_sriov_vf(adev
))
91 amdgpu_gpu_reset(adev
);
94 /* Disable *all* interrupts */
95 static void amdgpu_irq_disable_all(struct amdgpu_device
*adev
)
97 unsigned long irqflags
;
101 spin_lock_irqsave(&adev
->irq
.lock
, irqflags
);
102 for (i
= 0; i
< AMDGPU_IH_CLIENTID_MAX
; ++i
) {
103 if (!adev
->irq
.client
[i
].sources
)
106 for (j
= 0; j
< AMDGPU_MAX_IRQ_SRC_ID
; ++j
) {
107 struct amdgpu_irq_src
*src
= adev
->irq
.client
[i
].sources
[j
];
109 if (!src
|| !src
->funcs
->set
|| !src
->num_types
)
112 for (k
= 0; k
< src
->num_types
; ++k
) {
113 atomic_set(&src
->enabled_types
[k
], 0);
114 r
= src
->funcs
->set(adev
, src
, k
,
115 AMDGPU_IRQ_STATE_DISABLE
);
117 DRM_ERROR("error disabling interrupt (%d)\n",
122 spin_unlock_irqrestore(&adev
->irq
.lock
, irqflags
);
126 * amdgpu_irq_preinstall - drm irq preinstall callback
128 * @dev: drm dev pointer
130 * Gets the hw ready to enable irqs (all asics).
131 * This function disables all interrupt sources on the GPU.
133 void amdgpu_irq_preinstall(struct drm_device
*dev
)
135 struct amdgpu_device
*adev
= dev
->dev_private
;
137 /* Disable *all* interrupts */
138 amdgpu_irq_disable_all(adev
);
140 amdgpu_ih_process(adev
);
144 * amdgpu_irq_postinstall - drm irq preinstall callback
146 * @dev: drm dev pointer
148 * Handles stuff to be done after enabling irqs (all asics).
149 * Returns 0 on success.
151 int amdgpu_irq_postinstall(struct drm_device
*dev
)
153 dev
->max_vblank_count
= 0x00ffffff;
158 * amdgpu_irq_uninstall - drm irq uninstall callback
160 * @dev: drm dev pointer
162 * This function disables all interrupt sources on the GPU (all asics).
164 void amdgpu_irq_uninstall(struct drm_device
*dev
)
166 struct amdgpu_device
*adev
= dev
->dev_private
;
171 amdgpu_irq_disable_all(adev
);
175 * amdgpu_irq_handler - irq handler
177 * @int irq, void *arg: args
179 * This is the irq handler for the amdgpu driver (all asics).
181 irqreturn_t
amdgpu_irq_handler(int irq
, void *arg
)
183 struct drm_device
*dev
= (struct drm_device
*) arg
;
184 struct amdgpu_device
*adev
= dev
->dev_private
;
187 ret
= amdgpu_ih_process(adev
);
188 if (ret
== IRQ_HANDLED
)
189 pm_runtime_mark_last_busy(dev
->dev
);
194 * amdgpu_msi_ok - asic specific msi checks
196 * @adev: amdgpu device pointer
198 * Handles asic specific MSI checks to determine if
199 * MSIs should be enabled on a particular chip (all asics).
200 * Returns true if MSIs should be enabled, false if MSIs
201 * should not be enabled.
203 static bool amdgpu_msi_ok(struct amdgpu_device
*adev
)
208 else if (amdgpu_msi
== 0)
215 * amdgpu_irq_init - init driver interrupt info
217 * @adev: amdgpu device pointer
219 * Sets up the work irq handlers, vblank init, MSIs, etc. (all asics).
220 * Returns 0 for success, error for failure.
222 int amdgpu_irq_init(struct amdgpu_device
*adev
)
226 spin_lock_init(&adev
->irq
.lock
);
229 adev
->irq
.msi_enabled
= false;
231 if (amdgpu_msi_ok(adev
)) {
232 int ret
= pci_enable_msi(adev
->pdev
);
234 adev
->irq
.msi_enabled
= true;
235 dev_info(adev
->dev
, "amdgpu: using MSI.\n");
239 if (!amdgpu_device_has_dc_support(adev
)) {
240 if (!adev
->enable_virtual_display
)
241 /* Disable vblank irqs aggressively for power-saving */
242 /* XXX: can this be enabled for DC? */
243 adev
->ddev
->vblank_disable_immediate
= true;
245 r
= drm_vblank_init(adev
->ddev
, adev
->mode_info
.num_crtc
);
250 INIT_WORK(&adev
->hotplug_work
,
251 amdgpu_hotplug_work_func
);
254 INIT_WORK(&adev
->reset_work
, amdgpu_irq_reset_work_func
);
256 adev
->irq
.installed
= true;
257 r
= drm_irq_install(adev
->ddev
, adev
->ddev
->pdev
->irq
);
259 adev
->irq
.installed
= false;
260 flush_work(&adev
->hotplug_work
);
261 cancel_work_sync(&adev
->reset_work
);
265 DRM_INFO("amdgpu: irq initialized.\n");
270 * amdgpu_irq_fini - tear down driver interrupt info
272 * @adev: amdgpu device pointer
274 * Tears down the work irq handlers, vblank handlers, MSIs, etc. (all asics).
276 void amdgpu_irq_fini(struct amdgpu_device
*adev
)
280 if (adev
->irq
.installed
) {
281 drm_irq_uninstall(adev
->ddev
);
282 adev
->irq
.installed
= false;
283 if (adev
->irq
.msi_enabled
)
284 pci_disable_msi(adev
->pdev
);
285 flush_work(&adev
->hotplug_work
);
286 cancel_work_sync(&adev
->reset_work
);
289 for (i
= 0; i
< AMDGPU_IH_CLIENTID_MAX
; ++i
) {
290 if (!adev
->irq
.client
[i
].sources
)
293 for (j
= 0; j
< AMDGPU_MAX_IRQ_SRC_ID
; ++j
) {
294 struct amdgpu_irq_src
*src
= adev
->irq
.client
[i
].sources
[j
];
299 kfree(src
->enabled_types
);
300 src
->enabled_types
= NULL
;
304 adev
->irq
.client
[i
].sources
[j
] = NULL
;
307 kfree(adev
->irq
.client
[i
].sources
);
312 * amdgpu_irq_add_id - register irq source
314 * @adev: amdgpu device pointer
315 * @src_id: source id for this source
316 * @source: irq source
319 int amdgpu_irq_add_id(struct amdgpu_device
*adev
,
320 unsigned client_id
, unsigned src_id
,
321 struct amdgpu_irq_src
*source
)
323 if (client_id
>= AMDGPU_IH_CLIENTID_MAX
)
326 if (src_id
>= AMDGPU_MAX_IRQ_SRC_ID
)
332 if (!adev
->irq
.client
[client_id
].sources
) {
333 adev
->irq
.client
[client_id
].sources
=
334 kcalloc(AMDGPU_MAX_IRQ_SRC_ID
,
335 sizeof(struct amdgpu_irq_src
*),
337 if (!adev
->irq
.client
[client_id
].sources
)
341 if (adev
->irq
.client
[client_id
].sources
[src_id
] != NULL
)
344 if (source
->num_types
&& !source
->enabled_types
) {
347 types
= kcalloc(source
->num_types
, sizeof(atomic_t
),
352 source
->enabled_types
= types
;
355 adev
->irq
.client
[client_id
].sources
[src_id
] = source
;
360 * amdgpu_irq_dispatch - dispatch irq to IP blocks
362 * @adev: amdgpu device pointer
363 * @entry: interrupt vector
365 * Dispatches the irq to the different IP blocks
367 void amdgpu_irq_dispatch(struct amdgpu_device
*adev
,
368 struct amdgpu_iv_entry
*entry
)
370 unsigned client_id
= entry
->client_id
;
371 unsigned src_id
= entry
->src_id
;
372 struct amdgpu_irq_src
*src
;
375 trace_amdgpu_iv(entry
);
377 if (client_id
>= AMDGPU_IH_CLIENTID_MAX
) {
378 DRM_DEBUG("Invalid client_id in IV: %d\n", client_id
);
382 if (src_id
>= AMDGPU_MAX_IRQ_SRC_ID
) {
383 DRM_DEBUG("Invalid src_id in IV: %d\n", src_id
);
387 if (adev
->irq
.virq
[src_id
]) {
388 generic_handle_irq(irq_find_mapping(adev
->irq
.domain
, src_id
));
390 if (!adev
->irq
.client
[client_id
].sources
) {
391 DRM_DEBUG("Unregistered interrupt client_id: %d src_id: %d\n",
396 src
= adev
->irq
.client
[client_id
].sources
[src_id
];
398 DRM_DEBUG("Unhandled interrupt src_id: %d\n", src_id
);
402 r
= src
->funcs
->process(adev
, src
, entry
);
404 DRM_ERROR("error processing interrupt (%d)\n", r
);
409 * amdgpu_irq_update - update hw interrupt state
411 * @adev: amdgpu device pointer
412 * @src: interrupt src you want to enable
413 * @type: type of interrupt you want to update
415 * Updates the interrupt state for a specific src (all asics).
417 int amdgpu_irq_update(struct amdgpu_device
*adev
,
418 struct amdgpu_irq_src
*src
, unsigned type
)
420 unsigned long irqflags
;
421 enum amdgpu_interrupt_state state
;
424 spin_lock_irqsave(&adev
->irq
.lock
, irqflags
);
426 /* we need to determine after taking the lock, otherwise
427 we might disable just enabled interrupts again */
428 if (amdgpu_irq_enabled(adev
, src
, type
))
429 state
= AMDGPU_IRQ_STATE_ENABLE
;
431 state
= AMDGPU_IRQ_STATE_DISABLE
;
433 r
= src
->funcs
->set(adev
, src
, type
, state
);
434 spin_unlock_irqrestore(&adev
->irq
.lock
, irqflags
);
438 void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device
*adev
)
442 for (i
= 0; i
< AMDGPU_IH_CLIENTID_MAX
; ++i
) {
443 if (!adev
->irq
.client
[i
].sources
)
446 for (j
= 0; j
< AMDGPU_MAX_IRQ_SRC_ID
; ++j
) {
447 struct amdgpu_irq_src
*src
= adev
->irq
.client
[i
].sources
[j
];
451 for (k
= 0; k
< src
->num_types
; k
++)
452 amdgpu_irq_update(adev
, src
, k
);
458 * amdgpu_irq_get - enable interrupt
460 * @adev: amdgpu device pointer
461 * @src: interrupt src you want to enable
462 * @type: type of interrupt you want to enable
464 * Enables the interrupt type for a specific src (all asics).
466 int amdgpu_irq_get(struct amdgpu_device
*adev
, struct amdgpu_irq_src
*src
,
469 if (!adev
->ddev
->irq_enabled
)
472 if (type
>= src
->num_types
)
475 if (!src
->enabled_types
|| !src
->funcs
->set
)
478 if (atomic_inc_return(&src
->enabled_types
[type
]) == 1)
479 return amdgpu_irq_update(adev
, src
, type
);
485 * amdgpu_irq_put - disable interrupt
487 * @adev: amdgpu device pointer
488 * @src: interrupt src you want to disable
489 * @type: type of interrupt you want to disable
491 * Disables the interrupt type for a specific src (all asics).
493 int amdgpu_irq_put(struct amdgpu_device
*adev
, struct amdgpu_irq_src
*src
,
496 if (!adev
->ddev
->irq_enabled
)
499 if (type
>= src
->num_types
)
502 if (!src
->enabled_types
|| !src
->funcs
->set
)
505 if (atomic_dec_and_test(&src
->enabled_types
[type
]))
506 return amdgpu_irq_update(adev
, src
, type
);
512 * amdgpu_irq_enabled - test if irq is enabled or not
514 * @adev: amdgpu device pointer
515 * @idx: interrupt src you want to test
517 * Tests if the given interrupt source is enabled or not
519 bool amdgpu_irq_enabled(struct amdgpu_device
*adev
, struct amdgpu_irq_src
*src
,
522 if (!adev
->ddev
->irq_enabled
)
525 if (type
>= src
->num_types
)
528 if (!src
->enabled_types
|| !src
->funcs
->set
)
531 return !!atomic_read(&src
->enabled_types
[type
]);
535 static void amdgpu_irq_mask(struct irq_data
*irqd
)
540 static void amdgpu_irq_unmask(struct irq_data
*irqd
)
545 static struct irq_chip amdgpu_irq_chip
= {
547 .irq_mask
= amdgpu_irq_mask
,
548 .irq_unmask
= amdgpu_irq_unmask
,
551 static int amdgpu_irqdomain_map(struct irq_domain
*d
,
552 unsigned int irq
, irq_hw_number_t hwirq
)
554 if (hwirq
>= AMDGPU_MAX_IRQ_SRC_ID
)
557 irq_set_chip_and_handler(irq
,
558 &amdgpu_irq_chip
, handle_simple_irq
);
562 static const struct irq_domain_ops amdgpu_hw_irqdomain_ops
= {
563 .map
= amdgpu_irqdomain_map
,
567 * amdgpu_irq_add_domain - create a linear irq domain
569 * @adev: amdgpu device pointer
571 * Create an irq domain for GPU interrupt sources
572 * that may be driven by another driver (e.g., ACP).
574 int amdgpu_irq_add_domain(struct amdgpu_device
*adev
)
576 adev
->irq
.domain
= irq_domain_add_linear(NULL
, AMDGPU_MAX_IRQ_SRC_ID
,
577 &amdgpu_hw_irqdomain_ops
, adev
);
578 if (!adev
->irq
.domain
) {
579 DRM_ERROR("GPU irq add domain failed\n");
587 * amdgpu_irq_remove_domain - remove the irq domain
589 * @adev: amdgpu device pointer
591 * Remove the irq domain for GPU interrupt sources
592 * that may be driven by another driver (e.g., ACP).
594 void amdgpu_irq_remove_domain(struct amdgpu_device
*adev
)
596 if (adev
->irq
.domain
) {
597 irq_domain_remove(adev
->irq
.domain
);
598 adev
->irq
.domain
= NULL
;
603 * amdgpu_irq_create_mapping - create a mapping between a domain irq and a
606 * @adev: amdgpu device pointer
607 * @src_id: IH source id
609 * Create a mapping between a domain irq (GPU IH src id) and a Linux irq
610 * Use this for components that generate a GPU interrupt, but are driven
611 * by a different driver (e.g., ACP).
612 * Returns the Linux irq.
614 unsigned amdgpu_irq_create_mapping(struct amdgpu_device
*adev
, unsigned src_id
)
616 adev
->irq
.virq
[src_id
] = irq_create_mapping(adev
->irq
.domain
, src_id
);
618 return adev
->irq
.virq
[src_id
];