]>
Commit | Line | Data |
---|---|---|
e8635b48 DD |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
1aa2b278 | 6 | * Copyright (C) 2005-2009, 2010 Cavium Networks |
e8635b48 DD |
7 | */ |
8 | #include <linux/kernel.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/msi.h> | |
11 | #include <linux/spinlock.h> | |
12 | #include <linux/interrupt.h> | |
13 | ||
14 | #include <asm/octeon/octeon.h> | |
15 | #include <asm/octeon/cvmx-npi-defs.h> | |
16 | #include <asm/octeon/cvmx-pci-defs.h> | |
17 | #include <asm/octeon/cvmx-npei-defs.h> | |
18 | #include <asm/octeon/cvmx-pexp-defs.h> | |
01a6221a | 19 | #include <asm/octeon/pci-octeon.h> |
e8635b48 DD |
20 | |
21 | /* | |
22 | * Each bit in msi_free_irq_bitmask represents a MSI interrupt that is | |
23 | * in use. | |
24 | */ | |
1aa2b278 | 25 | static u64 msi_free_irq_bitmask[4]; |
e8635b48 DD |
26 | |
27 | /* | |
28 | * Each bit in msi_multiple_irq_bitmask tells that the device using | |
29 | * this bit in msi_free_irq_bitmask is also using the next bit. This | |
30 | * is used so we can disable all of the MSI interrupts when a device | |
31 | * uses multiple. | |
32 | */ | |
1aa2b278 | 33 | static u64 msi_multiple_irq_bitmask[4]; |
e8635b48 DD |
34 | |
35 | /* | |
36 | * This lock controls updates to msi_free_irq_bitmask and | |
37 | * msi_multiple_irq_bitmask. | |
38 | */ | |
39 | static DEFINE_SPINLOCK(msi_free_irq_bitmask_lock); | |
40 | ||
1aa2b278 DD |
41 | /* |
42 | * Number of MSI IRQs used. This variable is set up in | |
43 | * the module init time. | |
44 | */ | |
45 | static int msi_irq_size; | |
e8635b48 DD |
46 | |
47 | /** | |
48 | * Called when a driver request MSI interrupts instead of the | |
49 | * legacy INT A-D. This routine will allocate multiple interrupts | |
50 | * for MSI devices that support them. A device can override this by | |
51 | * programming the MSI control bits [6:4] before calling | |
52 | * pci_enable_msi(). | |
53 | * | |
01a6221a DD |
54 | * @dev: Device requesting MSI interrupts |
55 | * @desc: MSI descriptor | |
e8635b48 DD |
56 | * |
57 | * Returns 0 on success. | |
58 | */ | |
59 | int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) | |
60 | { | |
61 | struct msi_msg msg; | |
1aa2b278 | 62 | u16 control; |
e8635b48 DD |
63 | int configured_private_bits; |
64 | int request_private_bits; | |
1aa2b278 | 65 | int irq = 0; |
e8635b48 | 66 | int irq_step; |
1aa2b278 DD |
67 | u64 search_mask; |
68 | int index; | |
e8635b48 DD |
69 | |
70 | /* | |
71 | * Read the MSI config to figure out how many IRQs this device | |
72 | * wants. Most devices only want 1, which will give | |
73 | * configured_private_bits and request_private_bits equal 0. | |
74 | */ | |
75 | pci_read_config_word(dev, desc->msi_attrib.pos + PCI_MSI_FLAGS, | |
76 | &control); | |
77 | ||
78 | /* | |
79 | * If the number of private bits has been configured then use | |
80 | * that value instead of the requested number. This gives the | |
81 | * driver the chance to override the number of interrupts | |
82 | * before calling pci_enable_msi(). | |
83 | */ | |
84 | configured_private_bits = (control & PCI_MSI_FLAGS_QSIZE) >> 4; | |
85 | if (configured_private_bits == 0) { | |
86 | /* Nothing is configured, so use the hardware requested size */ | |
87 | request_private_bits = (control & PCI_MSI_FLAGS_QMASK) >> 1; | |
88 | } else { | |
89 | /* | |
90 | * Use the number of configured bits, assuming the | |
91 | * driver wanted to override the hardware request | |
92 | * value. | |
93 | */ | |
94 | request_private_bits = configured_private_bits; | |
95 | } | |
96 | ||
97 | /* | |
98 | * The PCI 2.3 spec mandates that there are at most 32 | |
99 | * interrupts. If this device asks for more, only give it one. | |
100 | */ | |
101 | if (request_private_bits > 5) | |
102 | request_private_bits = 0; | |
103 | ||
104 | try_only_one: | |
105 | /* | |
106 | * The IRQs have to be aligned on a power of two based on the | |
107 | * number being requested. | |
108 | */ | |
109 | irq_step = 1 << request_private_bits; | |
110 | ||
111 | /* Mask with one bit for each IRQ */ | |
112 | search_mask = (1 << irq_step) - 1; | |
113 | ||
114 | /* | |
115 | * We're going to search msi_free_irq_bitmask_lock for zero | |
116 | * bits. This represents an MSI interrupt number that isn't in | |
117 | * use. | |
118 | */ | |
119 | spin_lock(&msi_free_irq_bitmask_lock); | |
1aa2b278 DD |
120 | for (index = 0; index < msi_irq_size/64; index++) { |
121 | for (irq = 0; irq < 64; irq += irq_step) { | |
122 | if ((msi_free_irq_bitmask[index] & (search_mask << irq)) == 0) { | |
123 | msi_free_irq_bitmask[index] |= search_mask << irq; | |
124 | msi_multiple_irq_bitmask[index] |= (search_mask >> 1) << irq; | |
125 | goto msi_irq_allocated; | |
126 | } | |
e8635b48 DD |
127 | } |
128 | } | |
1aa2b278 | 129 | msi_irq_allocated: |
e8635b48 DD |
130 | spin_unlock(&msi_free_irq_bitmask_lock); |
131 | ||
132 | /* Make sure the search for available interrupts didn't fail */ | |
133 | if (irq >= 64) { | |
134 | if (request_private_bits) { | |
1aa2b278 | 135 | pr_err("arch_setup_msi_irq: Unable to find %d free interrupts, trying just one", |
e8635b48 DD |
136 | 1 << request_private_bits); |
137 | request_private_bits = 0; | |
138 | goto try_only_one; | |
139 | } else | |
1aa2b278 | 140 | panic("arch_setup_msi_irq: Unable to find a free MSI interrupt"); |
e8635b48 DD |
141 | } |
142 | ||
143 | /* MSI interrupts start at logical IRQ OCTEON_IRQ_MSI_BIT0 */ | |
1aa2b278 | 144 | irq += index*64; |
e8635b48 DD |
145 | irq += OCTEON_IRQ_MSI_BIT0; |
146 | ||
147 | switch (octeon_dma_bar_type) { | |
148 | case OCTEON_DMA_BAR_TYPE_SMALL: | |
149 | /* When not using big bar, Bar 0 is based at 128MB */ | |
150 | msg.address_lo = | |
151 | ((128ul << 20) + CVMX_PCI_MSI_RCV) & 0xffffffff; | |
152 | msg.address_hi = ((128ul << 20) + CVMX_PCI_MSI_RCV) >> 32; | |
7f02c463 | 153 | break; |
e8635b48 DD |
154 | case OCTEON_DMA_BAR_TYPE_BIG: |
155 | /* When using big bar, Bar 0 is based at 0 */ | |
156 | msg.address_lo = (0 + CVMX_PCI_MSI_RCV) & 0xffffffff; | |
157 | msg.address_hi = (0 + CVMX_PCI_MSI_RCV) >> 32; | |
158 | break; | |
159 | case OCTEON_DMA_BAR_TYPE_PCIE: | |
160 | /* When using PCIe, Bar 0 is based at 0 */ | |
161 | /* FIXME CVMX_NPEI_MSI_RCV* other than 0? */ | |
162 | msg.address_lo = (0 + CVMX_NPEI_PCIE_MSI_RCV) & 0xffffffff; | |
163 | msg.address_hi = (0 + CVMX_NPEI_PCIE_MSI_RCV) >> 32; | |
164 | break; | |
165 | default: | |
ab75dc02 | 166 | panic("arch_setup_msi_irq: Invalid octeon_dma_bar_type"); |
e8635b48 DD |
167 | } |
168 | msg.data = irq - OCTEON_IRQ_MSI_BIT0; | |
169 | ||
170 | /* Update the number of IRQs the device has available to it */ | |
171 | control &= ~PCI_MSI_FLAGS_QSIZE; | |
172 | control |= request_private_bits << 4; | |
173 | pci_write_config_word(dev, desc->msi_attrib.pos + PCI_MSI_FLAGS, | |
174 | control); | |
175 | ||
e4ec7989 | 176 | irq_set_msi_desc(irq, desc); |
e8635b48 DD |
177 | write_msi_msg(irq, &msg); |
178 | return 0; | |
179 | } | |
180 | ||
52a0f00b CC |
181 | int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) |
182 | { | |
183 | struct msi_desc *entry; | |
184 | int ret; | |
185 | ||
186 | /* | |
187 | * MSI-X is not supported. | |
188 | */ | |
189 | if (type == PCI_CAP_ID_MSIX) | |
190 | return -EINVAL; | |
191 | ||
192 | /* | |
193 | * If an architecture wants to support multiple MSI, it needs to | |
194 | * override arch_setup_msi_irqs() | |
195 | */ | |
196 | if (type == PCI_CAP_ID_MSI && nvec > 1) | |
197 | return 1; | |
198 | ||
199 | list_for_each_entry(entry, &dev->msi_list, list) { | |
200 | ret = arch_setup_msi_irq(dev, entry); | |
201 | if (ret < 0) | |
202 | return ret; | |
203 | if (ret > 0) | |
204 | return -ENOSPC; | |
205 | } | |
206 | ||
207 | return 0; | |
208 | } | |
e8635b48 DD |
209 | |
210 | /** | |
211 | * Called when a device no longer needs its MSI interrupts. All | |
212 | * MSI interrupts for the device are freed. | |
213 | * | |
214 | * @irq: The devices first irq number. There may be multple in sequence. | |
215 | */ | |
216 | void arch_teardown_msi_irq(unsigned int irq) | |
217 | { | |
218 | int number_irqs; | |
1aa2b278 DD |
219 | u64 bitmask; |
220 | int index = 0; | |
221 | int irq0; | |
e8635b48 | 222 | |
1aa2b278 DD |
223 | if ((irq < OCTEON_IRQ_MSI_BIT0) |
224 | || (irq > msi_irq_size + OCTEON_IRQ_MSI_BIT0)) | |
e8635b48 DD |
225 | panic("arch_teardown_msi_irq: Attempted to teardown illegal " |
226 | "MSI interrupt (%d)", irq); | |
1aa2b278 | 227 | |
e8635b48 | 228 | irq -= OCTEON_IRQ_MSI_BIT0; |
1aa2b278 DD |
229 | index = irq / 64; |
230 | irq0 = irq % 64; | |
e8635b48 DD |
231 | |
232 | /* | |
233 | * Count the number of IRQs we need to free by looking at the | |
234 | * msi_multiple_irq_bitmask. Each bit set means that the next | |
235 | * IRQ is also owned by this device. | |
236 | */ | |
237 | number_irqs = 0; | |
1aa2b278 DD |
238 | while ((irq0 + number_irqs < 64) && |
239 | (msi_multiple_irq_bitmask[index] | |
240 | & (1ull << (irq0 + number_irqs)))) | |
e8635b48 DD |
241 | number_irqs++; |
242 | number_irqs++; | |
243 | /* Mask with one bit for each IRQ */ | |
244 | bitmask = (1 << number_irqs) - 1; | |
245 | /* Shift the mask to the correct bit location */ | |
1aa2b278 DD |
246 | bitmask <<= irq0; |
247 | if ((msi_free_irq_bitmask[index] & bitmask) != bitmask) | |
e8635b48 DD |
248 | panic("arch_teardown_msi_irq: Attempted to teardown MSI " |
249 | "interrupt (%d) not in use", irq); | |
250 | ||
251 | /* Checks are done, update the in use bitmask */ | |
252 | spin_lock(&msi_free_irq_bitmask_lock); | |
1aa2b278 DD |
253 | msi_free_irq_bitmask[index] &= ~bitmask; |
254 | msi_multiple_irq_bitmask[index] &= ~bitmask; | |
e8635b48 DD |
255 | spin_unlock(&msi_free_irq_bitmask_lock); |
256 | } | |
257 | ||
1aa2b278 | 258 | static DEFINE_RAW_SPINLOCK(octeon_irq_msi_lock); |
e8635b48 | 259 | |
1aa2b278 DD |
260 | static u64 msi_rcv_reg[4]; |
261 | static u64 mis_ena_reg[4]; | |
262 | ||
0c326387 | 263 | static void octeon_irq_msi_enable_pcie(struct irq_data *data) |
e8635b48 | 264 | { |
1aa2b278 DD |
265 | u64 en; |
266 | unsigned long flags; | |
0c326387 | 267 | int msi_number = data->irq - OCTEON_IRQ_MSI_BIT0; |
1aa2b278 DD |
268 | int irq_index = msi_number >> 6; |
269 | int irq_bit = msi_number & 0x3f; | |
270 | ||
271 | raw_spin_lock_irqsave(&octeon_irq_msi_lock, flags); | |
272 | en = cvmx_read_csr(mis_ena_reg[irq_index]); | |
273 | en |= 1ull << irq_bit; | |
274 | cvmx_write_csr(mis_ena_reg[irq_index], en); | |
275 | cvmx_read_csr(mis_ena_reg[irq_index]); | |
276 | raw_spin_unlock_irqrestore(&octeon_irq_msi_lock, flags); | |
277 | } | |
e8635b48 | 278 | |
0c326387 | 279 | static void octeon_irq_msi_disable_pcie(struct irq_data *data) |
1aa2b278 DD |
280 | { |
281 | u64 en; | |
282 | unsigned long flags; | |
0c326387 | 283 | int msi_number = data->irq - OCTEON_IRQ_MSI_BIT0; |
1aa2b278 DD |
284 | int irq_index = msi_number >> 6; |
285 | int irq_bit = msi_number & 0x3f; | |
286 | ||
287 | raw_spin_lock_irqsave(&octeon_irq_msi_lock, flags); | |
288 | en = cvmx_read_csr(mis_ena_reg[irq_index]); | |
289 | en &= ~(1ull << irq_bit); | |
290 | cvmx_write_csr(mis_ena_reg[irq_index], en); | |
291 | cvmx_read_csr(mis_ena_reg[irq_index]); | |
292 | raw_spin_unlock_irqrestore(&octeon_irq_msi_lock, flags); | |
e8635b48 DD |
293 | } |
294 | ||
1aa2b278 DD |
295 | static struct irq_chip octeon_irq_chip_msi_pcie = { |
296 | .name = "MSI", | |
0c326387 DD |
297 | .irq_enable = octeon_irq_msi_enable_pcie, |
298 | .irq_disable = octeon_irq_msi_disable_pcie, | |
1aa2b278 | 299 | }; |
a894f14d | 300 | |
0c326387 | 301 | static void octeon_irq_msi_enable_pci(struct irq_data *data) |
a894f14d | 302 | { |
1aa2b278 DD |
303 | /* |
304 | * Octeon PCI doesn't have the ability to mask/unmask MSI | |
305 | * interrupts individually. Instead of masking/unmasking them | |
306 | * in groups of 16, we simple assume MSI devices are well | |
307 | * behaved. MSI interrupts are always enable and the ACK is | |
308 | * assumed to be enough | |
309 | */ | |
a894f14d DD |
310 | } |
311 | ||
0c326387 | 312 | static void octeon_irq_msi_disable_pci(struct irq_data *data) |
a894f14d | 313 | { |
1aa2b278 | 314 | /* See comment in enable */ |
a894f14d DD |
315 | } |
316 | ||
1aa2b278 | 317 | static struct irq_chip octeon_irq_chip_msi_pci = { |
a894f14d | 318 | .name = "MSI", |
0c326387 DD |
319 | .irq_enable = octeon_irq_msi_enable_pci, |
320 | .irq_disable = octeon_irq_msi_disable_pci, | |
a894f14d | 321 | }; |
e8635b48 | 322 | |
01a6221a | 323 | /* |
1aa2b278 DD |
324 | * Called by the interrupt handling code when an MSI interrupt |
325 | * occurs. | |
e8635b48 | 326 | */ |
1aa2b278 | 327 | static irqreturn_t __octeon_msi_do_interrupt(int index, u64 msi_bits) |
e8635b48 | 328 | { |
a894f14d | 329 | int irq; |
1aa2b278 | 330 | int bit; |
a894f14d | 331 | |
1aa2b278 DD |
332 | bit = fls64(msi_bits); |
333 | if (bit) { | |
334 | bit--; | |
335 | /* Acknowledge it first. */ | |
336 | cvmx_write_csr(msi_rcv_reg[index], 1ull << bit); | |
337 | ||
338 | irq = bit + OCTEON_IRQ_MSI_BIT0 + 64 * index; | |
339 | do_IRQ(irq); | |
340 | return IRQ_HANDLED; | |
a894f14d | 341 | } |
1aa2b278 DD |
342 | return IRQ_NONE; |
343 | } | |
344 | ||
345 | #define OCTEON_MSI_INT_HANDLER_X(x) \ | |
346 | static irqreturn_t octeon_msi_interrupt##x(int cpl, void *dev_id) \ | |
347 | { \ | |
348 | u64 msi_bits = cvmx_read_csr(msi_rcv_reg[(x)]); \ | |
349 | return __octeon_msi_do_interrupt((x), msi_bits); \ | |
350 | } | |
351 | ||
352 | /* | |
353 | * Create octeon_msi_interrupt{0-3} function body | |
354 | */ | |
355 | OCTEON_MSI_INT_HANDLER_X(0); | |
356 | OCTEON_MSI_INT_HANDLER_X(1); | |
357 | OCTEON_MSI_INT_HANDLER_X(2); | |
358 | OCTEON_MSI_INT_HANDLER_X(3); | |
359 | ||
360 | /* | |
361 | * Initializes the MSI interrupt handling code | |
362 | */ | |
363 | int __init octeon_msi_initialize(void) | |
364 | { | |
365 | int irq; | |
366 | struct irq_chip *msi; | |
367 | ||
368 | if (octeon_dma_bar_type == OCTEON_DMA_BAR_TYPE_PCIE) { | |
369 | msi_rcv_reg[0] = CVMX_PEXP_NPEI_MSI_RCV0; | |
370 | msi_rcv_reg[1] = CVMX_PEXP_NPEI_MSI_RCV1; | |
371 | msi_rcv_reg[2] = CVMX_PEXP_NPEI_MSI_RCV2; | |
372 | msi_rcv_reg[3] = CVMX_PEXP_NPEI_MSI_RCV3; | |
373 | mis_ena_reg[0] = CVMX_PEXP_NPEI_MSI_ENB0; | |
374 | mis_ena_reg[1] = CVMX_PEXP_NPEI_MSI_ENB1; | |
375 | mis_ena_reg[2] = CVMX_PEXP_NPEI_MSI_ENB2; | |
376 | mis_ena_reg[3] = CVMX_PEXP_NPEI_MSI_ENB3; | |
377 | msi = &octeon_irq_chip_msi_pcie; | |
378 | } else { | |
379 | msi_rcv_reg[0] = CVMX_NPI_NPI_MSI_RCV; | |
380 | #define INVALID_GENERATE_ADE 0x8700000000000000ULL; | |
381 | msi_rcv_reg[1] = INVALID_GENERATE_ADE; | |
382 | msi_rcv_reg[2] = INVALID_GENERATE_ADE; | |
383 | msi_rcv_reg[3] = INVALID_GENERATE_ADE; | |
384 | mis_ena_reg[0] = INVALID_GENERATE_ADE; | |
385 | mis_ena_reg[1] = INVALID_GENERATE_ADE; | |
386 | mis_ena_reg[2] = INVALID_GENERATE_ADE; | |
387 | mis_ena_reg[3] = INVALID_GENERATE_ADE; | |
388 | msi = &octeon_irq_chip_msi_pci; | |
389 | } | |
390 | ||
391 | for (irq = OCTEON_IRQ_MSI_BIT0; irq <= OCTEON_IRQ_MSI_LAST; irq++) | |
e4ec7989 | 392 | irq_set_chip_and_handler(irq, msi, handle_simple_irq); |
a894f14d | 393 | |
e8635b48 | 394 | if (octeon_has_feature(OCTEON_FEATURE_PCIE)) { |
1aa2b278 DD |
395 | if (request_irq(OCTEON_IRQ_PCI_MSI0, octeon_msi_interrupt0, |
396 | 0, "MSI[0:63]", octeon_msi_interrupt0)) | |
01a6221a | 397 | panic("request_irq(OCTEON_IRQ_PCI_MSI0) failed"); |
1aa2b278 DD |
398 | |
399 | if (request_irq(OCTEON_IRQ_PCI_MSI1, octeon_msi_interrupt1, | |
400 | 0, "MSI[64:127]", octeon_msi_interrupt1)) | |
401 | panic("request_irq(OCTEON_IRQ_PCI_MSI1) failed"); | |
402 | ||
403 | if (request_irq(OCTEON_IRQ_PCI_MSI2, octeon_msi_interrupt2, | |
404 | 0, "MSI[127:191]", octeon_msi_interrupt2)) | |
405 | panic("request_irq(OCTEON_IRQ_PCI_MSI2) failed"); | |
406 | ||
407 | if (request_irq(OCTEON_IRQ_PCI_MSI3, octeon_msi_interrupt3, | |
408 | 0, "MSI[192:255]", octeon_msi_interrupt3)) | |
409 | panic("request_irq(OCTEON_IRQ_PCI_MSI3) failed"); | |
410 | ||
411 | msi_irq_size = 256; | |
e8635b48 | 412 | } else if (octeon_is_pci_host()) { |
1aa2b278 DD |
413 | if (request_irq(OCTEON_IRQ_PCI_MSI0, octeon_msi_interrupt0, |
414 | 0, "MSI[0:15]", octeon_msi_interrupt0)) | |
01a6221a DD |
415 | panic("request_irq(OCTEON_IRQ_PCI_MSI0) failed"); |
416 | ||
1aa2b278 DD |
417 | if (request_irq(OCTEON_IRQ_PCI_MSI1, octeon_msi_interrupt0, |
418 | 0, "MSI[16:31]", octeon_msi_interrupt0)) | |
01a6221a DD |
419 | panic("request_irq(OCTEON_IRQ_PCI_MSI1) failed"); |
420 | ||
1aa2b278 DD |
421 | if (request_irq(OCTEON_IRQ_PCI_MSI2, octeon_msi_interrupt0, |
422 | 0, "MSI[32:47]", octeon_msi_interrupt0)) | |
01a6221a DD |
423 | panic("request_irq(OCTEON_IRQ_PCI_MSI2) failed"); |
424 | ||
1aa2b278 DD |
425 | if (request_irq(OCTEON_IRQ_PCI_MSI3, octeon_msi_interrupt0, |
426 | 0, "MSI[48:63]", octeon_msi_interrupt0)) | |
01a6221a | 427 | panic("request_irq(OCTEON_IRQ_PCI_MSI3) failed"); |
1aa2b278 | 428 | msi_irq_size = 64; |
e8635b48 DD |
429 | } |
430 | return 0; | |
431 | } | |
e8635b48 | 432 | subsys_initcall(octeon_msi_initialize); |