]>
Commit | Line | Data |
---|---|---|
d94d71cb | 1 | // SPDX-License-Identifier: GPL-2.0-only |
695093e3 | 2 | /* |
695093e3 VS |
3 | * |
4 | * Copyright (C) 2013 Freescale Semiconductor, Inc. | |
695093e3 VS |
5 | */ |
6 | ||
7 | #define pr_fmt(fmt) "fsl-pamu: %s: " fmt, __func__ | |
8 | ||
cd70d465 EM |
9 | #include "fsl_pamu.h" |
10 | ||
94848654 | 11 | #include <linux/fsl/guts.h> |
695093e3 | 12 | #include <linux/interrupt.h> |
695093e3 | 13 | #include <linux/genalloc.h> |
695093e3 | 14 | |
cd70d465 | 15 | #include <asm/mpc85xx.h> |
695093e3 VS |
16 | |
17 | /* define indexes for each operation mapping scenario */ | |
18 | #define OMI_QMAN 0x00 | |
19 | #define OMI_FMAN 0x01 | |
20 | #define OMI_QMAN_PRIV 0x02 | |
21 | #define OMI_CAAM 0x03 | |
22 | ||
23 | #define make64(high, low) (((u64)(high) << 32) | (low)) | |
24 | ||
25 | struct pamu_isr_data { | |
cd70d465 | 26 | void __iomem *pamu_reg_base; /* Base address of PAMU regs */ |
695093e3 VS |
27 | unsigned int count; /* The number of PAMUs */ |
28 | }; | |
29 | ||
30 | static struct paace *ppaact; | |
31 | static struct paace *spaact; | |
695093e3 | 32 | |
07eb6fdf JR |
33 | static bool probed; /* Has PAMU been probed? */ |
34 | ||
695093e3 VS |
35 | /* |
36 | * Table for matching compatible strings, for device tree | |
37 | * guts node, for QorIQ SOCs. | |
38 | * "fsl,qoriq-device-config-2.0" corresponds to T4 & B4 | |
39 | * SOCs. For the older SOCs "fsl,qoriq-device-config-1.0" | |
40 | * string would be used. | |
cd70d465 | 41 | */ |
57fb907d | 42 | static const struct of_device_id guts_device_ids[] = { |
695093e3 VS |
43 | { .compatible = "fsl,qoriq-device-config-1.0", }, |
44 | { .compatible = "fsl,qoriq-device-config-2.0", }, | |
45 | {} | |
46 | }; | |
47 | ||
695093e3 VS |
48 | /* |
49 | * Table for matching compatible strings, for device tree | |
50 | * L3 cache controller node. | |
51 | * "fsl,t4240-l3-cache-controller" corresponds to T4, | |
52 | * "fsl,b4860-l3-cache-controller" corresponds to B4 & | |
53 | * "fsl,p4080-l3-cache-controller" corresponds to other, | |
54 | * SOCs. | |
cd70d465 | 55 | */ |
695093e3 VS |
56 | static const struct of_device_id l3_device_ids[] = { |
57 | { .compatible = "fsl,t4240-l3-cache-controller", }, | |
58 | { .compatible = "fsl,b4860-l3-cache-controller", }, | |
59 | { .compatible = "fsl,p4080-l3-cache-controller", }, | |
60 | {} | |
61 | }; | |
62 | ||
63 | /* maximum subwindows permitted per liodn */ | |
64 | static u32 max_subwindow_count; | |
65 | ||
695093e3 VS |
66 | /** |
67 | * pamu_get_ppaace() - Return the primary PACCE | |
68 | * @liodn: liodn PAACT index for desired PAACE | |
69 | * | |
70 | * Returns the ppace pointer upon success else return | |
71 | * null. | |
72 | */ | |
73 | static struct paace *pamu_get_ppaace(int liodn) | |
74 | { | |
75 | if (!ppaact || liodn >= PAACE_NUMBER_ENTRIES) { | |
76 | pr_debug("PPAACT doesn't exist\n"); | |
77 | return NULL; | |
78 | } | |
79 | ||
80 | return &ppaact[liodn]; | |
81 | } | |
82 | ||
83 | /** | |
84 | * pamu_enable_liodn() - Set valid bit of PACCE | |
85 | * @liodn: liodn PAACT index for desired PAACE | |
86 | * | |
87 | * Returns 0 upon success else error code < 0 returned | |
88 | */ | |
89 | int pamu_enable_liodn(int liodn) | |
90 | { | |
91 | struct paace *ppaace; | |
92 | ||
93 | ppaace = pamu_get_ppaace(liodn); | |
94 | if (!ppaace) { | |
95 | pr_debug("Invalid primary paace entry\n"); | |
96 | return -ENOENT; | |
97 | } | |
98 | ||
99 | if (!get_bf(ppaace->addr_bitfields, PPAACE_AF_WSE)) { | |
100 | pr_debug("liodn %d not configured\n", liodn); | |
101 | return -EINVAL; | |
102 | } | |
103 | ||
104 | /* Ensure that all other stores to the ppaace complete first */ | |
105 | mb(); | |
106 | ||
107 | set_bf(ppaace->addr_bitfields, PAACE_AF_V, PAACE_V_VALID); | |
108 | mb(); | |
109 | ||
110 | return 0; | |
111 | } | |
112 | ||
113 | /** | |
114 | * pamu_disable_liodn() - Clears valid bit of PACCE | |
115 | * @liodn: liodn PAACT index for desired PAACE | |
116 | * | |
117 | * Returns 0 upon success else error code < 0 returned | |
118 | */ | |
119 | int pamu_disable_liodn(int liodn) | |
120 | { | |
121 | struct paace *ppaace; | |
122 | ||
123 | ppaace = pamu_get_ppaace(liodn); | |
124 | if (!ppaace) { | |
125 | pr_debug("Invalid primary paace entry\n"); | |
126 | return -ENOENT; | |
127 | } | |
128 | ||
129 | set_bf(ppaace->addr_bitfields, PAACE_AF_V, PAACE_V_INVALID); | |
130 | mb(); | |
131 | ||
132 | return 0; | |
133 | } | |
134 | ||
135 | /* Derive the window size encoding for a particular PAACE entry */ | |
136 | static unsigned int map_addrspace_size_to_wse(phys_addr_t addrspace_size) | |
137 | { | |
138 | /* Bug if not a power of 2 */ | |
cd70d465 | 139 | BUG_ON(addrspace_size & (addrspace_size - 1)); |
695093e3 VS |
140 | |
141 | /* window size is 2^(WSE+1) bytes */ | |
d033f48f | 142 | return fls64(addrspace_size) - 2; |
695093e3 VS |
143 | } |
144 | ||
695093e3 VS |
145 | /* |
146 | * Set the PAACE type as primary and set the coherency required domain | |
147 | * attribute | |
148 | */ | |
149 | static void pamu_init_ppaace(struct paace *ppaace) | |
150 | { | |
151 | set_bf(ppaace->addr_bitfields, PAACE_AF_PT, PAACE_PT_PRIMARY); | |
152 | ||
153 | set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, | |
154 | PAACE_M_COHERENCE_REQ); | |
155 | } | |
156 | ||
695093e3 VS |
157 | /* |
158 | * Function used for updating stash destination for the coressponding | |
159 | * LIODN. | |
160 | */ | |
ba58d121 | 161 | int pamu_update_paace_stash(int liodn, u32 value) |
695093e3 VS |
162 | { |
163 | struct paace *paace; | |
164 | ||
165 | paace = pamu_get_ppaace(liodn); | |
166 | if (!paace) { | |
167 | pr_debug("Invalid liodn entry\n"); | |
168 | return -ENOENT; | |
169 | } | |
695093e3 VS |
170 | set_bf(paace->impl_attr, PAACE_IA_CID, value); |
171 | ||
172 | mb(); | |
173 | ||
174 | return 0; | |
175 | } | |
176 | ||
695093e3 VS |
177 | /** |
178 | * pamu_config_paace() - Sets up PPAACE entry for specified liodn | |
179 | * | |
180 | * @liodn: Logical IO device number | |
695093e3 | 181 | * @omi: Operation mapping index -- if ~omi == 0 then omi not defined |
695093e3 VS |
182 | * @stashid: cache stash id for associated cpu -- if ~stashid == 0 then |
183 | * stashid not defined | |
695093e3 VS |
184 | * @prot: window permissions |
185 | * | |
186 | * Returns 0 upon success else error code < 0 returned | |
187 | */ | |
57fa44be | 188 | int pamu_config_ppaace(int liodn, u32 omi, u32 stashid, int prot) |
695093e3 VS |
189 | { |
190 | struct paace *ppaace; | |
695093e3 | 191 | |
695093e3 | 192 | ppaace = pamu_get_ppaace(liodn); |
cd70d465 | 193 | if (!ppaace) |
695093e3 | 194 | return -ENOENT; |
695093e3 VS |
195 | |
196 | /* window size is 2^(WSE+1) bytes */ | |
197 | set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, | |
57fa44be | 198 | map_addrspace_size_to_wse(1ULL << 36)); |
695093e3 VS |
199 | |
200 | pamu_init_ppaace(ppaace); | |
201 | ||
57fa44be CH |
202 | ppaace->wbah = 0; |
203 | set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 0); | |
695093e3 VS |
204 | |
205 | /* set up operation mapping if it's configured */ | |
206 | if (omi < OME_NUMBER_ENTRIES) { | |
207 | set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); | |
208 | ppaace->op_encode.index_ot.omi = omi; | |
209 | } else if (~omi != 0) { | |
210 | pr_debug("bad operation mapping index: %d\n", omi); | |
211 | return -EINVAL; | |
212 | } | |
213 | ||
214 | /* configure stash id */ | |
215 | if (~stashid != 0) | |
216 | set_bf(ppaace->impl_attr, PAACE_IA_CID, stashid); | |
217 | ||
ba58d121 | 218 | set_bf(ppaace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); |
151f9414 CH |
219 | ppaace->twbah = 0; |
220 | set_bf(ppaace->win_bitfields, PAACE_WIN_TWBAL, 0); | |
ba58d121 CH |
221 | set_bf(ppaace->addr_bitfields, PAACE_AF_AP, prot); |
222 | set_bf(ppaace->impl_attr, PAACE_IA_WCE, 0); | |
223 | set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); | |
695093e3 VS |
224 | mb(); |
225 | ||
226 | return 0; | |
227 | } | |
228 | ||
229 | /** | |
cd70d465 EM |
230 | * get_ome_index() - Returns the index in the operation mapping table |
231 | * for device. | |
232 | * @*omi_index: pointer for storing the index value | |
233 | * | |
234 | */ | |
695093e3 VS |
235 | void get_ome_index(u32 *omi_index, struct device *dev) |
236 | { | |
237 | if (of_device_is_compatible(dev->of_node, "fsl,qman-portal")) | |
238 | *omi_index = OMI_QMAN; | |
239 | if (of_device_is_compatible(dev->of_node, "fsl,qman")) | |
240 | *omi_index = OMI_QMAN_PRIV; | |
241 | } | |
242 | ||
243 | /** | |
244 | * get_stash_id - Returns stash destination id corresponding to a | |
245 | * cache type and vcpu. | |
246 | * @stash_dest_hint: L1, L2 or L3 | |
247 | * @vcpu: vpcu target for a particular cache type. | |
248 | * | |
249 | * Returs stash on success or ~(u32)0 on failure. | |
250 | * | |
251 | */ | |
252 | u32 get_stash_id(u32 stash_dest_hint, u32 vcpu) | |
253 | { | |
254 | const u32 *prop; | |
255 | struct device_node *node; | |
256 | u32 cache_level; | |
257 | int len, found = 0; | |
258 | int i; | |
259 | ||
260 | /* Fastpath, exit early if L3/CPC cache is target for stashing */ | |
261 | if (stash_dest_hint == PAMU_ATTR_CACHE_L3) { | |
262 | node = of_find_matching_node(NULL, l3_device_ids); | |
263 | if (node) { | |
cd70d465 | 264 | prop = of_get_property(node, "cache-stash-id", NULL); |
695093e3 | 265 | if (!prop) { |
6bd4f1c7 RH |
266 | pr_debug("missing cache-stash-id at %pOF\n", |
267 | node); | |
695093e3 VS |
268 | of_node_put(node); |
269 | return ~(u32)0; | |
270 | } | |
271 | of_node_put(node); | |
272 | return be32_to_cpup(prop); | |
273 | } | |
274 | return ~(u32)0; | |
275 | } | |
276 | ||
a9a455e8 | 277 | for_each_of_cpu_node(node) { |
695093e3 VS |
278 | prop = of_get_property(node, "reg", &len); |
279 | for (i = 0; i < len / sizeof(u32); i++) { | |
280 | if (be32_to_cpup(&prop[i]) == vcpu) { | |
281 | found = 1; | |
282 | goto found_cpu_node; | |
283 | } | |
284 | } | |
285 | } | |
286 | found_cpu_node: | |
287 | ||
288 | /* find the hwnode that represents the cache */ | |
289 | for (cache_level = PAMU_ATTR_CACHE_L1; (cache_level < PAMU_ATTR_CACHE_L3) && found; cache_level++) { | |
290 | if (stash_dest_hint == cache_level) { | |
cd70d465 | 291 | prop = of_get_property(node, "cache-stash-id", NULL); |
695093e3 | 292 | if (!prop) { |
6bd4f1c7 RH |
293 | pr_debug("missing cache-stash-id at %pOF\n", |
294 | node); | |
695093e3 VS |
295 | of_node_put(node); |
296 | return ~(u32)0; | |
297 | } | |
298 | of_node_put(node); | |
299 | return be32_to_cpup(prop); | |
300 | } | |
301 | ||
cd70d465 | 302 | prop = of_get_property(node, "next-level-cache", NULL); |
695093e3 | 303 | if (!prop) { |
6bd4f1c7 | 304 | pr_debug("can't find next-level-cache at %pOF\n", node); |
695093e3 VS |
305 | of_node_put(node); |
306 | return ~(u32)0; /* can't traverse any further */ | |
307 | } | |
308 | of_node_put(node); | |
309 | ||
310 | /* advance to next node in cache hierarchy */ | |
311 | node = of_find_node_by_phandle(*prop); | |
312 | if (!node) { | |
d6a71bf7 | 313 | pr_debug("Invalid node for cache hierarchy\n"); |
695093e3 VS |
314 | return ~(u32)0; |
315 | } | |
316 | } | |
317 | ||
318 | pr_debug("stash dest not found for %d on vcpu %d\n", | |
cd70d465 | 319 | stash_dest_hint, vcpu); |
695093e3 VS |
320 | return ~(u32)0; |
321 | } | |
322 | ||
323 | /* Identify if the PAACT table entry belongs to QMAN, BMAN or QMAN Portal */ | |
324 | #define QMAN_PAACE 1 | |
325 | #define QMAN_PORTAL_PAACE 2 | |
326 | #define BMAN_PAACE 3 | |
327 | ||
328 | /** | |
329 | * Setup operation mapping and stash destinations for QMAN and QMAN portal. | |
330 | * Memory accesses to QMAN and BMAN private memory need not be coherent, so | |
331 | * clear the PAACE entry coherency attribute for them. | |
332 | */ | |
57fb907d | 333 | static void setup_qbman_paace(struct paace *ppaace, int paace_type) |
695093e3 VS |
334 | { |
335 | switch (paace_type) { | |
336 | case QMAN_PAACE: | |
337 | set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); | |
338 | ppaace->op_encode.index_ot.omi = OMI_QMAN_PRIV; | |
339 | /* setup QMAN Private data stashing for the L3 cache */ | |
340 | set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); | |
341 | set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, | |
342 | 0); | |
343 | break; | |
344 | case QMAN_PORTAL_PAACE: | |
345 | set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); | |
346 | ppaace->op_encode.index_ot.omi = OMI_QMAN; | |
cd70d465 | 347 | /* Set DQRR and Frame stashing for the L3 cache */ |
695093e3 VS |
348 | set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); |
349 | break; | |
350 | case BMAN_PAACE: | |
351 | set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, | |
352 | 0); | |
353 | break; | |
354 | } | |
355 | } | |
356 | ||
357 | /** | |
358 | * Setup the operation mapping table for various devices. This is a static | |
359 | * table where each table index corresponds to a particular device. PAMU uses | |
360 | * this table to translate device transaction to appropriate corenet | |
361 | * transaction. | |
362 | */ | |
57fb907d | 363 | static void setup_omt(struct ome *omt) |
695093e3 VS |
364 | { |
365 | struct ome *ome; | |
366 | ||
367 | /* Configure OMI_QMAN */ | |
368 | ome = &omt[OMI_QMAN]; | |
369 | ||
370 | ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READ; | |
371 | ome->moe[IOE_EREAD0_IDX] = EOE_VALID | EOE_RSA; | |
372 | ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; | |
373 | ome->moe[IOE_EWRITE0_IDX] = EOE_VALID | EOE_WWSAO; | |
374 | ||
375 | ome->moe[IOE_DIRECT0_IDX] = EOE_VALID | EOE_LDEC; | |
376 | ome->moe[IOE_DIRECT1_IDX] = EOE_VALID | EOE_LDECPE; | |
377 | ||
378 | /* Configure OMI_FMAN */ | |
379 | ome = &omt[OMI_FMAN]; | |
380 | ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READI; | |
381 | ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; | |
382 | ||
383 | /* Configure OMI_QMAN private */ | |
384 | ome = &omt[OMI_QMAN_PRIV]; | |
385 | ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READ; | |
386 | ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; | |
387 | ome->moe[IOE_EREAD0_IDX] = EOE_VALID | EOE_RSA; | |
388 | ome->moe[IOE_EWRITE0_IDX] = EOE_VALID | EOE_WWSA; | |
389 | ||
390 | /* Configure OMI_CAAM */ | |
391 | ome = &omt[OMI_CAAM]; | |
392 | ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READI; | |
393 | ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; | |
394 | } | |
395 | ||
396 | /* | |
397 | * Get the maximum number of PAACT table entries | |
398 | * and subwindows supported by PAMU | |
399 | */ | |
57fb907d | 400 | static void get_pamu_cap_values(unsigned long pamu_reg_base) |
695093e3 VS |
401 | { |
402 | u32 pc_val; | |
403 | ||
404 | pc_val = in_be32((u32 *)(pamu_reg_base + PAMU_PC3)); | |
405 | /* Maximum number of subwindows per liodn */ | |
406 | max_subwindow_count = 1 << (1 + PAMU_PC3_MWCE(pc_val)); | |
407 | } | |
408 | ||
409 | /* Setup PAMU registers pointing to PAACT, SPAACT and OMT */ | |
57fb907d EM |
410 | static int setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, |
411 | phys_addr_t ppaact_phys, phys_addr_t spaact_phys, | |
412 | phys_addr_t omt_phys) | |
695093e3 VS |
413 | { |
414 | u32 *pc; | |
415 | struct pamu_mmap_regs *pamu_regs; | |
416 | ||
417 | pc = (u32 *) (pamu_reg_base + PAMU_PC); | |
418 | pamu_regs = (struct pamu_mmap_regs *) | |
419 | (pamu_reg_base + PAMU_MMAP_REGS_BASE); | |
420 | ||
421 | /* set up pointers to corenet control blocks */ | |
422 | ||
423 | out_be32(&pamu_regs->ppbah, upper_32_bits(ppaact_phys)); | |
424 | out_be32(&pamu_regs->ppbal, lower_32_bits(ppaact_phys)); | |
425 | ppaact_phys = ppaact_phys + PAACT_SIZE; | |
426 | out_be32(&pamu_regs->pplah, upper_32_bits(ppaact_phys)); | |
427 | out_be32(&pamu_regs->pplal, lower_32_bits(ppaact_phys)); | |
428 | ||
429 | out_be32(&pamu_regs->spbah, upper_32_bits(spaact_phys)); | |
430 | out_be32(&pamu_regs->spbal, lower_32_bits(spaact_phys)); | |
431 | spaact_phys = spaact_phys + SPAACT_SIZE; | |
432 | out_be32(&pamu_regs->splah, upper_32_bits(spaact_phys)); | |
433 | out_be32(&pamu_regs->splal, lower_32_bits(spaact_phys)); | |
434 | ||
435 | out_be32(&pamu_regs->obah, upper_32_bits(omt_phys)); | |
436 | out_be32(&pamu_regs->obal, lower_32_bits(omt_phys)); | |
437 | omt_phys = omt_phys + OMT_SIZE; | |
438 | out_be32(&pamu_regs->olah, upper_32_bits(omt_phys)); | |
439 | out_be32(&pamu_regs->olal, lower_32_bits(omt_phys)); | |
440 | ||
441 | /* | |
442 | * set PAMU enable bit, | |
443 | * allow ppaact & omt to be cached | |
444 | * & enable PAMU access violation interrupts. | |
445 | */ | |
446 | ||
447 | out_be32((u32 *)(pamu_reg_base + PAMU_PICS), | |
cd70d465 | 448 | PAMU_ACCESS_VIOLATION_ENABLE); |
695093e3 VS |
449 | out_be32(pc, PAMU_PC_PE | PAMU_PC_OCE | PAMU_PC_SPCC | PAMU_PC_PPCC); |
450 | return 0; | |
451 | } | |
452 | ||
453 | /* Enable all device LIODNS */ | |
57fb907d | 454 | static void setup_liodns(void) |
695093e3 VS |
455 | { |
456 | int i, len; | |
457 | struct paace *ppaace; | |
458 | struct device_node *node = NULL; | |
459 | const u32 *prop; | |
460 | ||
461 | for_each_node_with_property(node, "fsl,liodn") { | |
462 | prop = of_get_property(node, "fsl,liodn", &len); | |
463 | for (i = 0; i < len / sizeof(u32); i++) { | |
464 | int liodn; | |
465 | ||
466 | liodn = be32_to_cpup(&prop[i]); | |
467 | if (liodn >= PAACE_NUMBER_ENTRIES) { | |
468 | pr_debug("Invalid LIODN value %d\n", liodn); | |
469 | continue; | |
470 | } | |
471 | ppaace = pamu_get_ppaace(liodn); | |
472 | pamu_init_ppaace(ppaace); | |
473 | /* window size is 2^(WSE+1) bytes */ | |
474 | set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, 35); | |
475 | ppaace->wbah = 0; | |
476 | set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 0); | |
477 | set_bf(ppaace->impl_attr, PAACE_IA_ATM, | |
cd70d465 | 478 | PAACE_ATM_NO_XLATE); |
695093e3 | 479 | set_bf(ppaace->addr_bitfields, PAACE_AF_AP, |
cd70d465 | 480 | PAACE_AP_PERMS_ALL); |
695093e3 VS |
481 | if (of_device_is_compatible(node, "fsl,qman-portal")) |
482 | setup_qbman_paace(ppaace, QMAN_PORTAL_PAACE); | |
483 | if (of_device_is_compatible(node, "fsl,qman")) | |
484 | setup_qbman_paace(ppaace, QMAN_PAACE); | |
485 | if (of_device_is_compatible(node, "fsl,bman")) | |
486 | setup_qbman_paace(ppaace, BMAN_PAACE); | |
487 | mb(); | |
488 | pamu_enable_liodn(liodn); | |
489 | } | |
490 | } | |
491 | } | |
492 | ||
cd70d465 | 493 | static irqreturn_t pamu_av_isr(int irq, void *arg) |
695093e3 VS |
494 | { |
495 | struct pamu_isr_data *data = arg; | |
496 | phys_addr_t phys; | |
497 | unsigned int i, j, ret; | |
498 | ||
634544bf | 499 | pr_emerg("access violation interrupt\n"); |
695093e3 VS |
500 | |
501 | for (i = 0; i < data->count; i++) { | |
502 | void __iomem *p = data->pamu_reg_base + i * PAMU_OFFSET; | |
503 | u32 pics = in_be32(p + PAMU_PICS); | |
504 | ||
505 | if (pics & PAMU_ACCESS_VIOLATION_STAT) { | |
506 | u32 avs1 = in_be32(p + PAMU_AVS1); | |
507 | struct paace *paace; | |
508 | ||
509 | pr_emerg("POES1=%08x\n", in_be32(p + PAMU_POES1)); | |
510 | pr_emerg("POES2=%08x\n", in_be32(p + PAMU_POES2)); | |
511 | pr_emerg("AVS1=%08x\n", avs1); | |
512 | pr_emerg("AVS2=%08x\n", in_be32(p + PAMU_AVS2)); | |
cd70d465 EM |
513 | pr_emerg("AVA=%016llx\n", |
514 | make64(in_be32(p + PAMU_AVAH), | |
515 | in_be32(p + PAMU_AVAL))); | |
695093e3 | 516 | pr_emerg("UDAD=%08x\n", in_be32(p + PAMU_UDAD)); |
cd70d465 EM |
517 | pr_emerg("POEA=%016llx\n", |
518 | make64(in_be32(p + PAMU_POEAH), | |
519 | in_be32(p + PAMU_POEAL))); | |
695093e3 VS |
520 | |
521 | phys = make64(in_be32(p + PAMU_POEAH), | |
cd70d465 | 522 | in_be32(p + PAMU_POEAL)); |
695093e3 VS |
523 | |
524 | /* Assume that POEA points to a PAACE */ | |
525 | if (phys) { | |
526 | u32 *paace = phys_to_virt(phys); | |
527 | ||
528 | /* Only the first four words are relevant */ | |
529 | for (j = 0; j < 4; j++) | |
cd70d465 EM |
530 | pr_emerg("PAACE[%u]=%08x\n", |
531 | j, in_be32(paace + j)); | |
695093e3 VS |
532 | } |
533 | ||
534 | /* clear access violation condition */ | |
cd70d465 | 535 | out_be32(p + PAMU_AVS1, avs1 & PAMU_AV_MASK); |
695093e3 VS |
536 | paace = pamu_get_ppaace(avs1 >> PAMU_AVS1_LIODN_SHIFT); |
537 | BUG_ON(!paace); | |
538 | /* check if we got a violation for a disabled LIODN */ | |
539 | if (!get_bf(paace->addr_bitfields, PAACE_AF_V)) { | |
540 | /* | |
541 | * As per hardware erratum A-003638, access | |
542 | * violation can be reported for a disabled | |
543 | * LIODN. If we hit that condition, disable | |
544 | * access violation reporting. | |
545 | */ | |
546 | pics &= ~PAMU_ACCESS_VIOLATION_ENABLE; | |
547 | } else { | |
548 | /* Disable the LIODN */ | |
549 | ret = pamu_disable_liodn(avs1 >> PAMU_AVS1_LIODN_SHIFT); | |
550 | BUG_ON(ret); | |
cd70d465 EM |
551 | pr_emerg("Disabling liodn %x\n", |
552 | avs1 >> PAMU_AVS1_LIODN_SHIFT); | |
695093e3 VS |
553 | } |
554 | out_be32((p + PAMU_PICS), pics); | |
555 | } | |
556 | } | |
557 | ||
695093e3 VS |
558 | return IRQ_HANDLED; |
559 | } | |
560 | ||
561 | #define LAWAR_EN 0x80000000 | |
562 | #define LAWAR_TARGET_MASK 0x0FF00000 | |
563 | #define LAWAR_TARGET_SHIFT 20 | |
564 | #define LAWAR_SIZE_MASK 0x0000003F | |
565 | #define LAWAR_CSDID_MASK 0x000FF000 | |
566 | #define LAWAR_CSDID_SHIFT 12 | |
567 | ||
568 | #define LAW_SIZE_4K 0xb | |
569 | ||
570 | struct ccsr_law { | |
571 | u32 lawbarh; /* LAWn base address high */ | |
572 | u32 lawbarl; /* LAWn base address low */ | |
573 | u32 lawar; /* LAWn attributes */ | |
574 | u32 reserved; | |
575 | }; | |
576 | ||
577 | /* | |
578 | * Create a coherence subdomain for a given memory block. | |
579 | */ | |
57fb907d | 580 | static int create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) |
695093e3 VS |
581 | { |
582 | struct device_node *np; | |
583 | const __be32 *iprop; | |
584 | void __iomem *lac = NULL; /* Local Access Control registers */ | |
585 | struct ccsr_law __iomem *law; | |
586 | void __iomem *ccm = NULL; | |
587 | u32 __iomem *csdids; | |
588 | unsigned int i, num_laws, num_csds; | |
589 | u32 law_target = 0; | |
590 | u32 csd_id = 0; | |
591 | int ret = 0; | |
592 | ||
593 | np = of_find_compatible_node(NULL, NULL, "fsl,corenet-law"); | |
594 | if (!np) | |
595 | return -ENODEV; | |
596 | ||
597 | iprop = of_get_property(np, "fsl,num-laws", NULL); | |
598 | if (!iprop) { | |
599 | ret = -ENODEV; | |
600 | goto error; | |
601 | } | |
602 | ||
603 | num_laws = be32_to_cpup(iprop); | |
604 | if (!num_laws) { | |
605 | ret = -ENODEV; | |
606 | goto error; | |
607 | } | |
608 | ||
609 | lac = of_iomap(np, 0); | |
610 | if (!lac) { | |
611 | ret = -ENODEV; | |
612 | goto error; | |
613 | } | |
614 | ||
615 | /* LAW registers are at offset 0xC00 */ | |
616 | law = lac + 0xC00; | |
617 | ||
618 | of_node_put(np); | |
619 | ||
620 | np = of_find_compatible_node(NULL, NULL, "fsl,corenet-cf"); | |
621 | if (!np) { | |
622 | ret = -ENODEV; | |
623 | goto error; | |
624 | } | |
625 | ||
626 | iprop = of_get_property(np, "fsl,ccf-num-csdids", NULL); | |
627 | if (!iprop) { | |
628 | ret = -ENODEV; | |
629 | goto error; | |
630 | } | |
631 | ||
632 | num_csds = be32_to_cpup(iprop); | |
633 | if (!num_csds) { | |
634 | ret = -ENODEV; | |
635 | goto error; | |
636 | } | |
637 | ||
638 | ccm = of_iomap(np, 0); | |
639 | if (!ccm) { | |
640 | ret = -ENOMEM; | |
641 | goto error; | |
642 | } | |
643 | ||
644 | /* The undocumented CSDID registers are at offset 0x600 */ | |
645 | csdids = ccm + 0x600; | |
646 | ||
647 | of_node_put(np); | |
648 | np = NULL; | |
649 | ||
650 | /* Find an unused coherence subdomain ID */ | |
651 | for (csd_id = 0; csd_id < num_csds; csd_id++) { | |
652 | if (!csdids[csd_id]) | |
653 | break; | |
654 | } | |
655 | ||
656 | /* Store the Port ID in the (undocumented) proper CIDMRxx register */ | |
657 | csdids[csd_id] = csd_port_id; | |
658 | ||
659 | /* Find the DDR LAW that maps to our buffer. */ | |
660 | for (i = 0; i < num_laws; i++) { | |
661 | if (law[i].lawar & LAWAR_EN) { | |
662 | phys_addr_t law_start, law_end; | |
663 | ||
664 | law_start = make64(law[i].lawbarh, law[i].lawbarl); | |
665 | law_end = law_start + | |
666 | (2ULL << (law[i].lawar & LAWAR_SIZE_MASK)); | |
667 | ||
668 | if (law_start <= phys && phys < law_end) { | |
669 | law_target = law[i].lawar & LAWAR_TARGET_MASK; | |
670 | break; | |
671 | } | |
672 | } | |
673 | } | |
674 | ||
675 | if (i == 0 || i == num_laws) { | |
cd70d465 | 676 | /* This should never happen */ |
695093e3 VS |
677 | ret = -ENOENT; |
678 | goto error; | |
679 | } | |
680 | ||
681 | /* Find a free LAW entry */ | |
682 | while (law[--i].lawar & LAWAR_EN) { | |
683 | if (i == 0) { | |
684 | /* No higher priority LAW slots available */ | |
685 | ret = -ENOENT; | |
686 | goto error; | |
687 | } | |
688 | } | |
689 | ||
690 | law[i].lawbarh = upper_32_bits(phys); | |
691 | law[i].lawbarl = lower_32_bits(phys); | |
692 | wmb(); | |
693 | law[i].lawar = LAWAR_EN | law_target | (csd_id << LAWAR_CSDID_SHIFT) | | |
694 | (LAW_SIZE_4K + get_order(size)); | |
695 | wmb(); | |
696 | ||
697 | error: | |
698 | if (ccm) | |
699 | iounmap(ccm); | |
700 | ||
701 | if (lac) | |
702 | iounmap(lac); | |
703 | ||
704 | if (np) | |
705 | of_node_put(np); | |
706 | ||
707 | return ret; | |
708 | } | |
709 | ||
710 | /* | |
711 | * Table of SVRs and the corresponding PORT_ID values. Port ID corresponds to a | |
712 | * bit map of snoopers for a given range of memory mapped by a LAW. | |
713 | * | |
714 | * All future CoreNet-enabled SOCs will have this erratum(A-004510) fixed, so this | |
715 | * table should never need to be updated. SVRs are guaranteed to be unique, so | |
716 | * there is no worry that a future SOC will inadvertently have one of these | |
717 | * values. | |
718 | */ | |
719 | static const struct { | |
720 | u32 svr; | |
721 | u32 port_id; | |
57fb907d | 722 | } port_id_map[] = { |
cd70d465 EM |
723 | {(SVR_P2040 << 8) | 0x10, 0xFF000000}, /* P2040 1.0 */ |
724 | {(SVR_P2040 << 8) | 0x11, 0xFF000000}, /* P2040 1.1 */ | |
725 | {(SVR_P2041 << 8) | 0x10, 0xFF000000}, /* P2041 1.0 */ | |
726 | {(SVR_P2041 << 8) | 0x11, 0xFF000000}, /* P2041 1.1 */ | |
727 | {(SVR_P3041 << 8) | 0x10, 0xFF000000}, /* P3041 1.0 */ | |
728 | {(SVR_P3041 << 8) | 0x11, 0xFF000000}, /* P3041 1.1 */ | |
729 | {(SVR_P4040 << 8) | 0x20, 0xFFF80000}, /* P4040 2.0 */ | |
730 | {(SVR_P4080 << 8) | 0x20, 0xFFF80000}, /* P4080 2.0 */ | |
731 | {(SVR_P5010 << 8) | 0x10, 0xFC000000}, /* P5010 1.0 */ | |
732 | {(SVR_P5010 << 8) | 0x20, 0xFC000000}, /* P5010 2.0 */ | |
733 | {(SVR_P5020 << 8) | 0x10, 0xFC000000}, /* P5020 1.0 */ | |
734 | {(SVR_P5021 << 8) | 0x10, 0xFF800000}, /* P5021 1.0 */ | |
735 | {(SVR_P5040 << 8) | 0x10, 0xFF800000}, /* P5040 1.0 */ | |
695093e3 VS |
736 | }; |
737 | ||
738 | #define SVR_SECURITY 0x80000 /* The Security (E) bit */ | |
739 | ||
57fb907d | 740 | static int fsl_pamu_probe(struct platform_device *pdev) |
695093e3 | 741 | { |
cd70d465 | 742 | struct device *dev = &pdev->dev; |
695093e3 VS |
743 | void __iomem *pamu_regs = NULL; |
744 | struct ccsr_guts __iomem *guts_regs = NULL; | |
745 | u32 pamubypenr, pamu_counter; | |
746 | unsigned long pamu_reg_off; | |
747 | unsigned long pamu_reg_base; | |
748 | struct pamu_isr_data *data = NULL; | |
749 | struct device_node *guts_node; | |
750 | u64 size; | |
751 | struct page *p; | |
752 | int ret = 0; | |
753 | int irq; | |
754 | phys_addr_t ppaact_phys; | |
755 | phys_addr_t spaact_phys; | |
57fb907d | 756 | struct ome *omt; |
695093e3 VS |
757 | phys_addr_t omt_phys; |
758 | size_t mem_size = 0; | |
759 | unsigned int order = 0; | |
760 | u32 csd_port_id = 0; | |
761 | unsigned i; | |
762 | /* | |
763 | * enumerate all PAMUs and allocate and setup PAMU tables | |
764 | * for each of them, | |
765 | * NOTE : All PAMUs share the same LIODN tables. | |
766 | */ | |
767 | ||
07eb6fdf JR |
768 | if (WARN_ON(probed)) |
769 | return -EBUSY; | |
770 | ||
cd70d465 | 771 | pamu_regs = of_iomap(dev->of_node, 0); |
695093e3 | 772 | if (!pamu_regs) { |
cd70d465 | 773 | dev_err(dev, "ioremap of PAMU node failed\n"); |
695093e3 VS |
774 | return -ENOMEM; |
775 | } | |
cd70d465 | 776 | of_get_address(dev->of_node, 0, &size, NULL); |
695093e3 | 777 | |
cd70d465 | 778 | irq = irq_of_parse_and_map(dev->of_node, 0); |
695093e3 | 779 | if (irq == NO_IRQ) { |
cd70d465 | 780 | dev_warn(dev, "no interrupts listed in PAMU node\n"); |
695093e3 VS |
781 | goto error; |
782 | } | |
783 | ||
cd70d465 | 784 | data = kzalloc(sizeof(*data), GFP_KERNEL); |
695093e3 | 785 | if (!data) { |
695093e3 VS |
786 | ret = -ENOMEM; |
787 | goto error; | |
788 | } | |
789 | data->pamu_reg_base = pamu_regs; | |
790 | data->count = size / PAMU_OFFSET; | |
791 | ||
792 | /* The ISR needs access to the regs, so we won't iounmap them */ | |
793 | ret = request_irq(irq, pamu_av_isr, 0, "pamu", data); | |
794 | if (ret < 0) { | |
cd70d465 | 795 | dev_err(dev, "error %i installing ISR for irq %i\n", ret, irq); |
695093e3 VS |
796 | goto error; |
797 | } | |
798 | ||
799 | guts_node = of_find_matching_node(NULL, guts_device_ids); | |
800 | if (!guts_node) { | |
6bd4f1c7 | 801 | dev_err(dev, "could not find GUTS node %pOF\n", dev->of_node); |
695093e3 VS |
802 | ret = -ENODEV; |
803 | goto error; | |
804 | } | |
805 | ||
806 | guts_regs = of_iomap(guts_node, 0); | |
807 | of_node_put(guts_node); | |
808 | if (!guts_regs) { | |
cd70d465 | 809 | dev_err(dev, "ioremap of GUTS node failed\n"); |
695093e3 VS |
810 | ret = -ENODEV; |
811 | goto error; | |
812 | } | |
813 | ||
814 | /* read in the PAMU capability registers */ | |
815 | get_pamu_cap_values((unsigned long)pamu_regs); | |
816 | /* | |
817 | * To simplify the allocation of a coherency domain, we allocate the | |
818 | * PAACT and the OMT in the same memory buffer. Unfortunately, this | |
819 | * wastes more memory compared to allocating the buffers separately. | |
820 | */ | |
821 | /* Determine how much memory we need */ | |
822 | mem_size = (PAGE_SIZE << get_order(PAACT_SIZE)) + | |
823 | (PAGE_SIZE << get_order(SPAACT_SIZE)) + | |
824 | (PAGE_SIZE << get_order(OMT_SIZE)); | |
825 | order = get_order(mem_size); | |
826 | ||
827 | p = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); | |
828 | if (!p) { | |
cd70d465 | 829 | dev_err(dev, "unable to allocate PAACT/SPAACT/OMT block\n"); |
695093e3 VS |
830 | ret = -ENOMEM; |
831 | goto error; | |
832 | } | |
833 | ||
834 | ppaact = page_address(p); | |
835 | ppaact_phys = page_to_phys(p); | |
836 | ||
837 | /* Make sure the memory is naturally aligned */ | |
838 | if (ppaact_phys & ((PAGE_SIZE << order) - 1)) { | |
cd70d465 | 839 | dev_err(dev, "PAACT/OMT block is unaligned\n"); |
695093e3 VS |
840 | ret = -ENOMEM; |
841 | goto error; | |
842 | } | |
843 | ||
844 | spaact = (void *)ppaact + (PAGE_SIZE << get_order(PAACT_SIZE)); | |
845 | omt = (void *)spaact + (PAGE_SIZE << get_order(SPAACT_SIZE)); | |
846 | ||
cd70d465 | 847 | dev_dbg(dev, "ppaact virt=%p phys=%pa\n", ppaact, &ppaact_phys); |
695093e3 VS |
848 | |
849 | /* Check to see if we need to implement the work-around on this SOC */ | |
850 | ||
851 | /* Determine the Port ID for our coherence subdomain */ | |
852 | for (i = 0; i < ARRAY_SIZE(port_id_map); i++) { | |
853 | if (port_id_map[i].svr == (mfspr(SPRN_SVR) & ~SVR_SECURITY)) { | |
854 | csd_port_id = port_id_map[i].port_id; | |
cd70d465 | 855 | dev_dbg(dev, "found matching SVR %08x\n", |
695093e3 VS |
856 | port_id_map[i].svr); |
857 | break; | |
858 | } | |
859 | } | |
860 | ||
861 | if (csd_port_id) { | |
cd70d465 EM |
862 | dev_dbg(dev, "creating coherency subdomain at address %pa, size %zu, port id 0x%08x", |
863 | &ppaact_phys, mem_size, csd_port_id); | |
695093e3 VS |
864 | |
865 | ret = create_csd(ppaact_phys, mem_size, csd_port_id); | |
866 | if (ret) { | |
cd70d465 | 867 | dev_err(dev, "could not create coherence subdomain\n"); |
695093e3 VS |
868 | return ret; |
869 | } | |
870 | } | |
871 | ||
872 | spaact_phys = virt_to_phys(spaact); | |
873 | omt_phys = virt_to_phys(omt); | |
874 | ||
695093e3 VS |
875 | pamubypenr = in_be32(&guts_regs->pamubypenr); |
876 | ||
877 | for (pamu_reg_off = 0, pamu_counter = 0x80000000; pamu_reg_off < size; | |
878 | pamu_reg_off += PAMU_OFFSET, pamu_counter >>= 1) { | |
879 | ||
cd70d465 | 880 | pamu_reg_base = (unsigned long)pamu_regs + pamu_reg_off; |
695093e3 | 881 | setup_one_pamu(pamu_reg_base, pamu_reg_off, ppaact_phys, |
cd70d465 | 882 | spaact_phys, omt_phys); |
695093e3 VS |
883 | /* Disable PAMU bypass for this PAMU */ |
884 | pamubypenr &= ~pamu_counter; | |
885 | } | |
886 | ||
887 | setup_omt(omt); | |
888 | ||
889 | /* Enable all relevant PAMU(s) */ | |
890 | out_be32(&guts_regs->pamubypenr, pamubypenr); | |
891 | ||
892 | iounmap(guts_regs); | |
893 | ||
cd70d465 | 894 | /* Enable DMA for the LIODNs in the device tree */ |
695093e3 VS |
895 | |
896 | setup_liodns(); | |
897 | ||
07eb6fdf JR |
898 | probed = true; |
899 | ||
695093e3 VS |
900 | return 0; |
901 | ||
695093e3 VS |
902 | error: |
903 | if (irq != NO_IRQ) | |
904 | free_irq(irq, data); | |
905 | ||
ce433d0f | 906 | kfree_sensitive(data); |
695093e3 VS |
907 | |
908 | if (pamu_regs) | |
909 | iounmap(pamu_regs); | |
910 | ||
911 | if (guts_regs) | |
912 | iounmap(guts_regs); | |
913 | ||
914 | if (ppaact) | |
915 | free_pages((unsigned long)ppaact, order); | |
916 | ||
917 | ppaact = NULL; | |
918 | ||
919 | return ret; | |
920 | } | |
921 | ||
57fb907d | 922 | static struct platform_driver fsl_of_pamu_driver = { |
695093e3 VS |
923 | .driver = { |
924 | .name = "fsl-of-pamu", | |
695093e3 VS |
925 | }, |
926 | .probe = fsl_pamu_probe, | |
927 | }; | |
928 | ||
929 | static __init int fsl_pamu_init(void) | |
930 | { | |
931 | struct platform_device *pdev = NULL; | |
932 | struct device_node *np; | |
933 | int ret; | |
934 | ||
935 | /* | |
936 | * The normal OF process calls the probe function at some | |
937 | * indeterminate later time, after most drivers have loaded. This is | |
938 | * too late for us, because PAMU clients (like the Qman driver) | |
939 | * depend on PAMU being initialized early. | |
940 | * | |
941 | * So instead, we "manually" call our probe function by creating the | |
942 | * platform devices ourselves. | |
943 | */ | |
944 | ||
945 | /* | |
946 | * We assume that there is only one PAMU node in the device tree. A | |
947 | * single PAMU node represents all of the PAMU devices in the SOC | |
948 | * already. Everything else already makes that assumption, and the | |
949 | * binding for the PAMU nodes doesn't allow for any parent-child | |
950 | * relationships anyway. In other words, support for more than one | |
951 | * PAMU node would require significant changes to a lot of code. | |
952 | */ | |
953 | ||
954 | np = of_find_compatible_node(NULL, NULL, "fsl,pamu"); | |
955 | if (!np) { | |
634544bf | 956 | pr_err("could not find a PAMU node\n"); |
695093e3 VS |
957 | return -ENODEV; |
958 | } | |
959 | ||
960 | ret = platform_driver_register(&fsl_of_pamu_driver); | |
961 | if (ret) { | |
634544bf | 962 | pr_err("could not register driver (err=%i)\n", ret); |
695093e3 VS |
963 | goto error_driver_register; |
964 | } | |
965 | ||
966 | pdev = platform_device_alloc("fsl-of-pamu", 0); | |
967 | if (!pdev) { | |
6bd4f1c7 | 968 | pr_err("could not allocate device %pOF\n", np); |
695093e3 VS |
969 | ret = -ENOMEM; |
970 | goto error_device_alloc; | |
971 | } | |
972 | pdev->dev.of_node = of_node_get(np); | |
973 | ||
974 | ret = pamu_domain_init(); | |
975 | if (ret) | |
976 | goto error_device_add; | |
977 | ||
978 | ret = platform_device_add(pdev); | |
979 | if (ret) { | |
6bd4f1c7 | 980 | pr_err("could not add device %pOF (err=%i)\n", np, ret); |
695093e3 VS |
981 | goto error_device_add; |
982 | } | |
983 | ||
984 | return 0; | |
985 | ||
986 | error_device_add: | |
987 | of_node_put(pdev->dev.of_node); | |
988 | pdev->dev.of_node = NULL; | |
989 | ||
990 | platform_device_put(pdev); | |
991 | ||
992 | error_device_alloc: | |
993 | platform_driver_unregister(&fsl_of_pamu_driver); | |
994 | ||
995 | error_driver_register: | |
996 | of_node_put(np); | |
997 | ||
998 | return ret; | |
999 | } | |
1000 | arch_initcall(fsl_pamu_init); |