]>
Commit | Line | Data |
---|---|---|
8cfab3cf | 1 | // SPDX-License-Identifier: GPL-2.0 |
425c1b22 AW |
2 | /* |
3 | * PCI Virtual Channel support | |
4 | * | |
5 | * Copyright (C) 2013 Red Hat, Inc. All rights reserved. | |
6 | * Author: Alex Williamson <alex.williamson@redhat.com> | |
425c1b22 AW |
7 | */ |
8 | ||
9 | #include <linux/device.h> | |
10 | #include <linux/kernel.h> | |
11 | #include <linux/module.h> | |
12 | #include <linux/pci.h> | |
13 | #include <linux/pci_regs.h> | |
14 | #include <linux/types.h> | |
15 | ||
ca784104 MW |
16 | #include "pci.h" |
17 | ||
425c1b22 AW |
18 | /** |
19 | * pci_vc_save_restore_dwords - Save or restore a series of dwords | |
20 | * @dev: device | |
21 | * @pos: starting config space position | |
22 | * @buf: buffer to save to or restore from | |
23 | * @dwords: number of dwords to save/restore | |
24 | * @save: whether to save or restore | |
25 | */ | |
26 | static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos, | |
27 | u32 *buf, int dwords, bool save) | |
28 | { | |
29 | int i; | |
30 | ||
31 | for (i = 0; i < dwords; i++, buf++) { | |
32 | if (save) | |
33 | pci_read_config_dword(dev, pos + (i * 4), buf); | |
34 | else | |
35 | pci_write_config_dword(dev, pos + (i * 4), *buf); | |
36 | } | |
37 | } | |
38 | ||
39 | /** | |
40 | * pci_vc_load_arb_table - load and wait for VC arbitration table | |
41 | * @dev: device | |
42 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
43 | * | |
44 | * Set Load VC Arbitration Table bit requesting hardware to apply the VC | |
45 | * Arbitration Table (previously loaded). When the VC Arbitration Table | |
46 | * Status clears, hardware has latched the table into VC arbitration logic. | |
47 | */ | |
48 | static void pci_vc_load_arb_table(struct pci_dev *dev, int pos) | |
49 | { | |
50 | u16 ctrl; | |
51 | ||
52 | pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, &ctrl); | |
53 | pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, | |
54 | ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE); | |
55 | if (pci_wait_for_pending(dev, pos + PCI_VC_PORT_STATUS, | |
56 | PCI_VC_PORT_STATUS_TABLE)) | |
57 | return; | |
58 | ||
7506dc79 | 59 | pci_err(dev, "VC arbitration table failed to load\n"); |
425c1b22 AW |
60 | } |
61 | ||
62 | /** | |
63 | * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table | |
64 | * @dev: device | |
65 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
66 | * @res: VC resource number, ie. VCn (0-7) | |
67 | * | |
68 | * Set Load Port Arbitration Table bit requesting hardware to apply the Port | |
69 | * Arbitration Table (previously loaded). When the Port Arbitration Table | |
70 | * Status clears, hardware has latched the table into port arbitration logic. | |
71 | */ | |
72 | static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res) | |
73 | { | |
74 | int ctrl_pos, status_pos; | |
75 | u32 ctrl; | |
76 | ||
77 | ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
78 | status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
79 | ||
80 | pci_read_config_dword(dev, ctrl_pos, &ctrl); | |
81 | pci_write_config_dword(dev, ctrl_pos, | |
82 | ctrl | PCI_VC_RES_CTRL_LOAD_TABLE); | |
83 | ||
84 | if (pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_TABLE)) | |
85 | return; | |
86 | ||
7506dc79 | 87 | pci_err(dev, "VC%d port arbitration table failed to load\n", res); |
425c1b22 AW |
88 | } |
89 | ||
90 | /** | |
91 | * pci_vc_enable - Enable virtual channel | |
92 | * @dev: device | |
93 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
94 | * @res: VC res number, ie. VCn (0-7) | |
95 | * | |
96 | * A VC is enabled by setting the enable bit in matching resource control | |
97 | * registers on both sides of a link. We therefore need to find the opposite | |
98 | * end of the link. To keep this simple we enable from the downstream device. | |
99 | * RC devices do not have an upstream device, nor does it seem that VC9 do | |
100 | * (spec is unclear). Once we find the upstream device, match the VC ID to | |
101 | * get the correct resource, disable and enable on both ends. | |
102 | */ | |
103 | static void pci_vc_enable(struct pci_dev *dev, int pos, int res) | |
104 | { | |
105 | int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2; | |
274127a1 | 106 | u32 ctrl, header, cap1, ctrl2; |
425c1b22 AW |
107 | struct pci_dev *link = NULL; |
108 | ||
109 | /* Enable VCs from the downstream device */ | |
ca784104 | 110 | if (!pci_is_pcie(dev) || !pcie_downstream_port(dev)) |
425c1b22 AW |
111 | return; |
112 | ||
113 | ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
114 | status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
115 | ||
116 | pci_read_config_dword(dev, ctrl_pos, &ctrl); | |
117 | id = ctrl & PCI_VC_RES_CTRL_ID; | |
118 | ||
119 | pci_read_config_dword(dev, pos, &header); | |
120 | ||
121 | /* If there is no opposite end of the link, skip to enable */ | |
122 | if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 || | |
123 | pci_is_root_bus(dev->bus)) | |
124 | goto enable; | |
125 | ||
126 | pos2 = pci_find_ext_capability(dev->bus->self, PCI_EXT_CAP_ID_VC); | |
127 | if (!pos2) | |
128 | goto enable; | |
129 | ||
274127a1 AW |
130 | pci_read_config_dword(dev->bus->self, pos2 + PCI_VC_PORT_CAP1, &cap1); |
131 | evcc = cap1 & PCI_VC_CAP1_EVCC; | |
425c1b22 AW |
132 | |
133 | /* VC0 is hardwired enabled, so we can start with 1 */ | |
134 | for (i = 1; i < evcc + 1; i++) { | |
135 | ctrl_pos2 = pos2 + PCI_VC_RES_CTRL + | |
136 | (i * PCI_CAP_VC_PER_VC_SIZEOF); | |
137 | status_pos2 = pos2 + PCI_VC_RES_STATUS + | |
138 | (i * PCI_CAP_VC_PER_VC_SIZEOF); | |
139 | pci_read_config_dword(dev->bus->self, ctrl_pos2, &ctrl2); | |
140 | if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) { | |
141 | link = dev->bus->self; | |
142 | break; | |
143 | } | |
144 | } | |
145 | ||
146 | if (!link) | |
147 | goto enable; | |
148 | ||
149 | /* Disable if enabled */ | |
150 | if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) { | |
151 | ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE; | |
152 | pci_write_config_dword(link, ctrl_pos2, ctrl2); | |
153 | } | |
154 | ||
155 | /* Enable on both ends */ | |
156 | ctrl2 |= PCI_VC_RES_CTRL_ENABLE; | |
157 | pci_write_config_dword(link, ctrl_pos2, ctrl2); | |
158 | enable: | |
159 | ctrl |= PCI_VC_RES_CTRL_ENABLE; | |
160 | pci_write_config_dword(dev, ctrl_pos, ctrl); | |
161 | ||
162 | if (!pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_NEGO)) | |
7506dc79 | 163 | pci_err(dev, "VC%d negotiation stuck pending\n", id); |
425c1b22 AW |
164 | |
165 | if (link && !pci_wait_for_pending(link, status_pos2, | |
166 | PCI_VC_RES_STATUS_NEGO)) | |
7506dc79 | 167 | pci_err(link, "VC%d negotiation stuck pending\n", id); |
425c1b22 AW |
168 | } |
169 | ||
170 | /** | |
171 | * pci_vc_do_save_buffer - Size, save, or restore VC state | |
172 | * @dev: device | |
173 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
174 | * @save_state: buffer for save/restore | |
175 | * @name: for error message | |
176 | * @save: if provided a buffer, this indicates what to do with it | |
177 | * | |
178 | * Walking Virtual Channel config space to size, save, or restore it | |
179 | * is complicated, so we do it all from one function to reduce code and | |
180 | * guarantee ordering matches in the buffer. When called with NULL | |
181 | * @save_state, return the size of the necessary save buffer. When called | |
182 | * with a non-NULL @save_state, @save determines whether we save to the | |
183 | * buffer or restore from it. | |
184 | */ | |
185 | static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos, | |
186 | struct pci_cap_saved_state *save_state, | |
187 | bool save) | |
188 | { | |
274127a1 | 189 | u32 cap1; |
425c1b22 AW |
190 | char evcc, lpevcc, parb_size; |
191 | int i, len = 0; | |
192 | u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL; | |
193 | ||
194 | /* Sanity check buffer size for save/restore */ | |
195 | if (buf && save_state->cap.size != | |
196 | pci_vc_do_save_buffer(dev, pos, NULL, save)) { | |
7506dc79 | 197 | pci_err(dev, "VC save buffer size does not match @0x%x\n", pos); |
425c1b22 AW |
198 | return -ENOMEM; |
199 | } | |
200 | ||
274127a1 | 201 | pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP1, &cap1); |
425c1b22 | 202 | /* Extended VC Count (not counting VC0) */ |
274127a1 | 203 | evcc = cap1 & PCI_VC_CAP1_EVCC; |
425c1b22 | 204 | /* Low Priority Extended VC Count (not counting VC0) */ |
274127a1 | 205 | lpevcc = (cap1 & PCI_VC_CAP1_LPEVCC) >> 4; |
425c1b22 | 206 | /* Port Arbitration Table Entry Size (bits) */ |
274127a1 | 207 | parb_size = 1 << ((cap1 & PCI_VC_CAP1_ARB_SIZE) >> 10); |
425c1b22 AW |
208 | |
209 | /* | |
210 | * Port VC Control Register contains VC Arbitration Select, which | |
211 | * cannot be modified when more than one LPVC is in operation. We | |
212 | * therefore save/restore it first, as only VC0 should be enabled | |
213 | * after device reset. | |
214 | */ | |
215 | if (buf) { | |
216 | if (save) | |
217 | pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, | |
218 | (u16 *)buf); | |
219 | else | |
220 | pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, | |
221 | *(u16 *)buf); | |
ef0dab4a | 222 | buf += 4; |
425c1b22 | 223 | } |
ef0dab4a | 224 | len += 4; |
425c1b22 AW |
225 | |
226 | /* | |
227 | * If we have any Low Priority VCs and a VC Arbitration Table Offset | |
228 | * in Port VC Capability Register 2 then save/restore it next. | |
229 | */ | |
230 | if (lpevcc) { | |
274127a1 | 231 | u32 cap2; |
425c1b22 AW |
232 | int vcarb_offset; |
233 | ||
274127a1 AW |
234 | pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP2, &cap2); |
235 | vcarb_offset = ((cap2 & PCI_VC_CAP2_ARB_OFF) >> 24) * 16; | |
425c1b22 AW |
236 | |
237 | if (vcarb_offset) { | |
238 | int size, vcarb_phases = 0; | |
239 | ||
274127a1 | 240 | if (cap2 & PCI_VC_CAP2_128_PHASE) |
425c1b22 | 241 | vcarb_phases = 128; |
274127a1 | 242 | else if (cap2 & PCI_VC_CAP2_64_PHASE) |
425c1b22 | 243 | vcarb_phases = 64; |
274127a1 | 244 | else if (cap2 & PCI_VC_CAP2_32_PHASE) |
425c1b22 AW |
245 | vcarb_phases = 32; |
246 | ||
247 | /* Fixed 4 bits per phase per lpevcc (plus VC0) */ | |
248 | size = ((lpevcc + 1) * vcarb_phases * 4) / 8; | |
249 | ||
250 | if (size && buf) { | |
251 | pci_vc_save_restore_dwords(dev, | |
252 | pos + vcarb_offset, | |
253 | (u32 *)buf, | |
254 | size / 4, save); | |
255 | /* | |
256 | * On restore, we need to signal hardware to | |
257 | * re-load the VC Arbitration Table. | |
258 | */ | |
259 | if (!save) | |
260 | pci_vc_load_arb_table(dev, pos); | |
261 | ||
262 | buf += size; | |
263 | } | |
264 | len += size; | |
265 | } | |
266 | } | |
267 | ||
268 | /* | |
269 | * In addition to each VC Resource Control Register, we may have a | |
270 | * Port Arbitration Table attached to each VC. The Port Arbitration | |
271 | * Table Offset in each VC Resource Capability Register tells us if | |
272 | * it exists. The entry size is global from the Port VC Capability | |
273 | * Register1 above. The number of phases is determined per VC. | |
274 | */ | |
275 | for (i = 0; i < evcc + 1; i++) { | |
276 | u32 cap; | |
277 | int parb_offset; | |
278 | ||
279 | pci_read_config_dword(dev, pos + PCI_VC_RES_CAP + | |
280 | (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap); | |
281 | parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16; | |
282 | if (parb_offset) { | |
283 | int size, parb_phases = 0; | |
284 | ||
285 | if (cap & PCI_VC_RES_CAP_256_PHASE) | |
286 | parb_phases = 256; | |
287 | else if (cap & (PCI_VC_RES_CAP_128_PHASE | | |
288 | PCI_VC_RES_CAP_128_PHASE_TB)) | |
289 | parb_phases = 128; | |
290 | else if (cap & PCI_VC_RES_CAP_64_PHASE) | |
291 | parb_phases = 64; | |
292 | else if (cap & PCI_VC_RES_CAP_32_PHASE) | |
293 | parb_phases = 32; | |
294 | ||
295 | size = (parb_size * parb_phases) / 8; | |
296 | ||
297 | if (size && buf) { | |
298 | pci_vc_save_restore_dwords(dev, | |
299 | pos + parb_offset, | |
300 | (u32 *)buf, | |
301 | size / 4, save); | |
302 | buf += size; | |
303 | } | |
304 | len += size; | |
305 | } | |
306 | ||
307 | /* VC Resource Control Register */ | |
308 | if (buf) { | |
309 | int ctrl_pos = pos + PCI_VC_RES_CTRL + | |
310 | (i * PCI_CAP_VC_PER_VC_SIZEOF); | |
311 | if (save) | |
312 | pci_read_config_dword(dev, ctrl_pos, | |
313 | (u32 *)buf); | |
314 | else { | |
315 | u32 tmp, ctrl = *(u32 *)buf; | |
316 | /* | |
317 | * For an FLR case, the VC config may remain. | |
318 | * Preserve enable bit, restore the rest. | |
319 | */ | |
320 | pci_read_config_dword(dev, ctrl_pos, &tmp); | |
321 | tmp &= PCI_VC_RES_CTRL_ENABLE; | |
322 | tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE; | |
323 | pci_write_config_dword(dev, ctrl_pos, tmp); | |
324 | /* Load port arbitration table if used */ | |
325 | if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT) | |
326 | pci_vc_load_port_arb_table(dev, pos, i); | |
327 | /* Re-enable if needed */ | |
328 | if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE) | |
329 | pci_vc_enable(dev, pos, i); | |
330 | } | |
331 | buf += 4; | |
332 | } | |
333 | len += 4; | |
334 | } | |
335 | ||
336 | return buf ? 0 : len; | |
337 | } | |
338 | ||
339 | static struct { | |
340 | u16 id; | |
341 | const char *name; | |
342 | } vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" }, | |
343 | { PCI_EXT_CAP_ID_VC, "VC" }, | |
344 | { PCI_EXT_CAP_ID_VC9, "VC9" } }; | |
345 | ||
346 | /** | |
347 | * pci_save_vc_state - Save VC state to pre-allocate save buffer | |
348 | * @dev: device | |
349 | * | |
350 | * For each type of VC capability, VC/VC9/MFVC, find the capability and | |
351 | * save it to the pre-allocated save buffer. | |
352 | */ | |
353 | int pci_save_vc_state(struct pci_dev *dev) | |
354 | { | |
355 | int i; | |
356 | ||
357 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { | |
358 | int pos, ret; | |
359 | struct pci_cap_saved_state *save_state; | |
360 | ||
361 | pos = pci_find_ext_capability(dev, vc_caps[i].id); | |
362 | if (!pos) | |
363 | continue; | |
364 | ||
365 | save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); | |
366 | if (!save_state) { | |
7506dc79 | 367 | pci_err(dev, "%s buffer not found in %s\n", |
425c1b22 AW |
368 | vc_caps[i].name, __func__); |
369 | return -ENOMEM; | |
370 | } | |
371 | ||
372 | ret = pci_vc_do_save_buffer(dev, pos, save_state, true); | |
373 | if (ret) { | |
7506dc79 | 374 | pci_err(dev, "%s save unsuccessful %s\n", |
425c1b22 AW |
375 | vc_caps[i].name, __func__); |
376 | return ret; | |
377 | } | |
378 | } | |
379 | ||
380 | return 0; | |
381 | } | |
382 | ||
383 | /** | |
384 | * pci_restore_vc_state - Restore VC state from save buffer | |
385 | * @dev: device | |
386 | * | |
387 | * For each type of VC capability, VC/VC9/MFVC, find the capability and | |
388 | * restore it from the previously saved buffer. | |
389 | */ | |
390 | void pci_restore_vc_state(struct pci_dev *dev) | |
391 | { | |
392 | int i; | |
393 | ||
394 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { | |
395 | int pos; | |
396 | struct pci_cap_saved_state *save_state; | |
397 | ||
398 | pos = pci_find_ext_capability(dev, vc_caps[i].id); | |
399 | save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); | |
400 | if (!save_state || !pos) | |
401 | continue; | |
402 | ||
403 | pci_vc_do_save_buffer(dev, pos, save_state, false); | |
404 | } | |
405 | } | |
406 | ||
407 | /** | |
408 | * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps | |
409 | * @dev: device | |
410 | * | |
411 | * For each type of VC capability, VC/VC9/MFVC, find the capability, size | |
412 | * it, and allocate a buffer for save/restore. | |
413 | */ | |
425c1b22 AW |
414 | void pci_allocate_vc_save_buffers(struct pci_dev *dev) |
415 | { | |
416 | int i; | |
417 | ||
418 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { | |
419 | int len, pos = pci_find_ext_capability(dev, vc_caps[i].id); | |
420 | ||
421 | if (!pos) | |
422 | continue; | |
423 | ||
424 | len = pci_vc_do_save_buffer(dev, pos, NULL, false); | |
425 | if (pci_add_ext_cap_save_buffer(dev, vc_caps[i].id, len)) | |
7506dc79 | 426 | pci_err(dev, "unable to preallocate %s save buffer\n", |
425c1b22 AW |
427 | vc_caps[i].name); |
428 | } | |
429 | } |