]>
Commit | Line | Data |
---|---|---|
8cfab3cf | 1 | // SPDX-License-Identifier: GPL-2.0 |
425c1b22 AW |
2 | /* |
3 | * PCI Virtual Channel support | |
4 | * | |
5 | * Copyright (C) 2013 Red Hat, Inc. All rights reserved. | |
6 | * Author: Alex Williamson <alex.williamson@redhat.com> | |
425c1b22 AW |
7 | */ |
8 | ||
9 | #include <linux/device.h> | |
10 | #include <linux/kernel.h> | |
11 | #include <linux/module.h> | |
12 | #include <linux/pci.h> | |
13 | #include <linux/pci_regs.h> | |
14 | #include <linux/types.h> | |
15 | ||
ca784104 MW |
16 | #include "pci.h" |
17 | ||
425c1b22 AW |
18 | /** |
19 | * pci_vc_save_restore_dwords - Save or restore a series of dwords | |
20 | * @dev: device | |
21 | * @pos: starting config space position | |
22 | * @buf: buffer to save to or restore from | |
23 | * @dwords: number of dwords to save/restore | |
24 | * @save: whether to save or restore | |
25 | */ | |
26 | static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos, | |
27 | u32 *buf, int dwords, bool save) | |
28 | { | |
29 | int i; | |
30 | ||
31 | for (i = 0; i < dwords; i++, buf++) { | |
32 | if (save) | |
33 | pci_read_config_dword(dev, pos + (i * 4), buf); | |
34 | else | |
35 | pci_write_config_dword(dev, pos + (i * 4), *buf); | |
36 | } | |
37 | } | |
38 | ||
39 | /** | |
40 | * pci_vc_load_arb_table - load and wait for VC arbitration table | |
41 | * @dev: device | |
42 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
43 | * | |
44 | * Set Load VC Arbitration Table bit requesting hardware to apply the VC | |
45 | * Arbitration Table (previously loaded). When the VC Arbitration Table | |
46 | * Status clears, hardware has latched the table into VC arbitration logic. | |
47 | */ | |
48 | static void pci_vc_load_arb_table(struct pci_dev *dev, int pos) | |
49 | { | |
50 | u16 ctrl; | |
51 | ||
52 | pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, &ctrl); | |
53 | pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, | |
54 | ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE); | |
55 | if (pci_wait_for_pending(dev, pos + PCI_VC_PORT_STATUS, | |
56 | PCI_VC_PORT_STATUS_TABLE)) | |
57 | return; | |
58 | ||
7506dc79 | 59 | pci_err(dev, "VC arbitration table failed to load\n"); |
425c1b22 AW |
60 | } |
61 | ||
62 | /** | |
63 | * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table | |
64 | * @dev: device | |
65 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
66 | * @res: VC resource number, ie. VCn (0-7) | |
67 | * | |
68 | * Set Load Port Arbitration Table bit requesting hardware to apply the Port | |
69 | * Arbitration Table (previously loaded). When the Port Arbitration Table | |
70 | * Status clears, hardware has latched the table into port arbitration logic. | |
71 | */ | |
72 | static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res) | |
73 | { | |
74 | int ctrl_pos, status_pos; | |
75 | u32 ctrl; | |
76 | ||
77 | ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
78 | status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
79 | ||
80 | pci_read_config_dword(dev, ctrl_pos, &ctrl); | |
81 | pci_write_config_dword(dev, ctrl_pos, | |
82 | ctrl | PCI_VC_RES_CTRL_LOAD_TABLE); | |
83 | ||
84 | if (pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_TABLE)) | |
85 | return; | |
86 | ||
7506dc79 | 87 | pci_err(dev, "VC%d port arbitration table failed to load\n", res); |
425c1b22 AW |
88 | } |
89 | ||
90 | /** | |
91 | * pci_vc_enable - Enable virtual channel | |
92 | * @dev: device | |
93 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
94 | * @res: VC res number, ie. VCn (0-7) | |
95 | * | |
96 | * A VC is enabled by setting the enable bit in matching resource control | |
97 | * registers on both sides of a link. We therefore need to find the opposite | |
98 | * end of the link. To keep this simple we enable from the downstream device. | |
99 | * RC devices do not have an upstream device, nor does it seem that VC9 do | |
100 | * (spec is unclear). Once we find the upstream device, match the VC ID to | |
101 | * get the correct resource, disable and enable on both ends. | |
102 | */ | |
103 | static void pci_vc_enable(struct pci_dev *dev, int pos, int res) | |
104 | { | |
105 | int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2; | |
274127a1 | 106 | u32 ctrl, header, cap1, ctrl2; |
425c1b22 AW |
107 | struct pci_dev *link = NULL; |
108 | ||
109 | /* Enable VCs from the downstream device */ | |
ca784104 | 110 | if (!pci_is_pcie(dev) || !pcie_downstream_port(dev)) |
425c1b22 AW |
111 | return; |
112 | ||
113 | ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
114 | status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); | |
115 | ||
116 | pci_read_config_dword(dev, ctrl_pos, &ctrl); | |
117 | id = ctrl & PCI_VC_RES_CTRL_ID; | |
118 | ||
119 | pci_read_config_dword(dev, pos, &header); | |
120 | ||
121 | /* If there is no opposite end of the link, skip to enable */ | |
122 | if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 || | |
123 | pci_is_root_bus(dev->bus)) | |
124 | goto enable; | |
125 | ||
126 | pos2 = pci_find_ext_capability(dev->bus->self, PCI_EXT_CAP_ID_VC); | |
127 | if (!pos2) | |
128 | goto enable; | |
129 | ||
274127a1 AW |
130 | pci_read_config_dword(dev->bus->self, pos2 + PCI_VC_PORT_CAP1, &cap1); |
131 | evcc = cap1 & PCI_VC_CAP1_EVCC; | |
425c1b22 AW |
132 | |
133 | /* VC0 is hardwired enabled, so we can start with 1 */ | |
134 | for (i = 1; i < evcc + 1; i++) { | |
135 | ctrl_pos2 = pos2 + PCI_VC_RES_CTRL + | |
136 | (i * PCI_CAP_VC_PER_VC_SIZEOF); | |
137 | status_pos2 = pos2 + PCI_VC_RES_STATUS + | |
138 | (i * PCI_CAP_VC_PER_VC_SIZEOF); | |
139 | pci_read_config_dword(dev->bus->self, ctrl_pos2, &ctrl2); | |
140 | if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) { | |
141 | link = dev->bus->self; | |
142 | break; | |
143 | } | |
144 | } | |
145 | ||
146 | if (!link) | |
147 | goto enable; | |
148 | ||
149 | /* Disable if enabled */ | |
150 | if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) { | |
151 | ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE; | |
152 | pci_write_config_dword(link, ctrl_pos2, ctrl2); | |
153 | } | |
154 | ||
155 | /* Enable on both ends */ | |
156 | ctrl2 |= PCI_VC_RES_CTRL_ENABLE; | |
157 | pci_write_config_dword(link, ctrl_pos2, ctrl2); | |
158 | enable: | |
159 | ctrl |= PCI_VC_RES_CTRL_ENABLE; | |
160 | pci_write_config_dword(dev, ctrl_pos, ctrl); | |
161 | ||
162 | if (!pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_NEGO)) | |
7506dc79 | 163 | pci_err(dev, "VC%d negotiation stuck pending\n", id); |
425c1b22 AW |
164 | |
165 | if (link && !pci_wait_for_pending(link, status_pos2, | |
166 | PCI_VC_RES_STATUS_NEGO)) | |
7506dc79 | 167 | pci_err(link, "VC%d negotiation stuck pending\n", id); |
425c1b22 AW |
168 | } |
169 | ||
170 | /** | |
171 | * pci_vc_do_save_buffer - Size, save, or restore VC state | |
172 | * @dev: device | |
173 | * @pos: starting position of VC capability (VC/VC9/MFVC) | |
174 | * @save_state: buffer for save/restore | |
425c1b22 AW |
175 | * @save: if provided a buffer, this indicates what to do with it |
176 | * | |
177 | * Walking Virtual Channel config space to size, save, or restore it | |
178 | * is complicated, so we do it all from one function to reduce code and | |
179 | * guarantee ordering matches in the buffer. When called with NULL | |
180 | * @save_state, return the size of the necessary save buffer. When called | |
181 | * with a non-NULL @save_state, @save determines whether we save to the | |
182 | * buffer or restore from it. | |
183 | */ | |
184 | static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos, | |
185 | struct pci_cap_saved_state *save_state, | |
186 | bool save) | |
187 | { | |
274127a1 | 188 | u32 cap1; |
425c1b22 AW |
189 | char evcc, lpevcc, parb_size; |
190 | int i, len = 0; | |
191 | u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL; | |
192 | ||
193 | /* Sanity check buffer size for save/restore */ | |
194 | if (buf && save_state->cap.size != | |
195 | pci_vc_do_save_buffer(dev, pos, NULL, save)) { | |
7506dc79 | 196 | pci_err(dev, "VC save buffer size does not match @0x%x\n", pos); |
425c1b22 AW |
197 | return -ENOMEM; |
198 | } | |
199 | ||
274127a1 | 200 | pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP1, &cap1); |
425c1b22 | 201 | /* Extended VC Count (not counting VC0) */ |
274127a1 | 202 | evcc = cap1 & PCI_VC_CAP1_EVCC; |
425c1b22 | 203 | /* Low Priority Extended VC Count (not counting VC0) */ |
274127a1 | 204 | lpevcc = (cap1 & PCI_VC_CAP1_LPEVCC) >> 4; |
425c1b22 | 205 | /* Port Arbitration Table Entry Size (bits) */ |
274127a1 | 206 | parb_size = 1 << ((cap1 & PCI_VC_CAP1_ARB_SIZE) >> 10); |
425c1b22 AW |
207 | |
208 | /* | |
209 | * Port VC Control Register contains VC Arbitration Select, which | |
210 | * cannot be modified when more than one LPVC is in operation. We | |
211 | * therefore save/restore it first, as only VC0 should be enabled | |
212 | * after device reset. | |
213 | */ | |
214 | if (buf) { | |
215 | if (save) | |
216 | pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, | |
217 | (u16 *)buf); | |
218 | else | |
219 | pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, | |
220 | *(u16 *)buf); | |
ef0dab4a | 221 | buf += 4; |
425c1b22 | 222 | } |
ef0dab4a | 223 | len += 4; |
425c1b22 AW |
224 | |
225 | /* | |
226 | * If we have any Low Priority VCs and a VC Arbitration Table Offset | |
227 | * in Port VC Capability Register 2 then save/restore it next. | |
228 | */ | |
229 | if (lpevcc) { | |
274127a1 | 230 | u32 cap2; |
425c1b22 AW |
231 | int vcarb_offset; |
232 | ||
274127a1 AW |
233 | pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP2, &cap2); |
234 | vcarb_offset = ((cap2 & PCI_VC_CAP2_ARB_OFF) >> 24) * 16; | |
425c1b22 AW |
235 | |
236 | if (vcarb_offset) { | |
237 | int size, vcarb_phases = 0; | |
238 | ||
274127a1 | 239 | if (cap2 & PCI_VC_CAP2_128_PHASE) |
425c1b22 | 240 | vcarb_phases = 128; |
274127a1 | 241 | else if (cap2 & PCI_VC_CAP2_64_PHASE) |
425c1b22 | 242 | vcarb_phases = 64; |
274127a1 | 243 | else if (cap2 & PCI_VC_CAP2_32_PHASE) |
425c1b22 AW |
244 | vcarb_phases = 32; |
245 | ||
246 | /* Fixed 4 bits per phase per lpevcc (plus VC0) */ | |
247 | size = ((lpevcc + 1) * vcarb_phases * 4) / 8; | |
248 | ||
249 | if (size && buf) { | |
250 | pci_vc_save_restore_dwords(dev, | |
251 | pos + vcarb_offset, | |
252 | (u32 *)buf, | |
253 | size / 4, save); | |
254 | /* | |
255 | * On restore, we need to signal hardware to | |
256 | * re-load the VC Arbitration Table. | |
257 | */ | |
258 | if (!save) | |
259 | pci_vc_load_arb_table(dev, pos); | |
260 | ||
261 | buf += size; | |
262 | } | |
263 | len += size; | |
264 | } | |
265 | } | |
266 | ||
267 | /* | |
268 | * In addition to each VC Resource Control Register, we may have a | |
269 | * Port Arbitration Table attached to each VC. The Port Arbitration | |
270 | * Table Offset in each VC Resource Capability Register tells us if | |
271 | * it exists. The entry size is global from the Port VC Capability | |
272 | * Register1 above. The number of phases is determined per VC. | |
273 | */ | |
274 | for (i = 0; i < evcc + 1; i++) { | |
275 | u32 cap; | |
276 | int parb_offset; | |
277 | ||
278 | pci_read_config_dword(dev, pos + PCI_VC_RES_CAP + | |
279 | (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap); | |
280 | parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16; | |
281 | if (parb_offset) { | |
282 | int size, parb_phases = 0; | |
283 | ||
284 | if (cap & PCI_VC_RES_CAP_256_PHASE) | |
285 | parb_phases = 256; | |
286 | else if (cap & (PCI_VC_RES_CAP_128_PHASE | | |
287 | PCI_VC_RES_CAP_128_PHASE_TB)) | |
288 | parb_phases = 128; | |
289 | else if (cap & PCI_VC_RES_CAP_64_PHASE) | |
290 | parb_phases = 64; | |
291 | else if (cap & PCI_VC_RES_CAP_32_PHASE) | |
292 | parb_phases = 32; | |
293 | ||
294 | size = (parb_size * parb_phases) / 8; | |
295 | ||
296 | if (size && buf) { | |
297 | pci_vc_save_restore_dwords(dev, | |
298 | pos + parb_offset, | |
299 | (u32 *)buf, | |
300 | size / 4, save); | |
301 | buf += size; | |
302 | } | |
303 | len += size; | |
304 | } | |
305 | ||
306 | /* VC Resource Control Register */ | |
307 | if (buf) { | |
308 | int ctrl_pos = pos + PCI_VC_RES_CTRL + | |
309 | (i * PCI_CAP_VC_PER_VC_SIZEOF); | |
310 | if (save) | |
311 | pci_read_config_dword(dev, ctrl_pos, | |
312 | (u32 *)buf); | |
313 | else { | |
314 | u32 tmp, ctrl = *(u32 *)buf; | |
315 | /* | |
316 | * For an FLR case, the VC config may remain. | |
317 | * Preserve enable bit, restore the rest. | |
318 | */ | |
319 | pci_read_config_dword(dev, ctrl_pos, &tmp); | |
320 | tmp &= PCI_VC_RES_CTRL_ENABLE; | |
321 | tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE; | |
322 | pci_write_config_dword(dev, ctrl_pos, tmp); | |
323 | /* Load port arbitration table if used */ | |
324 | if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT) | |
325 | pci_vc_load_port_arb_table(dev, pos, i); | |
326 | /* Re-enable if needed */ | |
327 | if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE) | |
328 | pci_vc_enable(dev, pos, i); | |
329 | } | |
330 | buf += 4; | |
331 | } | |
332 | len += 4; | |
333 | } | |
334 | ||
335 | return buf ? 0 : len; | |
336 | } | |
337 | ||
338 | static struct { | |
339 | u16 id; | |
340 | const char *name; | |
341 | } vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" }, | |
342 | { PCI_EXT_CAP_ID_VC, "VC" }, | |
343 | { PCI_EXT_CAP_ID_VC9, "VC9" } }; | |
344 | ||
345 | /** | |
346 | * pci_save_vc_state - Save VC state to pre-allocate save buffer | |
347 | * @dev: device | |
348 | * | |
349 | * For each type of VC capability, VC/VC9/MFVC, find the capability and | |
350 | * save it to the pre-allocated save buffer. | |
351 | */ | |
352 | int pci_save_vc_state(struct pci_dev *dev) | |
353 | { | |
354 | int i; | |
355 | ||
356 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { | |
357 | int pos, ret; | |
358 | struct pci_cap_saved_state *save_state; | |
359 | ||
360 | pos = pci_find_ext_capability(dev, vc_caps[i].id); | |
361 | if (!pos) | |
362 | continue; | |
363 | ||
364 | save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); | |
365 | if (!save_state) { | |
7506dc79 | 366 | pci_err(dev, "%s buffer not found in %s\n", |
425c1b22 AW |
367 | vc_caps[i].name, __func__); |
368 | return -ENOMEM; | |
369 | } | |
370 | ||
371 | ret = pci_vc_do_save_buffer(dev, pos, save_state, true); | |
372 | if (ret) { | |
7506dc79 | 373 | pci_err(dev, "%s save unsuccessful %s\n", |
425c1b22 AW |
374 | vc_caps[i].name, __func__); |
375 | return ret; | |
376 | } | |
377 | } | |
378 | ||
379 | return 0; | |
380 | } | |
381 | ||
382 | /** | |
383 | * pci_restore_vc_state - Restore VC state from save buffer | |
384 | * @dev: device | |
385 | * | |
386 | * For each type of VC capability, VC/VC9/MFVC, find the capability and | |
387 | * restore it from the previously saved buffer. | |
388 | */ | |
389 | void pci_restore_vc_state(struct pci_dev *dev) | |
390 | { | |
391 | int i; | |
392 | ||
393 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { | |
394 | int pos; | |
395 | struct pci_cap_saved_state *save_state; | |
396 | ||
397 | pos = pci_find_ext_capability(dev, vc_caps[i].id); | |
398 | save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); | |
399 | if (!save_state || !pos) | |
400 | continue; | |
401 | ||
402 | pci_vc_do_save_buffer(dev, pos, save_state, false); | |
403 | } | |
404 | } | |
405 | ||
406 | /** | |
407 | * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps | |
408 | * @dev: device | |
409 | * | |
410 | * For each type of VC capability, VC/VC9/MFVC, find the capability, size | |
411 | * it, and allocate a buffer for save/restore. | |
412 | */ | |
425c1b22 AW |
413 | void pci_allocate_vc_save_buffers(struct pci_dev *dev) |
414 | { | |
415 | int i; | |
416 | ||
417 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { | |
418 | int len, pos = pci_find_ext_capability(dev, vc_caps[i].id); | |
419 | ||
420 | if (!pos) | |
421 | continue; | |
422 | ||
423 | len = pci_vc_do_save_buffer(dev, pos, NULL, false); | |
424 | if (pci_add_ext_cap_save_buffer(dev, vc_caps[i].id, len)) | |
7506dc79 | 425 | pci_err(dev, "unable to preallocate %s save buffer\n", |
425c1b22 AW |
426 | vc_caps[i].name); |
427 | } | |
428 | } |