]>
Commit | Line | Data |
---|---|---|
a1b36958 XY |
1 | /* |
2 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
3 | * redistributing this file, you may do so under either license. | |
4 | * | |
5 | * GPL LICENSE SUMMARY | |
6 | * | |
7 | * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. | |
443b9a14 | 8 | * Copyright (C) 2016 T-Platforms. All Rights Reserved. |
a1b36958 XY |
9 | * |
10 | * This program is free software; you can redistribute it and/or modify | |
11 | * it under the terms of version 2 of the GNU General Public License as | |
12 | * published by the Free Software Foundation. | |
13 | * | |
14 | * BSD LICENSE | |
15 | * | |
16 | * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. | |
443b9a14 | 17 | * Copyright (C) 2016 T-Platforms. All Rights Reserved. |
a1b36958 XY |
18 | * |
19 | * Redistribution and use in source and binary forms, with or without | |
20 | * modification, are permitted provided that the following conditions | |
21 | * are met: | |
22 | * | |
23 | * * Redistributions of source code must retain the above copyright | |
24 | * notice, this list of conditions and the following disclaimer. | |
25 | * * Redistributions in binary form must reproduce the above copy | |
26 | * notice, this list of conditions and the following disclaimer in | |
27 | * the documentation and/or other materials provided with the | |
28 | * distribution. | |
29 | * * Neither the name of AMD Corporation nor the names of its | |
30 | * contributors may be used to endorse or promote products derived | |
31 | * from this software without specific prior written permission. | |
32 | * | |
33 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
34 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
35 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
36 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
37 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
38 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
39 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
40 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
41 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
42 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
43 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
44 | * | |
45 | * AMD PCIe NTB Linux driver | |
46 | * | |
47 | * Contact Information: | |
48 | * Xiangliang Yu <Xiangliang.Yu@amd.com> | |
49 | */ | |
50 | ||
51 | #include <linux/debugfs.h> | |
52 | #include <linux/delay.h> | |
53 | #include <linux/init.h> | |
54 | #include <linux/interrupt.h> | |
55 | #include <linux/module.h> | |
56 | #include <linux/acpi.h> | |
57 | #include <linux/pci.h> | |
58 | #include <linux/random.h> | |
59 | #include <linux/slab.h> | |
60 | #include <linux/ntb.h> | |
61 | ||
62 | #include "ntb_hw_amd.h" | |
63 | ||
64 | #define NTB_NAME "ntb_hw_amd" | |
65 | #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver" | |
66 | #define NTB_VER "1.0" | |
67 | ||
68 | MODULE_DESCRIPTION(NTB_DESC); | |
69 | MODULE_VERSION(NTB_VER); | |
70 | MODULE_LICENSE("Dual BSD/GPL"); | |
71 | MODULE_AUTHOR("AMD Inc."); | |
72 | ||
73 | static const struct file_operations amd_ntb_debugfs_info; | |
74 | static struct dentry *debugfs_dir; | |
75 | ||
76 | static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx) | |
77 | { | |
78 | if (idx < 0 || idx > ndev->mw_count) | |
79 | return -EINVAL; | |
80 | ||
81 | return 1 << idx; | |
82 | } | |
83 | ||
443b9a14 | 84 | static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx) |
a1b36958 | 85 | { |
443b9a14 SS |
86 | if (pidx != NTB_DEF_PEER_IDX) |
87 | return -EINVAL; | |
88 | ||
a1b36958 XY |
89 | return ntb_ndev(ntb)->mw_count; |
90 | } | |
91 | ||
443b9a14 SS |
92 | static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, |
93 | resource_size_t *addr_align, | |
94 | resource_size_t *size_align, | |
95 | resource_size_t *size_max) | |
a1b36958 XY |
96 | { |
97 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
98 | int bar; | |
99 | ||
443b9a14 SS |
100 | if (pidx != NTB_DEF_PEER_IDX) |
101 | return -EINVAL; | |
102 | ||
a1b36958 XY |
103 | bar = ndev_mw_to_bar(ndev, idx); |
104 | if (bar < 0) | |
105 | return bar; | |
106 | ||
443b9a14 SS |
107 | if (addr_align) |
108 | *addr_align = SZ_4K; | |
a1b36958 | 109 | |
443b9a14 SS |
110 | if (size_align) |
111 | *size_align = 1; | |
a1b36958 | 112 | |
443b9a14 SS |
113 | if (size_max) |
114 | *size_max = pci_resource_len(ndev->ntb.pdev, bar); | |
a1b36958 XY |
115 | |
116 | return 0; | |
117 | } | |
118 | ||
443b9a14 | 119 | static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, |
a1b36958 XY |
120 | dma_addr_t addr, resource_size_t size) |
121 | { | |
122 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
123 | unsigned long xlat_reg, limit_reg = 0; | |
124 | resource_size_t mw_size; | |
125 | void __iomem *mmio, *peer_mmio; | |
126 | u64 base_addr, limit, reg_val; | |
127 | int bar; | |
128 | ||
443b9a14 SS |
129 | if (pidx != NTB_DEF_PEER_IDX) |
130 | return -EINVAL; | |
131 | ||
a1b36958 XY |
132 | bar = ndev_mw_to_bar(ndev, idx); |
133 | if (bar < 0) | |
134 | return bar; | |
135 | ||
0f9bfb97 | 136 | mw_size = pci_resource_len(ntb->pdev, bar); |
a1b36958 XY |
137 | |
138 | /* make sure the range fits in the usable mw size */ | |
139 | if (size > mw_size) | |
140 | return -EINVAL; | |
141 | ||
142 | mmio = ndev->self_mmio; | |
143 | peer_mmio = ndev->peer_mmio; | |
144 | ||
0f9bfb97 | 145 | base_addr = pci_resource_start(ntb->pdev, bar); |
a1b36958 XY |
146 | |
147 | if (bar != 1) { | |
872deb21 SS |
148 | xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2); |
149 | limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2); | |
a1b36958 XY |
150 | |
151 | /* Set the limit if supported */ | |
872deb21 | 152 | limit = size; |
a1b36958 XY |
153 | |
154 | /* set and verify setting the translation address */ | |
155 | write64(addr, peer_mmio + xlat_reg); | |
156 | reg_val = read64(peer_mmio + xlat_reg); | |
157 | if (reg_val != addr) { | |
158 | write64(0, peer_mmio + xlat_reg); | |
159 | return -EIO; | |
160 | } | |
161 | ||
162 | /* set and verify setting the limit */ | |
a5c3666f SM |
163 | write64(limit, peer_mmio + limit_reg); |
164 | reg_val = read64(peer_mmio + limit_reg); | |
a1b36958 XY |
165 | if (reg_val != limit) { |
166 | write64(base_addr, mmio + limit_reg); | |
167 | write64(0, peer_mmio + xlat_reg); | |
168 | return -EIO; | |
169 | } | |
170 | } else { | |
171 | xlat_reg = AMD_BAR1XLAT_OFFSET; | |
172 | limit_reg = AMD_BAR1LMT_OFFSET; | |
173 | ||
a1b36958 | 174 | /* Set the limit if supported */ |
872deb21 | 175 | limit = size; |
a1b36958 XY |
176 | |
177 | /* set and verify setting the translation address */ | |
178 | write64(addr, peer_mmio + xlat_reg); | |
179 | reg_val = read64(peer_mmio + xlat_reg); | |
180 | if (reg_val != addr) { | |
181 | write64(0, peer_mmio + xlat_reg); | |
182 | return -EIO; | |
183 | } | |
184 | ||
185 | /* set and verify setting the limit */ | |
a5c3666f SM |
186 | writel(limit, peer_mmio + limit_reg); |
187 | reg_val = readl(peer_mmio + limit_reg); | |
a1b36958 XY |
188 | if (reg_val != limit) { |
189 | writel(base_addr, mmio + limit_reg); | |
190 | writel(0, peer_mmio + xlat_reg); | |
191 | return -EIO; | |
192 | } | |
193 | } | |
194 | ||
195 | return 0; | |
196 | } | |
197 | ||
198 | static int amd_link_is_up(struct amd_ntb_dev *ndev) | |
199 | { | |
200 | if (!ndev->peer_sta) | |
201 | return NTB_LNK_STA_ACTIVE(ndev->cntl_sta); | |
202 | ||
e5b0d2d1 XY |
203 | if (ndev->peer_sta & AMD_LINK_UP_EVENT) { |
204 | ndev->peer_sta = 0; | |
205 | return 1; | |
206 | } | |
207 | ||
a1b36958 XY |
208 | /* If peer_sta is reset or D0 event, the ISR has |
209 | * started a timer to check link status of hardware. | |
210 | * So here just clear status bit. And if peer_sta is | |
211 | * D3 or PME_TO, D0/reset event will be happened when | |
212 | * system wakeup/poweron, so do nothing here. | |
213 | */ | |
214 | if (ndev->peer_sta & AMD_PEER_RESET_EVENT) | |
215 | ndev->peer_sta &= ~AMD_PEER_RESET_EVENT; | |
e5b0d2d1 | 216 | else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT)) |
a1b36958 XY |
217 | ndev->peer_sta = 0; |
218 | ||
219 | return 0; | |
220 | } | |
221 | ||
4e8c11b7 | 222 | static u64 amd_ntb_link_is_up(struct ntb_dev *ntb, |
a1b36958 XY |
223 | enum ntb_speed *speed, |
224 | enum ntb_width *width) | |
225 | { | |
226 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
227 | int ret = 0; | |
228 | ||
229 | if (amd_link_is_up(ndev)) { | |
230 | if (speed) | |
231 | *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); | |
232 | if (width) | |
233 | *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); | |
234 | ||
0f9bfb97 | 235 | dev_dbg(&ntb->pdev->dev, "link is up.\n"); |
a1b36958 XY |
236 | |
237 | ret = 1; | |
238 | } else { | |
239 | if (speed) | |
240 | *speed = NTB_SPEED_NONE; | |
241 | if (width) | |
242 | *width = NTB_WIDTH_NONE; | |
243 | ||
0f9bfb97 | 244 | dev_dbg(&ntb->pdev->dev, "link is down.\n"); |
a1b36958 XY |
245 | } |
246 | ||
247 | return ret; | |
248 | } | |
249 | ||
250 | static int amd_ntb_link_enable(struct ntb_dev *ntb, | |
251 | enum ntb_speed max_speed, | |
252 | enum ntb_width max_width) | |
253 | { | |
254 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
255 | void __iomem *mmio = ndev->self_mmio; | |
256 | u32 ntb_ctl; | |
257 | ||
258 | /* Enable event interrupt */ | |
259 | ndev->int_mask &= ~AMD_EVENT_INTMASK; | |
260 | writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); | |
261 | ||
262 | if (ndev->ntb.topo == NTB_TOPO_SEC) | |
263 | return -EINVAL; | |
0f9bfb97 | 264 | dev_dbg(&ntb->pdev->dev, "Enabling Link.\n"); |
a1b36958 XY |
265 | |
266 | ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); | |
267 | ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); | |
268 | writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); | |
269 | ||
270 | return 0; | |
271 | } | |
272 | ||
273 | static int amd_ntb_link_disable(struct ntb_dev *ntb) | |
274 | { | |
275 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
276 | void __iomem *mmio = ndev->self_mmio; | |
277 | u32 ntb_ctl; | |
278 | ||
279 | /* Disable event interrupt */ | |
280 | ndev->int_mask |= AMD_EVENT_INTMASK; | |
281 | writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); | |
282 | ||
283 | if (ndev->ntb.topo == NTB_TOPO_SEC) | |
284 | return -EINVAL; | |
0f9bfb97 | 285 | dev_dbg(&ntb->pdev->dev, "Enabling Link.\n"); |
a1b36958 XY |
286 | |
287 | ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); | |
288 | ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); | |
289 | writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); | |
290 | ||
291 | return 0; | |
292 | } | |
293 | ||
443b9a14 SS |
294 | static int amd_ntb_peer_mw_count(struct ntb_dev *ntb) |
295 | { | |
296 | /* The same as for inbound MWs */ | |
297 | return ntb_ndev(ntb)->mw_count; | |
298 | } | |
299 | ||
300 | static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, | |
301 | phys_addr_t *base, resource_size_t *size) | |
302 | { | |
303 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
304 | int bar; | |
305 | ||
306 | bar = ndev_mw_to_bar(ndev, idx); | |
307 | if (bar < 0) | |
308 | return bar; | |
309 | ||
310 | if (base) | |
311 | *base = pci_resource_start(ndev->ntb.pdev, bar); | |
312 | ||
313 | if (size) | |
314 | *size = pci_resource_len(ndev->ntb.pdev, bar); | |
315 | ||
316 | return 0; | |
317 | } | |
318 | ||
a1b36958 XY |
319 | static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb) |
320 | { | |
321 | return ntb_ndev(ntb)->db_valid_mask; | |
322 | } | |
323 | ||
324 | static int amd_ntb_db_vector_count(struct ntb_dev *ntb) | |
325 | { | |
326 | return ntb_ndev(ntb)->db_count; | |
327 | } | |
328 | ||
329 | static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) | |
330 | { | |
331 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
332 | ||
333 | if (db_vector < 0 || db_vector > ndev->db_count) | |
334 | return 0; | |
335 | ||
1e590dec | 336 | return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector); |
a1b36958 XY |
337 | } |
338 | ||
339 | static u64 amd_ntb_db_read(struct ntb_dev *ntb) | |
340 | { | |
341 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
342 | void __iomem *mmio = ndev->self_mmio; | |
343 | ||
344 | return (u64)readw(mmio + AMD_DBSTAT_OFFSET); | |
345 | } | |
346 | ||
347 | static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) | |
348 | { | |
349 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
350 | void __iomem *mmio = ndev->self_mmio; | |
351 | ||
352 | writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET); | |
353 | ||
354 | return 0; | |
355 | } | |
356 | ||
357 | static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) | |
358 | { | |
359 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
360 | void __iomem *mmio = ndev->self_mmio; | |
361 | unsigned long flags; | |
362 | ||
363 | if (db_bits & ~ndev->db_valid_mask) | |
364 | return -EINVAL; | |
365 | ||
366 | spin_lock_irqsave(&ndev->db_mask_lock, flags); | |
367 | ndev->db_mask |= db_bits; | |
368 | writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); | |
369 | spin_unlock_irqrestore(&ndev->db_mask_lock, flags); | |
370 | ||
371 | return 0; | |
372 | } | |
373 | ||
374 | static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) | |
375 | { | |
376 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
377 | void __iomem *mmio = ndev->self_mmio; | |
378 | unsigned long flags; | |
379 | ||
380 | if (db_bits & ~ndev->db_valid_mask) | |
381 | return -EINVAL; | |
382 | ||
383 | spin_lock_irqsave(&ndev->db_mask_lock, flags); | |
384 | ndev->db_mask &= ~db_bits; | |
385 | writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); | |
386 | spin_unlock_irqrestore(&ndev->db_mask_lock, flags); | |
387 | ||
388 | return 0; | |
389 | } | |
390 | ||
a1b36958 XY |
391 | static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) |
392 | { | |
393 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
394 | void __iomem *mmio = ndev->self_mmio; | |
395 | ||
396 | writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET); | |
397 | ||
398 | return 0; | |
399 | } | |
400 | ||
401 | static int amd_ntb_spad_count(struct ntb_dev *ntb) | |
402 | { | |
403 | return ntb_ndev(ntb)->spad_count; | |
404 | } | |
405 | ||
406 | static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx) | |
407 | { | |
408 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
409 | void __iomem *mmio = ndev->self_mmio; | |
410 | u32 offset; | |
411 | ||
412 | if (idx < 0 || idx >= ndev->spad_count) | |
413 | return 0; | |
414 | ||
415 | offset = ndev->self_spad + (idx << 2); | |
416 | return readl(mmio + AMD_SPAD_OFFSET + offset); | |
417 | } | |
418 | ||
419 | static int amd_ntb_spad_write(struct ntb_dev *ntb, | |
420 | int idx, u32 val) | |
421 | { | |
422 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
423 | void __iomem *mmio = ndev->self_mmio; | |
424 | u32 offset; | |
425 | ||
426 | if (idx < 0 || idx >= ndev->spad_count) | |
427 | return -EINVAL; | |
428 | ||
429 | offset = ndev->self_spad + (idx << 2); | |
430 | writel(val, mmio + AMD_SPAD_OFFSET + offset); | |
431 | ||
432 | return 0; | |
433 | } | |
434 | ||
d67288a3 | 435 | static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) |
a1b36958 XY |
436 | { |
437 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
438 | void __iomem *mmio = ndev->self_mmio; | |
439 | u32 offset; | |
440 | ||
d67288a3 | 441 | if (sidx < 0 || sidx >= ndev->spad_count) |
a1b36958 XY |
442 | return -EINVAL; |
443 | ||
d67288a3 | 444 | offset = ndev->peer_spad + (sidx << 2); |
a1b36958 XY |
445 | return readl(mmio + AMD_SPAD_OFFSET + offset); |
446 | } | |
447 | ||
d67288a3 SS |
448 | static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, |
449 | int sidx, u32 val) | |
a1b36958 XY |
450 | { |
451 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); | |
452 | void __iomem *mmio = ndev->self_mmio; | |
453 | u32 offset; | |
454 | ||
d67288a3 | 455 | if (sidx < 0 || sidx >= ndev->spad_count) |
a1b36958 XY |
456 | return -EINVAL; |
457 | ||
d67288a3 | 458 | offset = ndev->peer_spad + (sidx << 2); |
a1b36958 XY |
459 | writel(val, mmio + AMD_SPAD_OFFSET + offset); |
460 | ||
461 | return 0; | |
462 | } | |
463 | ||
464 | static const struct ntb_dev_ops amd_ntb_ops = { | |
465 | .mw_count = amd_ntb_mw_count, | |
443b9a14 | 466 | .mw_get_align = amd_ntb_mw_get_align, |
a1b36958 | 467 | .mw_set_trans = amd_ntb_mw_set_trans, |
443b9a14 SS |
468 | .peer_mw_count = amd_ntb_peer_mw_count, |
469 | .peer_mw_get_addr = amd_ntb_peer_mw_get_addr, | |
a1b36958 XY |
470 | .link_is_up = amd_ntb_link_is_up, |
471 | .link_enable = amd_ntb_link_enable, | |
472 | .link_disable = amd_ntb_link_disable, | |
473 | .db_valid_mask = amd_ntb_db_valid_mask, | |
474 | .db_vector_count = amd_ntb_db_vector_count, | |
475 | .db_vector_mask = amd_ntb_db_vector_mask, | |
476 | .db_read = amd_ntb_db_read, | |
477 | .db_clear = amd_ntb_db_clear, | |
478 | .db_set_mask = amd_ntb_db_set_mask, | |
479 | .db_clear_mask = amd_ntb_db_clear_mask, | |
a1b36958 XY |
480 | .peer_db_set = amd_ntb_peer_db_set, |
481 | .spad_count = amd_ntb_spad_count, | |
482 | .spad_read = amd_ntb_spad_read, | |
483 | .spad_write = amd_ntb_spad_write, | |
a1b36958 XY |
484 | .peer_spad_read = amd_ntb_peer_spad_read, |
485 | .peer_spad_write = amd_ntb_peer_spad_write, | |
486 | }; | |
487 | ||
488 | static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit) | |
489 | { | |
490 | void __iomem *mmio = ndev->self_mmio; | |
491 | int reg; | |
492 | ||
493 | reg = readl(mmio + AMD_SMUACK_OFFSET); | |
494 | reg |= bit; | |
495 | writel(reg, mmio + AMD_SMUACK_OFFSET); | |
496 | ||
497 | ndev->peer_sta |= bit; | |
498 | } | |
499 | ||
500 | static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) | |
501 | { | |
502 | void __iomem *mmio = ndev->self_mmio; | |
0f9bfb97 | 503 | struct device *dev = &ndev->ntb.pdev->dev; |
a1b36958 XY |
504 | u32 status; |
505 | ||
506 | status = readl(mmio + AMD_INTSTAT_OFFSET); | |
507 | if (!(status & AMD_EVENT_INTMASK)) | |
508 | return; | |
509 | ||
0f9bfb97 | 510 | dev_dbg(dev, "status = 0x%x and vec = %d\n", status, vec); |
a1b36958 XY |
511 | |
512 | status &= AMD_EVENT_INTMASK; | |
513 | switch (status) { | |
514 | case AMD_PEER_FLUSH_EVENT: | |
0f9bfb97 | 515 | dev_info(dev, "Flush is done.\n"); |
a1b36958 XY |
516 | break; |
517 | case AMD_PEER_RESET_EVENT: | |
518 | amd_ack_smu(ndev, AMD_PEER_RESET_EVENT); | |
519 | ||
520 | /* link down first */ | |
521 | ntb_link_event(&ndev->ntb); | |
522 | /* polling peer status */ | |
523 | schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); | |
524 | ||
525 | break; | |
526 | case AMD_PEER_D3_EVENT: | |
527 | case AMD_PEER_PMETO_EVENT: | |
e5b0d2d1 XY |
528 | case AMD_LINK_UP_EVENT: |
529 | case AMD_LINK_DOWN_EVENT: | |
a1b36958 XY |
530 | amd_ack_smu(ndev, status); |
531 | ||
532 | /* link down */ | |
533 | ntb_link_event(&ndev->ntb); | |
534 | ||
535 | break; | |
536 | case AMD_PEER_D0_EVENT: | |
537 | mmio = ndev->peer_mmio; | |
538 | status = readl(mmio + AMD_PMESTAT_OFFSET); | |
539 | /* check if this is WAKEUP event */ | |
540 | if (status & 0x1) | |
0f9bfb97 | 541 | dev_info(dev, "Wakeup is done.\n"); |
a1b36958 XY |
542 | |
543 | amd_ack_smu(ndev, AMD_PEER_D0_EVENT); | |
544 | ||
545 | /* start a timer to poll link status */ | |
546 | schedule_delayed_work(&ndev->hb_timer, | |
547 | AMD_LINK_HB_TIMEOUT); | |
548 | break; | |
549 | default: | |
0f9bfb97 | 550 | dev_info(dev, "event status = 0x%x.\n", status); |
a1b36958 XY |
551 | break; |
552 | } | |
553 | } | |
554 | ||
555 | static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec) | |
556 | { | |
0f9bfb97 | 557 | dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec); |
a1b36958 XY |
558 | |
559 | if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1)) | |
560 | amd_handle_event(ndev, vec); | |
561 | ||
562 | if (vec < AMD_DB_CNT) | |
563 | ntb_db_event(&ndev->ntb, vec); | |
564 | ||
565 | return IRQ_HANDLED; | |
566 | } | |
567 | ||
568 | static irqreturn_t ndev_vec_isr(int irq, void *dev) | |
569 | { | |
570 | struct amd_ntb_vec *nvec = dev; | |
571 | ||
572 | return ndev_interrupt(nvec->ndev, nvec->num); | |
573 | } | |
574 | ||
575 | static irqreturn_t ndev_irq_isr(int irq, void *dev) | |
576 | { | |
577 | struct amd_ntb_dev *ndev = dev; | |
578 | ||
0f9bfb97 | 579 | return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq); |
a1b36958 XY |
580 | } |
581 | ||
582 | static int ndev_init_isr(struct amd_ntb_dev *ndev, | |
583 | int msix_min, int msix_max) | |
584 | { | |
585 | struct pci_dev *pdev; | |
586 | int rc, i, msix_count, node; | |
587 | ||
0f9bfb97 | 588 | pdev = ndev->ntb.pdev; |
a1b36958 XY |
589 | |
590 | node = dev_to_node(&pdev->dev); | |
591 | ||
592 | ndev->db_mask = ndev->db_valid_mask; | |
593 | ||
594 | /* Try to set up msix irq */ | |
590b5b7d | 595 | ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec), |
a1b36958 XY |
596 | GFP_KERNEL, node); |
597 | if (!ndev->vec) | |
598 | goto err_msix_vec_alloc; | |
599 | ||
590b5b7d | 600 | ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix), |
a1b36958 XY |
601 | GFP_KERNEL, node); |
602 | if (!ndev->msix) | |
603 | goto err_msix_alloc; | |
604 | ||
605 | for (i = 0; i < msix_max; ++i) | |
606 | ndev->msix[i].entry = i; | |
607 | ||
608 | msix_count = pci_enable_msix_range(pdev, ndev->msix, | |
609 | msix_min, msix_max); | |
610 | if (msix_count < 0) | |
611 | goto err_msix_enable; | |
612 | ||
613 | /* NOTE: Disable MSIX if msix count is less than 16 because of | |
614 | * hardware limitation. | |
615 | */ | |
616 | if (msix_count < msix_min) { | |
617 | pci_disable_msix(pdev); | |
618 | goto err_msix_enable; | |
619 | } | |
620 | ||
621 | for (i = 0; i < msix_count; ++i) { | |
622 | ndev->vec[i].ndev = ndev; | |
623 | ndev->vec[i].num = i; | |
624 | rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, | |
625 | "ndev_vec_isr", &ndev->vec[i]); | |
626 | if (rc) | |
627 | goto err_msix_request; | |
628 | } | |
629 | ||
0f9bfb97 | 630 | dev_dbg(&pdev->dev, "Using msix interrupts\n"); |
a1b36958 XY |
631 | ndev->db_count = msix_min; |
632 | ndev->msix_vec_count = msix_max; | |
633 | return 0; | |
634 | ||
635 | err_msix_request: | |
636 | while (i-- > 0) | |
28734e8f | 637 | free_irq(ndev->msix[i].vector, &ndev->vec[i]); |
a1b36958 XY |
638 | pci_disable_msix(pdev); |
639 | err_msix_enable: | |
640 | kfree(ndev->msix); | |
641 | err_msix_alloc: | |
642 | kfree(ndev->vec); | |
643 | err_msix_vec_alloc: | |
644 | ndev->msix = NULL; | |
645 | ndev->vec = NULL; | |
646 | ||
647 | /* Try to set up msi irq */ | |
648 | rc = pci_enable_msi(pdev); | |
649 | if (rc) | |
650 | goto err_msi_enable; | |
651 | ||
652 | rc = request_irq(pdev->irq, ndev_irq_isr, 0, | |
653 | "ndev_irq_isr", ndev); | |
654 | if (rc) | |
655 | goto err_msi_request; | |
656 | ||
0f9bfb97 | 657 | dev_dbg(&pdev->dev, "Using msi interrupts\n"); |
a1b36958 XY |
658 | ndev->db_count = 1; |
659 | ndev->msix_vec_count = 1; | |
660 | return 0; | |
661 | ||
662 | err_msi_request: | |
663 | pci_disable_msi(pdev); | |
664 | err_msi_enable: | |
665 | ||
666 | /* Try to set up intx irq */ | |
667 | pci_intx(pdev, 1); | |
668 | ||
669 | rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, | |
670 | "ndev_irq_isr", ndev); | |
671 | if (rc) | |
672 | goto err_intx_request; | |
673 | ||
0f9bfb97 | 674 | dev_dbg(&pdev->dev, "Using intx interrupts\n"); |
a1b36958 XY |
675 | ndev->db_count = 1; |
676 | ndev->msix_vec_count = 1; | |
677 | return 0; | |
678 | ||
679 | err_intx_request: | |
680 | return rc; | |
681 | } | |
682 | ||
683 | static void ndev_deinit_isr(struct amd_ntb_dev *ndev) | |
684 | { | |
685 | struct pci_dev *pdev; | |
686 | void __iomem *mmio = ndev->self_mmio; | |
687 | int i; | |
688 | ||
0f9bfb97 | 689 | pdev = ndev->ntb.pdev; |
a1b36958 XY |
690 | |
691 | /* Mask all doorbell interrupts */ | |
692 | ndev->db_mask = ndev->db_valid_mask; | |
693 | writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET); | |
694 | ||
695 | if (ndev->msix) { | |
696 | i = ndev->msix_vec_count; | |
697 | while (i--) | |
698 | free_irq(ndev->msix[i].vector, &ndev->vec[i]); | |
699 | pci_disable_msix(pdev); | |
700 | kfree(ndev->msix); | |
701 | kfree(ndev->vec); | |
702 | } else { | |
703 | free_irq(pdev->irq, ndev); | |
704 | if (pci_dev_msi_enabled(pdev)) | |
705 | pci_disable_msi(pdev); | |
706 | else | |
707 | pci_intx(pdev, 0); | |
708 | } | |
709 | } | |
710 | ||
711 | static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, | |
712 | size_t count, loff_t *offp) | |
713 | { | |
714 | struct amd_ntb_dev *ndev; | |
715 | void __iomem *mmio; | |
716 | char *buf; | |
717 | size_t buf_size; | |
718 | ssize_t ret, off; | |
719 | union { u64 v64; u32 v32; u16 v16; } u; | |
720 | ||
721 | ndev = filp->private_data; | |
722 | mmio = ndev->self_mmio; | |
723 | ||
724 | buf_size = min(count, 0x800ul); | |
725 | ||
726 | buf = kmalloc(buf_size, GFP_KERNEL); | |
727 | if (!buf) | |
728 | return -ENOMEM; | |
729 | ||
730 | off = 0; | |
731 | ||
732 | off += scnprintf(buf + off, buf_size - off, | |
733 | "NTB Device Information:\n"); | |
734 | ||
735 | off += scnprintf(buf + off, buf_size - off, | |
736 | "Connection Topology -\t%s\n", | |
737 | ntb_topo_string(ndev->ntb.topo)); | |
738 | ||
739 | off += scnprintf(buf + off, buf_size - off, | |
740 | "LNK STA -\t\t%#06x\n", ndev->lnk_sta); | |
741 | ||
742 | if (!amd_link_is_up(ndev)) { | |
743 | off += scnprintf(buf + off, buf_size - off, | |
744 | "Link Status -\t\tDown\n"); | |
745 | } else { | |
746 | off += scnprintf(buf + off, buf_size - off, | |
747 | "Link Status -\t\tUp\n"); | |
748 | off += scnprintf(buf + off, buf_size - off, | |
749 | "Link Speed -\t\tPCI-E Gen %u\n", | |
750 | NTB_LNK_STA_SPEED(ndev->lnk_sta)); | |
751 | off += scnprintf(buf + off, buf_size - off, | |
752 | "Link Width -\t\tx%u\n", | |
753 | NTB_LNK_STA_WIDTH(ndev->lnk_sta)); | |
754 | } | |
755 | ||
756 | off += scnprintf(buf + off, buf_size - off, | |
757 | "Memory Window Count -\t%u\n", ndev->mw_count); | |
758 | off += scnprintf(buf + off, buf_size - off, | |
759 | "Scratchpad Count -\t%u\n", ndev->spad_count); | |
760 | off += scnprintf(buf + off, buf_size - off, | |
761 | "Doorbell Count -\t%u\n", ndev->db_count); | |
762 | off += scnprintf(buf + off, buf_size - off, | |
763 | "MSIX Vector Count -\t%u\n", ndev->msix_vec_count); | |
764 | ||
765 | off += scnprintf(buf + off, buf_size - off, | |
766 | "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); | |
767 | ||
768 | u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET); | |
769 | off += scnprintf(buf + off, buf_size - off, | |
770 | "Doorbell Mask -\t\t\t%#06x\n", u.v32); | |
771 | ||
772 | u.v32 = readl(mmio + AMD_DBSTAT_OFFSET); | |
773 | off += scnprintf(buf + off, buf_size - off, | |
774 | "Doorbell Bell -\t\t\t%#06x\n", u.v32); | |
775 | ||
776 | off += scnprintf(buf + off, buf_size - off, | |
777 | "\nNTB Incoming XLAT:\n"); | |
778 | ||
779 | u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET); | |
780 | off += scnprintf(buf + off, buf_size - off, | |
781 | "XLAT1 -\t\t%#018llx\n", u.v64); | |
782 | ||
783 | u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET); | |
784 | off += scnprintf(buf + off, buf_size - off, | |
785 | "XLAT23 -\t\t%#018llx\n", u.v64); | |
786 | ||
787 | u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET); | |
788 | off += scnprintf(buf + off, buf_size - off, | |
789 | "XLAT45 -\t\t%#018llx\n", u.v64); | |
790 | ||
791 | u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET); | |
792 | off += scnprintf(buf + off, buf_size - off, | |
793 | "LMT1 -\t\t\t%#06x\n", u.v32); | |
794 | ||
795 | u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET); | |
796 | off += scnprintf(buf + off, buf_size - off, | |
797 | "LMT23 -\t\t\t%#018llx\n", u.v64); | |
798 | ||
799 | u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET); | |
800 | off += scnprintf(buf + off, buf_size - off, | |
801 | "LMT45 -\t\t\t%#018llx\n", u.v64); | |
802 | ||
803 | ret = simple_read_from_buffer(ubuf, count, offp, buf, off); | |
804 | kfree(buf); | |
805 | return ret; | |
806 | } | |
807 | ||
808 | static void ndev_init_debugfs(struct amd_ntb_dev *ndev) | |
809 | { | |
810 | if (!debugfs_dir) { | |
811 | ndev->debugfs_dir = NULL; | |
812 | ndev->debugfs_info = NULL; | |
813 | } else { | |
814 | ndev->debugfs_dir = | |
0f9bfb97 LG |
815 | debugfs_create_dir(pci_name(ndev->ntb.pdev), |
816 | debugfs_dir); | |
a1b36958 XY |
817 | if (!ndev->debugfs_dir) |
818 | ndev->debugfs_info = NULL; | |
819 | else | |
820 | ndev->debugfs_info = | |
821 | debugfs_create_file("info", S_IRUSR, | |
822 | ndev->debugfs_dir, ndev, | |
823 | &amd_ntb_debugfs_info); | |
824 | } | |
825 | } | |
826 | ||
827 | static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev) | |
828 | { | |
829 | debugfs_remove_recursive(ndev->debugfs_dir); | |
830 | } | |
831 | ||
832 | static inline void ndev_init_struct(struct amd_ntb_dev *ndev, | |
833 | struct pci_dev *pdev) | |
834 | { | |
835 | ndev->ntb.pdev = pdev; | |
836 | ndev->ntb.topo = NTB_TOPO_NONE; | |
837 | ndev->ntb.ops = &amd_ntb_ops; | |
838 | ndev->int_mask = AMD_EVENT_INTMASK; | |
839 | spin_lock_init(&ndev->db_mask_lock); | |
840 | } | |
841 | ||
842 | static int amd_poll_link(struct amd_ntb_dev *ndev) | |
843 | { | |
844 | void __iomem *mmio = ndev->peer_mmio; | |
845 | u32 reg, stat; | |
846 | int rc; | |
847 | ||
848 | reg = readl(mmio + AMD_SIDEINFO_OFFSET); | |
849 | reg &= NTB_LIN_STA_ACTIVE_BIT; | |
850 | ||
0f9bfb97 | 851 | dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg); |
a1b36958 XY |
852 | |
853 | if (reg == ndev->cntl_sta) | |
854 | return 0; | |
855 | ||
856 | ndev->cntl_sta = reg; | |
857 | ||
858 | rc = pci_read_config_dword(ndev->ntb.pdev, | |
859 | AMD_LINK_STATUS_OFFSET, &stat); | |
860 | if (rc) | |
861 | return 0; | |
862 | ndev->lnk_sta = stat; | |
863 | ||
864 | return 1; | |
865 | } | |
866 | ||
867 | static void amd_link_hb(struct work_struct *work) | |
868 | { | |
869 | struct amd_ntb_dev *ndev = hb_ndev(work); | |
870 | ||
871 | if (amd_poll_link(ndev)) | |
872 | ntb_link_event(&ndev->ntb); | |
873 | ||
874 | if (!amd_link_is_up(ndev)) | |
875 | schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); | |
876 | } | |
877 | ||
878 | static int amd_init_isr(struct amd_ntb_dev *ndev) | |
879 | { | |
880 | return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT); | |
881 | } | |
882 | ||
883 | static void amd_init_side_info(struct amd_ntb_dev *ndev) | |
884 | { | |
885 | void __iomem *mmio = ndev->self_mmio; | |
886 | unsigned int reg; | |
887 | ||
888 | reg = readl(mmio + AMD_SIDEINFO_OFFSET); | |
889 | if (!(reg & AMD_SIDE_READY)) { | |
890 | reg |= AMD_SIDE_READY; | |
891 | writel(reg, mmio + AMD_SIDEINFO_OFFSET); | |
892 | } | |
893 | } | |
894 | ||
895 | static void amd_deinit_side_info(struct amd_ntb_dev *ndev) | |
896 | { | |
897 | void __iomem *mmio = ndev->self_mmio; | |
898 | unsigned int reg; | |
899 | ||
900 | reg = readl(mmio + AMD_SIDEINFO_OFFSET); | |
901 | if (reg & AMD_SIDE_READY) { | |
902 | reg &= ~AMD_SIDE_READY; | |
903 | writel(reg, mmio + AMD_SIDEINFO_OFFSET); | |
904 | readl(mmio + AMD_SIDEINFO_OFFSET); | |
905 | } | |
906 | } | |
907 | ||
908 | static int amd_init_ntb(struct amd_ntb_dev *ndev) | |
909 | { | |
910 | void __iomem *mmio = ndev->self_mmio; | |
911 | ||
912 | ndev->mw_count = AMD_MW_CNT; | |
913 | ndev->spad_count = AMD_SPADS_CNT; | |
914 | ndev->db_count = AMD_DB_CNT; | |
915 | ||
916 | switch (ndev->ntb.topo) { | |
917 | case NTB_TOPO_PRI: | |
918 | case NTB_TOPO_SEC: | |
919 | ndev->spad_count >>= 1; | |
920 | if (ndev->ntb.topo == NTB_TOPO_PRI) { | |
921 | ndev->self_spad = 0; | |
922 | ndev->peer_spad = 0x20; | |
923 | } else { | |
924 | ndev->self_spad = 0x20; | |
925 | ndev->peer_spad = 0; | |
926 | } | |
927 | ||
928 | INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb); | |
929 | schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); | |
930 | ||
931 | break; | |
932 | default: | |
0f9bfb97 LG |
933 | dev_err(&ndev->ntb.pdev->dev, |
934 | "AMD NTB does not support B2B mode.\n"); | |
a1b36958 XY |
935 | return -EINVAL; |
936 | } | |
937 | ||
938 | ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; | |
939 | ||
940 | /* Mask event interrupts */ | |
941 | writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); | |
942 | ||
943 | return 0; | |
944 | } | |
945 | ||
946 | static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev) | |
947 | { | |
948 | void __iomem *mmio = ndev->self_mmio; | |
949 | u32 info; | |
950 | ||
951 | info = readl(mmio + AMD_SIDEINFO_OFFSET); | |
952 | if (info & AMD_SIDE_MASK) | |
953 | return NTB_TOPO_SEC; | |
954 | else | |
955 | return NTB_TOPO_PRI; | |
956 | } | |
957 | ||
958 | static int amd_init_dev(struct amd_ntb_dev *ndev) | |
959 | { | |
960 | struct pci_dev *pdev; | |
961 | int rc = 0; | |
962 | ||
0f9bfb97 | 963 | pdev = ndev->ntb.pdev; |
a1b36958 XY |
964 | |
965 | ndev->ntb.topo = amd_get_topo(ndev); | |
0f9bfb97 | 966 | dev_dbg(&pdev->dev, "AMD NTB topo is %s\n", |
a1b36958 XY |
967 | ntb_topo_string(ndev->ntb.topo)); |
968 | ||
969 | rc = amd_init_ntb(ndev); | |
970 | if (rc) | |
971 | return rc; | |
972 | ||
973 | rc = amd_init_isr(ndev); | |
974 | if (rc) { | |
0f9bfb97 | 975 | dev_err(&pdev->dev, "fail to init isr.\n"); |
a1b36958 XY |
976 | return rc; |
977 | } | |
978 | ||
979 | ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; | |
980 | ||
981 | return 0; | |
982 | } | |
983 | ||
984 | static void amd_deinit_dev(struct amd_ntb_dev *ndev) | |
985 | { | |
986 | cancel_delayed_work_sync(&ndev->hb_timer); | |
987 | ||
988 | ndev_deinit_isr(ndev); | |
989 | } | |
990 | ||
991 | static int amd_ntb_init_pci(struct amd_ntb_dev *ndev, | |
992 | struct pci_dev *pdev) | |
993 | { | |
994 | int rc; | |
995 | ||
996 | pci_set_drvdata(pdev, ndev); | |
997 | ||
998 | rc = pci_enable_device(pdev); | |
999 | if (rc) | |
1000 | goto err_pci_enable; | |
1001 | ||
1002 | rc = pci_request_regions(pdev, NTB_NAME); | |
1003 | if (rc) | |
1004 | goto err_pci_regions; | |
1005 | ||
1006 | pci_set_master(pdev); | |
1007 | ||
1008 | rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); | |
1009 | if (rc) { | |
1010 | rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); | |
1011 | if (rc) | |
1012 | goto err_dma_mask; | |
0f9bfb97 | 1013 | dev_warn(&pdev->dev, "Cannot DMA highmem\n"); |
a1b36958 XY |
1014 | } |
1015 | ||
1016 | rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); | |
1017 | if (rc) { | |
1018 | rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); | |
1019 | if (rc) | |
1020 | goto err_dma_mask; | |
0f9bfb97 | 1021 | dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); |
a1b36958 | 1022 | } |
417cf39c SS |
1023 | rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev, |
1024 | dma_get_mask(&pdev->dev)); | |
1025 | if (rc) | |
1026 | goto err_dma_mask; | |
a1b36958 XY |
1027 | |
1028 | ndev->self_mmio = pci_iomap(pdev, 0, 0); | |
1029 | if (!ndev->self_mmio) { | |
1030 | rc = -EIO; | |
1031 | goto err_dma_mask; | |
1032 | } | |
1033 | ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET; | |
1034 | ||
1035 | return 0; | |
1036 | ||
1037 | err_dma_mask: | |
1038 | pci_clear_master(pdev); | |
1039 | err_pci_regions: | |
1040 | pci_disable_device(pdev); | |
1041 | err_pci_enable: | |
1042 | pci_set_drvdata(pdev, NULL); | |
1043 | return rc; | |
1044 | } | |
1045 | ||
1046 | static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev) | |
1047 | { | |
0f9bfb97 | 1048 | struct pci_dev *pdev = ndev->ntb.pdev; |
a1b36958 XY |
1049 | |
1050 | pci_iounmap(pdev, ndev->self_mmio); | |
1051 | ||
1052 | pci_clear_master(pdev); | |
1053 | pci_release_regions(pdev); | |
1054 | pci_disable_device(pdev); | |
1055 | pci_set_drvdata(pdev, NULL); | |
1056 | } | |
1057 | ||
1058 | static int amd_ntb_pci_probe(struct pci_dev *pdev, | |
1059 | const struct pci_device_id *id) | |
1060 | { | |
1061 | struct amd_ntb_dev *ndev; | |
1062 | int rc, node; | |
1063 | ||
1064 | node = dev_to_node(&pdev->dev); | |
1065 | ||
1066 | ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); | |
1067 | if (!ndev) { | |
1068 | rc = -ENOMEM; | |
1069 | goto err_ndev; | |
1070 | } | |
1071 | ||
1072 | ndev_init_struct(ndev, pdev); | |
1073 | ||
1074 | rc = amd_ntb_init_pci(ndev, pdev); | |
1075 | if (rc) | |
1076 | goto err_init_pci; | |
1077 | ||
1078 | rc = amd_init_dev(ndev); | |
1079 | if (rc) | |
1080 | goto err_init_dev; | |
1081 | ||
1082 | /* write side info */ | |
1083 | amd_init_side_info(ndev); | |
1084 | ||
1085 | amd_poll_link(ndev); | |
1086 | ||
1087 | ndev_init_debugfs(ndev); | |
1088 | ||
1089 | rc = ntb_register_device(&ndev->ntb); | |
1090 | if (rc) | |
1091 | goto err_register; | |
1092 | ||
1093 | dev_info(&pdev->dev, "NTB device registered.\n"); | |
1094 | ||
1095 | return 0; | |
1096 | ||
1097 | err_register: | |
1098 | ndev_deinit_debugfs(ndev); | |
1099 | amd_deinit_dev(ndev); | |
1100 | err_init_dev: | |
1101 | amd_ntb_deinit_pci(ndev); | |
1102 | err_init_pci: | |
1103 | kfree(ndev); | |
1104 | err_ndev: | |
1105 | return rc; | |
1106 | } | |
1107 | ||
1108 | static void amd_ntb_pci_remove(struct pci_dev *pdev) | |
1109 | { | |
1110 | struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); | |
1111 | ||
1112 | ntb_unregister_device(&ndev->ntb); | |
1113 | ndev_deinit_debugfs(ndev); | |
1114 | amd_deinit_side_info(ndev); | |
1115 | amd_deinit_dev(ndev); | |
1116 | amd_ntb_deinit_pci(ndev); | |
1117 | kfree(ndev); | |
1118 | } | |
1119 | ||
1120 | static const struct file_operations amd_ntb_debugfs_info = { | |
1121 | .owner = THIS_MODULE, | |
1122 | .open = simple_open, | |
1123 | .read = ndev_debugfs_read, | |
1124 | }; | |
1125 | ||
1126 | static const struct pci_device_id amd_ntb_pci_tbl[] = { | |
1127 | {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)}, | |
1128 | {0} | |
1129 | }; | |
1130 | MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl); | |
1131 | ||
1132 | static struct pci_driver amd_ntb_pci_driver = { | |
1133 | .name = KBUILD_MODNAME, | |
1134 | .id_table = amd_ntb_pci_tbl, | |
1135 | .probe = amd_ntb_pci_probe, | |
1136 | .remove = amd_ntb_pci_remove, | |
1137 | }; | |
1138 | ||
1139 | static int __init amd_ntb_pci_driver_init(void) | |
1140 | { | |
1141 | pr_info("%s %s\n", NTB_DESC, NTB_VER); | |
1142 | ||
1143 | if (debugfs_initialized()) | |
1144 | debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); | |
1145 | ||
1146 | return pci_register_driver(&amd_ntb_pci_driver); | |
1147 | } | |
1148 | module_init(amd_ntb_pci_driver_init); | |
1149 | ||
1150 | static void __exit amd_ntb_pci_driver_exit(void) | |
1151 | { | |
1152 | pci_unregister_driver(&amd_ntb_pci_driver); | |
1153 | debugfs_remove_recursive(debugfs_dir); | |
1154 | } | |
1155 | module_exit(amd_ntb_pci_driver_exit); |