]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/ntb/hw/amd/ntb_hw_amd.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'scsi-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi
[mirror_ubuntu-artful-kernel.git] / drivers / ntb / hw / amd / ntb_hw_amd.c
CommitLineData
a1b36958
XY		 1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
443b9a14 8 * Copyright (C) 2016 T-Platforms. All Rights Reserved.
a1b36958
XY		 9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
13 *
14 * BSD LICENSE
15 *
16 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
443b9a14 17 * Copyright (C) 2016 T-Platforms. All Rights Reserved.
a1b36958
XY		 18 *
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions
21 * are met:
22 *
23 * * Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * * Redistributions in binary form must reproduce the above copy
26 * notice, this list of conditions and the following disclaimer in
27 * the documentation and/or other materials provided with the
28 * distribution.
29 * * Neither the name of AMD Corporation nor the names of its
30 * contributors may be used to endorse or promote products derived
31 * from this software without specific prior written permission.
32 *
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44 *
45 * AMD PCIe NTB Linux driver
46 *
47 * Contact Information:
48 * Xiangliang Yu <Xiangliang.Yu@amd.com>
49 */
50
51#include <linux/debugfs.h>
52#include <linux/delay.h>
53#include <linux/init.h>
54#include <linux/interrupt.h>
55#include <linux/module.h>
56#include <linux/acpi.h>
57#include <linux/pci.h>
58#include <linux/random.h>
59#include <linux/slab.h>
60#include <linux/ntb.h>
61
62#include "ntb_hw_amd.h"
63
64#define NTB_NAME "ntb_hw_amd"
65#define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver"
66#define NTB_VER "1.0"
67
68MODULE_DESCRIPTION(NTB_DESC);
69MODULE_VERSION(NTB_VER);
70MODULE_LICENSE("Dual BSD/GPL");
71MODULE_AUTHOR("AMD Inc.");
72
73static const struct file_operations amd_ntb_debugfs_info;
74static struct dentry *debugfs_dir;
75
76static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
77{
78 if (idx < 0 || idx > ndev->mw_count)
79 return -EINVAL;
80
81 return 1 << idx;
82}
83
443b9a14 84static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
a1b36958 85{
443b9a14
SS		 86 if (pidx != NTB_DEF_PEER_IDX)
87 return -EINVAL;
88
a1b36958
XY		 89 return ntb_ndev(ntb)->mw_count;
90}
91
443b9a14
SS		 92static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
93 resource_size_t *addr_align,
94 resource_size_t *size_align,
95 resource_size_t *size_max)
a1b36958
XY		 96{
97 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
98 int bar;
99
443b9a14
SS		 100 if (pidx != NTB_DEF_PEER_IDX)
101 return -EINVAL;
102
a1b36958
XY		 103 bar = ndev_mw_to_bar(ndev, idx);
104 if (bar < 0)
105 return bar;
106
443b9a14
SS		 107 if (addr_align)
108 *addr_align = SZ_4K;
a1b36958 109
443b9a14
SS		 110 if (size_align)
111 *size_align = 1;
a1b36958 112
443b9a14
SS		 113 if (size_max)
114 *size_max = pci_resource_len(ndev->ntb.pdev, bar);
a1b36958
XY		 115
116 return 0;
117}
118
443b9a14 119static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
a1b36958
XY		 120 dma_addr_t addr, resource_size_t size)
121{
122 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
123 unsigned long xlat_reg, limit_reg = 0;
124 resource_size_t mw_size;
125 void __iomem *mmio, *peer_mmio;
126 u64 base_addr, limit, reg_val;
127 int bar;
128
443b9a14
SS		 129 if (pidx != NTB_DEF_PEER_IDX)
130 return -EINVAL;
131
a1b36958
XY		 132 bar = ndev_mw_to_bar(ndev, idx);
133 if (bar < 0)
134 return bar;
135
0f9bfb97 136 mw_size = pci_resource_len(ntb->pdev, bar);
a1b36958
XY		 137
138 /* make sure the range fits in the usable mw size */
139 if (size > mw_size)
140 return -EINVAL;
141
142 mmio = ndev->self_mmio;
143 peer_mmio = ndev->peer_mmio;
144
0f9bfb97 145 base_addr = pci_resource_start(ntb->pdev, bar);
a1b36958
XY		 146
147 if (bar != 1) {
872deb21
SS		 148 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2);
149 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2);
a1b36958
XY		 150
151 /* Set the limit if supported */
872deb21 152 limit = size;
a1b36958
XY		 153
154 /* set and verify setting the translation address */
155 write64(addr, peer_mmio + xlat_reg);
156 reg_val = read64(peer_mmio + xlat_reg);
157 if (reg_val != addr) {
158 write64(0, peer_mmio + xlat_reg);
159 return -EIO;
160 }
161
162 /* set and verify setting the limit */
163 write64(limit, mmio + limit_reg);
164 reg_val = read64(mmio + limit_reg);
165 if (reg_val != limit) {
166 write64(base_addr, mmio + limit_reg);
167 write64(0, peer_mmio + xlat_reg);
168 return -EIO;
169 }
170 } else {
171 xlat_reg = AMD_BAR1XLAT_OFFSET;
172 limit_reg = AMD_BAR1LMT_OFFSET;
173
a1b36958 174 /* Set the limit if supported */
872deb21 175 limit = size;
a1b36958
XY		 176
177 /* set and verify setting the translation address */
178 write64(addr, peer_mmio + xlat_reg);
179 reg_val = read64(peer_mmio + xlat_reg);
180 if (reg_val != addr) {
181 write64(0, peer_mmio + xlat_reg);
182 return -EIO;
183 }
184
185 /* set and verify setting the limit */
186 writel(limit, mmio + limit_reg);
187 reg_val = readl(mmio + limit_reg);
188 if (reg_val != limit) {
189 writel(base_addr, mmio + limit_reg);
190 writel(0, peer_mmio + xlat_reg);
191 return -EIO;
192 }
193 }
194
195 return 0;
196}
197
198static int amd_link_is_up(struct amd_ntb_dev *ndev)
199{
200 if (!ndev->peer_sta)
201 return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
202
e5b0d2d1
XY		 203 if (ndev->peer_sta & AMD_LINK_UP_EVENT) {
204 ndev->peer_sta = 0;
205 return 1;
206 }
207
a1b36958
XY		 208 /* If peer_sta is reset or D0 event, the ISR has
209 * started a timer to check link status of hardware.
210 * So here just clear status bit. And if peer_sta is
211 * D3 or PME_TO, D0/reset event will be happened when
212 * system wakeup/poweron, so do nothing here.
213 */
214 if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
215 ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
e5b0d2d1 216 else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT))
a1b36958
XY		 217 ndev->peer_sta = 0;
218
219 return 0;
220}
221
4e8c11b7 222static u64 amd_ntb_link_is_up(struct ntb_dev *ntb,
a1b36958
XY		 223 enum ntb_speed *speed,
224 enum ntb_width *width)
225{
226 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
227 int ret = 0;
228
229 if (amd_link_is_up(ndev)) {
230 if (speed)
231 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
232 if (width)
233 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
234
0f9bfb97 235 dev_dbg(&ntb->pdev->dev, "link is up.\n");
a1b36958
XY		 236
237 ret = 1;
238 } else {
239 if (speed)
240 *speed = NTB_SPEED_NONE;
241 if (width)
242 *width = NTB_WIDTH_NONE;
243
0f9bfb97 244 dev_dbg(&ntb->pdev->dev, "link is down.\n");
a1b36958
XY		 245 }
246
247 return ret;
248}
249
250static int amd_ntb_link_enable(struct ntb_dev *ntb,
251 enum ntb_speed max_speed,
252 enum ntb_width max_width)
253{
254 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
255 void __iomem *mmio = ndev->self_mmio;
256 u32 ntb_ctl;
257
258 /* Enable event interrupt */
259 ndev->int_mask &= ~AMD_EVENT_INTMASK;
260 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
261
262 if (ndev->ntb.topo == NTB_TOPO_SEC)
263 return -EINVAL;
0f9bfb97 264 dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
a1b36958
XY		 265
266 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
267 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
268 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
269
270 return 0;
271}
272
273static int amd_ntb_link_disable(struct ntb_dev *ntb)
274{
275 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
276 void __iomem *mmio = ndev->self_mmio;
277 u32 ntb_ctl;
278
279 /* Disable event interrupt */
280 ndev->int_mask |= AMD_EVENT_INTMASK;
281 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
282
283 if (ndev->ntb.topo == NTB_TOPO_SEC)
284 return -EINVAL;
0f9bfb97 285 dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
a1b36958
XY		 286
287 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
288 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
289 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
290
291 return 0;
292}
293
443b9a14
SS		 294static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
295{
296 /* The same as for inbound MWs */
297 return ntb_ndev(ntb)->mw_count;
298}
299
300static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
301 phys_addr_t *base, resource_size_t *size)
302{
303 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
304 int bar;
305
306 bar = ndev_mw_to_bar(ndev, idx);
307 if (bar < 0)
308 return bar;
309
310 if (base)
311 *base = pci_resource_start(ndev->ntb.pdev, bar);
312
313 if (size)
314 *size = pci_resource_len(ndev->ntb.pdev, bar);
315
316 return 0;
317}
318
a1b36958
XY		 319static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
320{
321 return ntb_ndev(ntb)->db_valid_mask;
322}
323
324static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
325{
326 return ntb_ndev(ntb)->db_count;
327}
328
329static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
330{
331 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
332
333 if (db_vector < 0 || db_vector > ndev->db_count)
334 return 0;
335
336 return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector);
337}
338
339static u64 amd_ntb_db_read(struct ntb_dev *ntb)
340{
341 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
342 void __iomem *mmio = ndev->self_mmio;
343
344 return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
345}
346
347static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
348{
349 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
350 void __iomem *mmio = ndev->self_mmio;
351
352 writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
353
354 return 0;
355}
356
357static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
358{
359 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
360 void __iomem *mmio = ndev->self_mmio;
361 unsigned long flags;
362
363 if (db_bits & ~ndev->db_valid_mask)
364 return -EINVAL;
365
366 spin_lock_irqsave(&ndev->db_mask_lock, flags);
367 ndev->db_mask |= db_bits;
368 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
369 spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
370
371 return 0;
372}
373
374static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
375{
376 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
377 void __iomem *mmio = ndev->self_mmio;
378 unsigned long flags;
379
380 if (db_bits & ~ndev->db_valid_mask)
381 return -EINVAL;
382
383 spin_lock_irqsave(&ndev->db_mask_lock, flags);
384 ndev->db_mask &= ~db_bits;
385 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
386 spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
387
388 return 0;
389}
390
a1b36958
XY		 391static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
392{
393 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
394 void __iomem *mmio = ndev->self_mmio;
395
396 writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
397
398 return 0;
399}
400
401static int amd_ntb_spad_count(struct ntb_dev *ntb)
402{
403 return ntb_ndev(ntb)->spad_count;
404}
405
406static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
407{
408 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
409 void __iomem *mmio = ndev->self_mmio;
410 u32 offset;
411
412 if (idx < 0 || idx >= ndev->spad_count)
413 return 0;
414
415 offset = ndev->self_spad + (idx << 2);
416 return readl(mmio + AMD_SPAD_OFFSET + offset);
417}
418
419static int amd_ntb_spad_write(struct ntb_dev *ntb,
420 int idx, u32 val)
421{
422 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
423 void __iomem *mmio = ndev->self_mmio;
424 u32 offset;
425
426 if (idx < 0 || idx >= ndev->spad_count)
427 return -EINVAL;
428
429 offset = ndev->self_spad + (idx << 2);
430 writel(val, mmio + AMD_SPAD_OFFSET + offset);
431
432 return 0;
433}
434
d67288a3 435static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
a1b36958
XY		 436{
437 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
438 void __iomem *mmio = ndev->self_mmio;
439 u32 offset;
440
d67288a3 441 if (sidx < 0 || sidx >= ndev->spad_count)
a1b36958
XY		 442 return -EINVAL;
443
d67288a3 444 offset = ndev->peer_spad + (sidx << 2);
a1b36958
XY		 445 return readl(mmio + AMD_SPAD_OFFSET + offset);
446}
447
d67288a3
SS		 448static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
449 int sidx, u32 val)
a1b36958
XY		 450{
451 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
452 void __iomem *mmio = ndev->self_mmio;
453 u32 offset;
454
d67288a3 455 if (sidx < 0 || sidx >= ndev->spad_count)
a1b36958
XY		 456 return -EINVAL;
457
d67288a3 458 offset = ndev->peer_spad + (sidx << 2);
a1b36958
XY		 459 writel(val, mmio + AMD_SPAD_OFFSET + offset);
460
461 return 0;
462}
463
464static const struct ntb_dev_ops amd_ntb_ops = {
465 .mw_count = amd_ntb_mw_count,
443b9a14 466 .mw_get_align = amd_ntb_mw_get_align,
a1b36958 467 .mw_set_trans = amd_ntb_mw_set_trans,
443b9a14
SS		 468 .peer_mw_count = amd_ntb_peer_mw_count,
469 .peer_mw_get_addr = amd_ntb_peer_mw_get_addr,
a1b36958
XY		 470 .link_is_up = amd_ntb_link_is_up,
471 .link_enable = amd_ntb_link_enable,
472 .link_disable = amd_ntb_link_disable,
473 .db_valid_mask = amd_ntb_db_valid_mask,
474 .db_vector_count = amd_ntb_db_vector_count,
475 .db_vector_mask = amd_ntb_db_vector_mask,
476 .db_read = amd_ntb_db_read,
477 .db_clear = amd_ntb_db_clear,
478 .db_set_mask = amd_ntb_db_set_mask,
479 .db_clear_mask = amd_ntb_db_clear_mask,
a1b36958
XY		 480 .peer_db_set = amd_ntb_peer_db_set,
481 .spad_count = amd_ntb_spad_count,
482 .spad_read = amd_ntb_spad_read,
483 .spad_write = amd_ntb_spad_write,
a1b36958
XY		 484 .peer_spad_read = amd_ntb_peer_spad_read,
485 .peer_spad_write = amd_ntb_peer_spad_write,
486};
487
488static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
489{
490 void __iomem *mmio = ndev->self_mmio;
491 int reg;
492
493 reg = readl(mmio + AMD_SMUACK_OFFSET);
494 reg |= bit;
495 writel(reg, mmio + AMD_SMUACK_OFFSET);
496
497 ndev->peer_sta |= bit;
498}
499
500static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
501{
502 void __iomem *mmio = ndev->self_mmio;
0f9bfb97 503 struct device *dev = &ndev->ntb.pdev->dev;
a1b36958
XY		 504 u32 status;
505
506 status = readl(mmio + AMD_INTSTAT_OFFSET);
507 if (!(status & AMD_EVENT_INTMASK))
508 return;
509
0f9bfb97 510 dev_dbg(dev, "status = 0x%x and vec = %d\n", status, vec);
a1b36958
XY		 511
512 status &= AMD_EVENT_INTMASK;
513 switch (status) {
514 case AMD_PEER_FLUSH_EVENT:
0f9bfb97 515 dev_info(dev, "Flush is done.\n");
a1b36958
XY		 516 break;
517 case AMD_PEER_RESET_EVENT:
518 amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
519
520 /* link down first */
521 ntb_link_event(&ndev->ntb);
522 /* polling peer status */
523 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
524
525 break;
526 case AMD_PEER_D3_EVENT:
527 case AMD_PEER_PMETO_EVENT:
e5b0d2d1
XY		 528 case AMD_LINK_UP_EVENT:
529 case AMD_LINK_DOWN_EVENT:
a1b36958
XY		 530 amd_ack_smu(ndev, status);
531
532 /* link down */
533 ntb_link_event(&ndev->ntb);
534
535 break;
536 case AMD_PEER_D0_EVENT:
537 mmio = ndev->peer_mmio;
538 status = readl(mmio + AMD_PMESTAT_OFFSET);
539 /* check if this is WAKEUP event */
540 if (status & 0x1)
0f9bfb97 541 dev_info(dev, "Wakeup is done.\n");
a1b36958
XY		 542
543 amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
544
545 /* start a timer to poll link status */
546 schedule_delayed_work(&ndev->hb_timer,
547 AMD_LINK_HB_TIMEOUT);
548 break;
549 default:
0f9bfb97 550 dev_info(dev, "event status = 0x%x.\n", status);
a1b36958
XY		 551 break;
552 }
553}
554
555static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
556{
0f9bfb97 557 dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec);
a1b36958
XY		 558
559 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
560 amd_handle_event(ndev, vec);
561
562 if (vec < AMD_DB_CNT)
563 ntb_db_event(&ndev->ntb, vec);
564
565 return IRQ_HANDLED;
566}
567
568static irqreturn_t ndev_vec_isr(int irq, void *dev)
569{
570 struct amd_ntb_vec *nvec = dev;
571
572 return ndev_interrupt(nvec->ndev, nvec->num);
573}
574
575static irqreturn_t ndev_irq_isr(int irq, void *dev)
576{
577 struct amd_ntb_dev *ndev = dev;
578
0f9bfb97 579 return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
a1b36958
XY		 580}
581
582static int ndev_init_isr(struct amd_ntb_dev *ndev,
583 int msix_min, int msix_max)
584{
585 struct pci_dev *pdev;
586 int rc, i, msix_count, node;
587
0f9bfb97 588 pdev = ndev->ntb.pdev;
a1b36958
XY		 589
590 node = dev_to_node(&pdev->dev);
591
592 ndev->db_mask = ndev->db_valid_mask;
593
594 /* Try to set up msix irq */
595 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
596 GFP_KERNEL, node);
597 if (!ndev->vec)
598 goto err_msix_vec_alloc;
599
600 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
601 GFP_KERNEL, node);
602 if (!ndev->msix)
603 goto err_msix_alloc;
604
605 for (i = 0; i < msix_max; ++i)
606 ndev->msix[i].entry = i;
607
608 msix_count = pci_enable_msix_range(pdev, ndev->msix,
609 msix_min, msix_max);
610 if (msix_count < 0)
611 goto err_msix_enable;
612
613 /* NOTE: Disable MSIX if msix count is less than 16 because of
614 * hardware limitation.
615 */
616 if (msix_count < msix_min) {
617 pci_disable_msix(pdev);
618 goto err_msix_enable;
619 }
620
621 for (i = 0; i < msix_count; ++i) {
622 ndev->vec[i].ndev = ndev;
623 ndev->vec[i].num = i;
624 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
625 "ndev_vec_isr", &ndev->vec[i]);
626 if (rc)
627 goto err_msix_request;
628 }
629
0f9bfb97 630 dev_dbg(&pdev->dev, "Using msix interrupts\n");
a1b36958
XY		 631 ndev->db_count = msix_min;
632 ndev->msix_vec_count = msix_max;
633 return 0;
634
635err_msix_request:
636 while (i-- > 0)
28734e8f 637 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
a1b36958
XY		 638 pci_disable_msix(pdev);
639err_msix_enable:
640 kfree(ndev->msix);
641err_msix_alloc:
642 kfree(ndev->vec);
643err_msix_vec_alloc:
644 ndev->msix = NULL;
645 ndev->vec = NULL;
646
647 /* Try to set up msi irq */
648 rc = pci_enable_msi(pdev);
649 if (rc)
650 goto err_msi_enable;
651
652 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
653 "ndev_irq_isr", ndev);
654 if (rc)
655 goto err_msi_request;
656
0f9bfb97 657 dev_dbg(&pdev->dev, "Using msi interrupts\n");
a1b36958
XY		 658 ndev->db_count = 1;
659 ndev->msix_vec_count = 1;
660 return 0;
661
662err_msi_request:
663 pci_disable_msi(pdev);
664err_msi_enable:
665
666 /* Try to set up intx irq */
667 pci_intx(pdev, 1);
668
669 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
670 "ndev_irq_isr", ndev);
671 if (rc)
672 goto err_intx_request;
673
0f9bfb97 674 dev_dbg(&pdev->dev, "Using intx interrupts\n");
a1b36958
XY		 675 ndev->db_count = 1;
676 ndev->msix_vec_count = 1;
677 return 0;
678
679err_intx_request:
680 return rc;
681}
682
683static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
684{
685 struct pci_dev *pdev;
686 void __iomem *mmio = ndev->self_mmio;
687 int i;
688
0f9bfb97 689 pdev = ndev->ntb.pdev;
a1b36958
XY		 690
691 /* Mask all doorbell interrupts */
692 ndev->db_mask = ndev->db_valid_mask;
693 writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
694
695 if (ndev->msix) {
696 i = ndev->msix_vec_count;
697 while (i--)
698 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
699 pci_disable_msix(pdev);
700 kfree(ndev->msix);
701 kfree(ndev->vec);
702 } else {
703 free_irq(pdev->irq, ndev);
704 if (pci_dev_msi_enabled(pdev))
705 pci_disable_msi(pdev);
706 else
707 pci_intx(pdev, 0);
708 }
709}
710
711static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
712 size_t count, loff_t *offp)
713{
714 struct amd_ntb_dev *ndev;
715 void __iomem *mmio;
716 char *buf;
717 size_t buf_size;
718 ssize_t ret, off;
719 union { u64 v64; u32 v32; u16 v16; } u;
720
721 ndev = filp->private_data;
722 mmio = ndev->self_mmio;
723
724 buf_size = min(count, 0x800ul);
725
726 buf = kmalloc(buf_size, GFP_KERNEL);
727 if (!buf)
728 return -ENOMEM;
729
730 off = 0;
731
732 off += scnprintf(buf + off, buf_size - off,
733 "NTB Device Information:\n");
734
735 off += scnprintf(buf + off, buf_size - off,
736 "Connection Topology -\t%s\n",
737 ntb_topo_string(ndev->ntb.topo));
738
739 off += scnprintf(buf + off, buf_size - off,
740 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
741
742 if (!amd_link_is_up(ndev)) {
743 off += scnprintf(buf + off, buf_size - off,
744 "Link Status -\t\tDown\n");
745 } else {
746 off += scnprintf(buf + off, buf_size - off,
747 "Link Status -\t\tUp\n");
748 off += scnprintf(buf + off, buf_size - off,
749 "Link Speed -\t\tPCI-E Gen %u\n",
750 NTB_LNK_STA_SPEED(ndev->lnk_sta));
751 off += scnprintf(buf + off, buf_size - off,
752 "Link Width -\t\tx%u\n",
753 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
754 }
755
756 off += scnprintf(buf + off, buf_size - off,
757 "Memory Window Count -\t%u\n", ndev->mw_count);
758 off += scnprintf(buf + off, buf_size - off,
759 "Scratchpad Count -\t%u\n", ndev->spad_count);
760 off += scnprintf(buf + off, buf_size - off,
761 "Doorbell Count -\t%u\n", ndev->db_count);
762 off += scnprintf(buf + off, buf_size - off,
763 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
764
765 off += scnprintf(buf + off, buf_size - off,
766 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
767
768 u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
769 off += scnprintf(buf + off, buf_size - off,
770 "Doorbell Mask -\t\t\t%#06x\n", u.v32);
771
772 u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
773 off += scnprintf(buf + off, buf_size - off,
774 "Doorbell Bell -\t\t\t%#06x\n", u.v32);
775
776 off += scnprintf(buf + off, buf_size - off,
777 "\nNTB Incoming XLAT:\n");
778
779 u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
780 off += scnprintf(buf + off, buf_size - off,
781 "XLAT1 -\t\t%#018llx\n", u.v64);
782
783 u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
784 off += scnprintf(buf + off, buf_size - off,
785 "XLAT23 -\t\t%#018llx\n", u.v64);
786
787 u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
788 off += scnprintf(buf + off, buf_size - off,
789 "XLAT45 -\t\t%#018llx\n", u.v64);
790
791 u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
792 off += scnprintf(buf + off, buf_size - off,
793 "LMT1 -\t\t\t%#06x\n", u.v32);
794
795 u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
796 off += scnprintf(buf + off, buf_size - off,
797 "LMT23 -\t\t\t%#018llx\n", u.v64);
798
799 u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
800 off += scnprintf(buf + off, buf_size - off,
801 "LMT45 -\t\t\t%#018llx\n", u.v64);
802
803 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
804 kfree(buf);
805 return ret;
806}
807
808static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
809{
810 if (!debugfs_dir) {
811 ndev->debugfs_dir = NULL;
812 ndev->debugfs_info = NULL;
813 } else {
814 ndev->debugfs_dir =
0f9bfb97
LG		 815 debugfs_create_dir(pci_name(ndev->ntb.pdev),
816 debugfs_dir);
a1b36958
XY		 817 if (!ndev->debugfs_dir)
818 ndev->debugfs_info = NULL;
819 else
820 ndev->debugfs_info =
821 debugfs_create_file("info", S_IRUSR,
822 ndev->debugfs_dir, ndev,
823 &amd_ntb_debugfs_info);
824 }
825}
826
827static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
828{
829 debugfs_remove_recursive(ndev->debugfs_dir);
830}
831
832static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
833 struct pci_dev *pdev)
834{
835 ndev->ntb.pdev = pdev;
836 ndev->ntb.topo = NTB_TOPO_NONE;
837 ndev->ntb.ops = &amd_ntb_ops;
838 ndev->int_mask = AMD_EVENT_INTMASK;
839 spin_lock_init(&ndev->db_mask_lock);
840}
841
842static int amd_poll_link(struct amd_ntb_dev *ndev)
843{
844 void __iomem *mmio = ndev->peer_mmio;
845 u32 reg, stat;
846 int rc;
847
848 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
849 reg &= NTB_LIN_STA_ACTIVE_BIT;
850
0f9bfb97 851 dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg);
a1b36958
XY		 852
853 if (reg == ndev->cntl_sta)
854 return 0;
855
856 ndev->cntl_sta = reg;
857
858 rc = pci_read_config_dword(ndev->ntb.pdev,
859 AMD_LINK_STATUS_OFFSET, &stat);
860 if (rc)
861 return 0;
862 ndev->lnk_sta = stat;
863
864 return 1;
865}
866
867static void amd_link_hb(struct work_struct *work)
868{
869 struct amd_ntb_dev *ndev = hb_ndev(work);
870
871 if (amd_poll_link(ndev))
872 ntb_link_event(&ndev->ntb);
873
874 if (!amd_link_is_up(ndev))
875 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
876}
877
878static int amd_init_isr(struct amd_ntb_dev *ndev)
879{
880 return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
881}
882
883static void amd_init_side_info(struct amd_ntb_dev *ndev)
884{
885 void __iomem *mmio = ndev->self_mmio;
886 unsigned int reg;
887
888 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
889 if (!(reg & AMD_SIDE_READY)) {
890 reg |= AMD_SIDE_READY;
891 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
892 }
893}
894
895static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
896{
897 void __iomem *mmio = ndev->self_mmio;
898 unsigned int reg;
899
900 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
901 if (reg & AMD_SIDE_READY) {
902 reg &= ~AMD_SIDE_READY;
903 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
904 readl(mmio + AMD_SIDEINFO_OFFSET);
905 }
906}
907
908static int amd_init_ntb(struct amd_ntb_dev *ndev)
909{
910 void __iomem *mmio = ndev->self_mmio;
911
912 ndev->mw_count = AMD_MW_CNT;
913 ndev->spad_count = AMD_SPADS_CNT;
914 ndev->db_count = AMD_DB_CNT;
915
916 switch (ndev->ntb.topo) {
917 case NTB_TOPO_PRI:
918 case NTB_TOPO_SEC:
919 ndev->spad_count >>= 1;
920 if (ndev->ntb.topo == NTB_TOPO_PRI) {
921 ndev->self_spad = 0;
922 ndev->peer_spad = 0x20;
923 } else {
924 ndev->self_spad = 0x20;
925 ndev->peer_spad = 0;
926 }
927
928 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
929 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
930
931 break;
932 default:
0f9bfb97
LG		 933 dev_err(&ndev->ntb.pdev->dev,
934 "AMD NTB does not support B2B mode.\n");
a1b36958
XY		 935 return -EINVAL;
936 }
937
938 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
939
940 /* Mask event interrupts */
941 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
942
943 return 0;
944}
945
946static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
947{
948 void __iomem *mmio = ndev->self_mmio;
949 u32 info;
950
951 info = readl(mmio + AMD_SIDEINFO_OFFSET);
952 if (info & AMD_SIDE_MASK)
953 return NTB_TOPO_SEC;
954 else
955 return NTB_TOPO_PRI;
956}
957
958static int amd_init_dev(struct amd_ntb_dev *ndev)
959{
960 struct pci_dev *pdev;
961 int rc = 0;
962
0f9bfb97 963 pdev = ndev->ntb.pdev;
a1b36958
XY		 964
965 ndev->ntb.topo = amd_get_topo(ndev);
0f9bfb97 966 dev_dbg(&pdev->dev, "AMD NTB topo is %s\n",
a1b36958
XY		 967 ntb_topo_string(ndev->ntb.topo));
968
969 rc = amd_init_ntb(ndev);
970 if (rc)
971 return rc;
972
973 rc = amd_init_isr(ndev);
974 if (rc) {
0f9bfb97 975 dev_err(&pdev->dev, "fail to init isr.\n");
a1b36958
XY		 976 return rc;
977 }
978
979 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
980
981 return 0;
982}
983
984static void amd_deinit_dev(struct amd_ntb_dev *ndev)
985{
986 cancel_delayed_work_sync(&ndev->hb_timer);
987
988 ndev_deinit_isr(ndev);
989}
990
991static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
992 struct pci_dev *pdev)
993{
994 int rc;
995
996 pci_set_drvdata(pdev, ndev);
997
998 rc = pci_enable_device(pdev);
999 if (rc)
1000 goto err_pci_enable;
1001
1002 rc = pci_request_regions(pdev, NTB_NAME);
1003 if (rc)
1004 goto err_pci_regions;
1005
1006 pci_set_master(pdev);
1007
1008 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1009 if (rc) {
1010 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1011 if (rc)
1012 goto err_dma_mask;
0f9bfb97 1013 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
a1b36958
XY		 1014 }
1015
1016 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1017 if (rc) {
1018 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1019 if (rc)
1020 goto err_dma_mask;
0f9bfb97 1021 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
a1b36958
XY		 1022 }
1023
1024 ndev->self_mmio = pci_iomap(pdev, 0, 0);
1025 if (!ndev->self_mmio) {
1026 rc = -EIO;
1027 goto err_dma_mask;
1028 }
1029 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
1030
1031 return 0;
1032
1033err_dma_mask:
1034 pci_clear_master(pdev);
1035err_pci_regions:
1036 pci_disable_device(pdev);
1037err_pci_enable:
1038 pci_set_drvdata(pdev, NULL);
1039 return rc;
1040}
1041
1042static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
1043{
0f9bfb97 1044 struct pci_dev *pdev = ndev->ntb.pdev;
a1b36958
XY		 1045
1046 pci_iounmap(pdev, ndev->self_mmio);
1047
1048 pci_clear_master(pdev);
1049 pci_release_regions(pdev);
1050 pci_disable_device(pdev);
1051 pci_set_drvdata(pdev, NULL);
1052}
1053
1054static int amd_ntb_pci_probe(struct pci_dev *pdev,
1055 const struct pci_device_id *id)
1056{
1057 struct amd_ntb_dev *ndev;
1058 int rc, node;
1059
1060 node = dev_to_node(&pdev->dev);
1061
1062 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1063 if (!ndev) {
1064 rc = -ENOMEM;
1065 goto err_ndev;
1066 }
1067
1068 ndev_init_struct(ndev, pdev);
1069
1070 rc = amd_ntb_init_pci(ndev, pdev);
1071 if (rc)
1072 goto err_init_pci;
1073
1074 rc = amd_init_dev(ndev);
1075 if (rc)
1076 goto err_init_dev;
1077
1078 /* write side info */
1079 amd_init_side_info(ndev);
1080
1081 amd_poll_link(ndev);
1082
1083 ndev_init_debugfs(ndev);
1084
1085 rc = ntb_register_device(&ndev->ntb);
1086 if (rc)
1087 goto err_register;
1088
1089 dev_info(&pdev->dev, "NTB device registered.\n");
1090
1091 return 0;
1092
1093err_register:
1094 ndev_deinit_debugfs(ndev);
1095 amd_deinit_dev(ndev);
1096err_init_dev:
1097 amd_ntb_deinit_pci(ndev);
1098err_init_pci:
1099 kfree(ndev);
1100err_ndev:
1101 return rc;
1102}
1103
1104static void amd_ntb_pci_remove(struct pci_dev *pdev)
1105{
1106 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1107
1108 ntb_unregister_device(&ndev->ntb);
1109 ndev_deinit_debugfs(ndev);
1110 amd_deinit_side_info(ndev);
1111 amd_deinit_dev(ndev);
1112 amd_ntb_deinit_pci(ndev);
1113 kfree(ndev);
1114}
1115
1116static const struct file_operations amd_ntb_debugfs_info = {
1117 .owner = THIS_MODULE,
1118 .open = simple_open,
1119 .read = ndev_debugfs_read,
1120};
1121
1122static const struct pci_device_id amd_ntb_pci_tbl[] = {
1123 {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)},
1124 {0}
1125};
1126MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
1127
1128static struct pci_driver amd_ntb_pci_driver = {
1129 .name = KBUILD_MODNAME,
1130 .id_table = amd_ntb_pci_tbl,
1131 .probe = amd_ntb_pci_probe,
1132 .remove = amd_ntb_pci_remove,
1133};
1134
1135static int __init amd_ntb_pci_driver_init(void)
1136{
1137 pr_info("%s %s\n", NTB_DESC, NTB_VER);
1138
1139 if (debugfs_initialized())
1140 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1141
1142 return pci_register_driver(&amd_ntb_pci_driver);
1143}
1144module_init(amd_ntb_pci_driver_init);
1145
1146static void __exit amd_ntb_pci_driver_exit(void)
1147{
1148 pci_unregister_driver(&amd_ntb_pci_driver);
1149 debugfs_remove_recursive(debugfs_dir);
1150}
1151module_exit(amd_ntb_pci_driver_exit);
Ubuntu Artful kernel mirror